trunk/bin/isi-download-results.pl

#!/usr/bin/perl

use warnings;
use strict;

my $q = 'TS=psychology AND AD=Croatia';

use WWW::Mechanize;
use Data::Dump qw/dump/;

my $mech = WWW::Mechanize->new(
        autocheck => 1,
        cookie_jar => undef,
);

our $step = 0;

sub save_mech {
        my ( $mech, $path ) = @_;
        $step++;
        $path ||= sprintf('/tmp/isi.%02d.%s', $step, $mech->{ct} =~ m{html}i ? 'html' : 'txt' );
        $mech->save_content( $path );
        warn "# [$step] $path ", -s $path, " ", $mech->ct;
        $mech->dump_all;
}

warn "# get session";
$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
save_mech $mech;

warn "# advanced serach";
$mech->follow_link( url_regex => qr/AdvancedSearch/ );
save_mech $mech;

warn "# cookie_jar ", dump $mech->cookie_jar;

$mech->submit_form(
        fields => {
                'value(input1)' => $q,
        }
);
save_mech $mech;

warn "# summary";
$mech->follow_link( url_regex => qr/summary/ );
save_mech $mech;

my $from = 1;
my $range_size = 50;

while ( $from ) {

        my $to = $from + $range_size;

        $mech->submit_form(
                form_name => 'summary_output_form',
                fields => {
                        record_select_type => 'range',
                        mark_from => $from,
                        mark_to => $to,
                        mark_id => 'WOS',

                        qo_fields => 'fullrecord',
                        citedref => 'citedref',

                        save_options => 'plain_text',

                        fields => 'Full',
                        format => 'save',
                },
                button => 'save',
        );
        save_mech $mech;

        last if $mech->content =~ m{invalid API call};

        warn "# save_file";
        $mech->follow_link( url_regex => qr/save_file/ );
        save_mech $mech => "/tmp/isi.$q.$from-$to.txt";

        $from += $range_size;

        $mech->back;
        $mech->back;
        #save_mech $mech;

}
1	dpavlin	1280	#!/usr/bin/perl
2
3			use warnings;
4			use strict;
5
6			my $q = 'TS=psychology AND AD=Croatia';
7
8			use WWW::Mechanize;
9			use Data::Dump qw/dump/;
10
11			my $mech = WWW::Mechanize->new(
12			autocheck => 1,
13			cookie_jar => undef,
14			);
15
16	dpavlin	1281	our $step = 0;
17	dpavlin	1280
18			sub save_mech {
19	dpavlin	1281	my ( $mech, $path ) = @_;
20			$step++;
21			$path \|\|= sprintf('/tmp/isi.%02d.%s', $step, $mech->{ct} =~ m{html}i ? 'html' : 'txt' );
22			$mech->save_content( $path );
23			warn "# [$step] $path ", -s $path, " ", $mech->ct;
24	dpavlin	1280	$mech->dump_all;
25			}
26
27			warn "# get session";
28			$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
29			save_mech $mech;
30
31			warn "# advanced serach";
32			$mech->follow_link( url_regex => qr/AdvancedSearch/ );
33			save_mech $mech;
34
35			warn "# cookie_jar ", dump $mech->cookie_jar;
36
37			$mech->submit_form(
38			fields => {
39			'value(input1)' => $q,
40			}
41			);
42			save_mech $mech;
43
44			warn "# summary";
45			$mech->follow_link( url_regex => qr/summary/ );
46			save_mech $mech;
47
48			my $from = 1;
49	dpavlin	1281	my $range_size = 50;
50	dpavlin	1280
51	dpavlin	1281	while ( $from ) {
52	dpavlin	1280
53	dpavlin	1281	my $to = $from + $range_size;
54	dpavlin	1280
55	dpavlin	1281	$mech->submit_form(
56			form_name => 'summary_output_form',
57			fields => {
58			record_select_type => 'range',
59			mark_from => $from,
60			mark_to => $to,
61			mark_id => 'WOS',
62	dpavlin	1280
63	dpavlin	1281	qo_fields => 'fullrecord',
64			citedref => 'citedref',
65	dpavlin	1280
66	dpavlin	1281	save_options => 'plain_text',
67	dpavlin	1280
68	dpavlin	1281	fields => 'Full',
69			format => 'save',
70			},
71			button => 'save',
72			);
73			save_mech $mech;
74
75			last if $mech->content =~ m{invalid API call};
76
77			warn "# save_file";
78			$mech->follow_link( url_regex => qr/save_file/ );
79			save_mech $mech => "/tmp/isi.$q.$from-$to.txt";
80
81			$from += $range_size;
82
83			$mech->back;
84			$mech->back;
85			#save_mech $mech;
86
87			}