trunk/bin/isi-download-results.pl

#!/usr/bin/perl

use warnings;
use strict;

my $q = 'TS=psychology AND AD=Croatia';
my $range_size = 50;

my $dump = @ARGV ? 1 : 0;

$q = 'AD=Croatia';
$range_size = 500;

use WWW::Mechanize;
use Data::Dump qw/dump/;

my $mech = WWW::Mechanize->new(
        autocheck => 1,
        cookie_jar => undef,
);

our $step = 0;

sub save_mech {
        my ( $mech, $path ) = @_;
        $step++;
        $path ||= sprintf('/tmp/isi.%02d.%s', $step, $mech->{ct} =~ m{html}i ? 'html' : 'txt' );
        $mech->save_content( $path );
        warn "# [$step] $path ", -s $path, " ", $mech->ct;
        $mech->dump_all if $dump;
}

warn "# get session";
$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
save_mech $mech;

warn "# advanced serach";
$mech->follow_link( url_regex => qr/AdvancedSearch/ );
save_mech $mech;

warn "# cookie_jar ", dump $mech->cookie_jar;

$mech->submit_form(
        fields => {
                'value(input1)' => $q,
        }
);
save_mech $mech;

warn "# summary";
$mech->follow_link( url_regex => qr/summary/ );
save_mech $mech;

my $from = 1;

while ( $from ) {

        my $to = $from + $range_size;

        $mech->submit_form(
                form_name => 'summary_output_form',
                fields => {
                        record_select_type => 'range',
                        mark_from => $from,
                        mark_to => $to,
                        mark_id => 'WOS',

                        qo_fields => 'fullrecord',
                        citedref => 'citedref',

                        save_options => 'plain_text',

                        fields => 'Full',
                        format => 'save',
                },
                button => 'save',
        );
        save_mech $mech;

        last if $mech->content =~ m{invalid API call};

        warn "# save_file";
        $mech->follow_link( url_regex => qr/save_file/ );
        save_mech $mech => "/tmp/isi.$q.$from-$to.txt";

        $from += $range_size;

        $mech->back;
        $mech->back;
        #save_mech $mech;

}
1	dpavlin	1280	#!/usr/bin/perl
2
3			use warnings;
4			use strict;
5
6			my $q = 'TS=psychology AND AD=Croatia';
7	dpavlin	1282	my $range_size = 50;
8	dpavlin	1280
9	dpavlin	1282	my $dump = @ARGV ? 1 : 0;
10
11			$q = 'AD=Croatia';
12			$range_size = 500;
13
14	dpavlin	1280	use WWW::Mechanize;
15			use Data::Dump qw/dump/;
16
17			my $mech = WWW::Mechanize->new(
18			autocheck => 1,
19			cookie_jar => undef,
20			);
21
22	dpavlin	1281	our $step = 0;
23	dpavlin	1280
24			sub save_mech {
25	dpavlin	1281	my ( $mech, $path ) = @_;
26			$step++;
27			$path \|\|= sprintf('/tmp/isi.%02d.%s', $step, $mech->{ct} =~ m{html}i ? 'html' : 'txt' );
28			$mech->save_content( $path );
29			warn "# [$step] $path ", -s $path, " ", $mech->ct;
30	dpavlin	1282	$mech->dump_all if $dump;
31	dpavlin	1280	}
32
33			warn "# get session";
34			$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
35			save_mech $mech;
36
37			warn "# advanced serach";
38			$mech->follow_link( url_regex => qr/AdvancedSearch/ );
39			save_mech $mech;
40
41			warn "# cookie_jar ", dump $mech->cookie_jar;
42
43			$mech->submit_form(
44			fields => {
45			'value(input1)' => $q,
46			}
47			);
48			save_mech $mech;
49
50			warn "# summary";
51			$mech->follow_link( url_regex => qr/summary/ );
52			save_mech $mech;
53
54			my $from = 1;
55
56	dpavlin	1281	while ( $from ) {
57	dpavlin	1280
58	dpavlin	1281	my $to = $from + $range_size;
59	dpavlin	1280
60	dpavlin	1281	$mech->submit_form(
61			form_name => 'summary_output_form',
62			fields => {
63			record_select_type => 'range',
64			mark_from => $from,
65			mark_to => $to,
66			mark_id => 'WOS',
67	dpavlin	1280
68	dpavlin	1281	qo_fields => 'fullrecord',
69			citedref => 'citedref',
70	dpavlin	1280
71	dpavlin	1281	save_options => 'plain_text',
72	dpavlin	1280
73	dpavlin	1281	fields => 'Full',
74			format => 'save',
75			},
76			button => 'save',
77			);
78			save_mech $mech;
79
80			last if $mech->content =~ m{invalid API call};
81
82			warn "# save_file";
83			$mech->follow_link( url_regex => qr/save_file/ );
84			save_mech $mech => "/tmp/isi.$q.$from-$to.txt";
85
86			$from += $range_size;
87
88			$mech->back;
89			$mech->back;
90			#save_mech $mech;
91
92			}