trunk/bin/isi-download-results.pl

#!/usr/bin/perl

use warnings;
use strict;

my $q = 'TS=psychology AND AD=Croatia';
my $range_size = 50;

my $dump = @ARGV ? 1 : 0;

$q = 'AD=Croatia';
$range_size = 500;

use WWW::Mechanize;
use Data::Dump qw/dump/;

my $mech = WWW::Mechanize->new(
        autocheck => 1,
        cookie_jar => undef,
);

our $step = 0;

sub save_mech {
        my ( $mech, $path ) = @_;
        $step++;
        $path ||= sprintf('/tmp/isi.%02d.%s', $step, $mech->{ct} =~ m{html}i ? 'html' : 'txt' );
        $mech->save_content( $path );
        warn "# [$step] $path ", -s $path, " ", $mech->ct;
        $mech->dump_all if $dump;
}

warn "# get session";
$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
save_mech $mech;

warn "# advanced serach";
$mech->follow_link( url_regex => qr/AdvancedSearch/ );
save_mech $mech;

warn "# cookie_jar ", dump $mech->cookie_jar;

$mech->submit_form(
        fields => {
                'value(input1)' => $q,
        }
);
save_mech $mech;

warn "# summary";
$mech->follow_link( url_regex => qr/summary/ );
save_mech $mech;

my $from = 1;

while ( $from ) {

        my $to = $from + $range_size;

        $mech->submit_form(
                form_name => 'summary_output_form',
                fields => {
                        record_select_type => 'range',
                        mark_from => $from,
                        mark_to => $to,
                        mark_id => 'WOS',

                        qo_fields => 'fullrecord',
                        citedref => 'citedref',

                        save_options => 'plain_text',

                        fields => 'Full',
                        format => 'save',
                },
                button => 'save',
        );
        save_mech $mech;

        last if $mech->content =~ m{invalid API call};

        warn "# save_file";
        $mech->follow_link( url_regex => qr/save_file/ );
        save_mech $mech => "/tmp/isi.$q.$from-$to.txt";

        $from += $range_size;

        $mech->back;
        $mech->back;
        #save_mech $mech;

}
1	#!/usr/bin/perl
2
3	use warnings;
4	use strict;
5
6	my $q = 'TS=psychology AND AD=Croatia';
7	my $range_size = 50;
8
9	my $dump = @ARGV ? 1 : 0;
10
11	$q = 'AD=Croatia';
12	$range_size = 500;
13
14	use WWW::Mechanize;
15	use Data::Dump qw/dump/;
16
17	my $mech = WWW::Mechanize->new(
18	autocheck => 1,
19	cookie_jar => undef,
20	);
21
22	our $step = 0;
23
24	sub save_mech {
25	my ( $mech, $path ) = @_;
26	$step++;
27	$path \|\|= sprintf('/tmp/isi.%02d.%s', $step, $mech->{ct} =~ m{html}i ? 'html' : 'txt' );
28	$mech->save_content( $path );
29	warn "# [$step] $path ", -s $path, " ", $mech->ct;
30	$mech->dump_all if $dump;
31	}
32
33	warn "# get session";
34	$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
35	save_mech $mech;
36
37	warn "# advanced serach";
38	$mech->follow_link( url_regex => qr/AdvancedSearch/ );
39	save_mech $mech;
40
41	warn "# cookie_jar ", dump $mech->cookie_jar;
42
43	$mech->submit_form(
44	fields => {
45	'value(input1)' => $q,
46	}
47	);
48	save_mech $mech;
49
50	warn "# summary";
51	$mech->follow_link( url_regex => qr/summary/ );
52	save_mech $mech;
53
54	my $from = 1;
55
56	while ( $from ) {
57
58	my $to = $from + $range_size;
59
60	$mech->submit_form(
61	form_name => 'summary_output_form',
62	fields => {
63	record_select_type => 'range',
64	mark_from => $from,
65	mark_to => $to,
66	mark_id => 'WOS',
67
68	qo_fields => 'fullrecord',
69	citedref => 'citedref',
70
71	save_options => 'plain_text',
72
73	fields => 'Full',
74	format => 'save',
75	},
76	button => 'save',
77	);
78	save_mech $mech;
79
80	last if $mech->content =~ m{invalid API call};
81
82	warn "# save_file";
83	$mech->follow_link( url_regex => qr/save_file/ );
84	save_mech $mech => "/tmp/isi.$q.$from-$to.txt";
85
86	$from += $range_size;
87
88	$mech->back;
89	$mech->back;
90	#save_mech $mech;
91
92	}