trunk/bin/isi-download-results.pl

#!/usr/bin/perl

use warnings;
use strict;

our $q = 'AD=Croatia';
my $range_size = 500;

my $dump = @ARGV ? 1 : 0;

$q = 'TS=psychology AND AD=Croatia';

use WWW::Mechanize;
use Data::Dump qw(dump);
use File::Path;

our $mech = WWW::Mechanize->new(
        autocheck => 1,
        cookie_jar => undef,
);

our $step = 0;

my $dir = '/tmp/isi/';
rmtree $dir if -e $dir;
mkdir $dir;

sub save_mech {
        my ( $mech, $path ) = @_;
        $step++;
        my $base_path = sprintf('%s/%04d', $dir,$step);
        $path ||= $base_path . ( $mech->{ct} =~ m{html}i ? '.html' : '.txt' );
        $mech->save_content( $path );
        warn "# [$step] $path ", -s $path, " ", $mech->ct, "\n";
        open(my $dump, '>', "$base_path.dump.txt");
        $mech->dump_all($dump);
}

warn "# get session";
$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
save_mech $mech;

sub search {
        warn "# advanced serach";
        $mech->follow_link( url_regex => qr/AdvancedSearch/ );
        save_mech $mech;

        warn "# cookie_jar ", dump $mech->cookie_jar;

        $mech->submit_form(
                fields => {
                        'value(input1)' => $q,
                }
        );
        save_mech $mech;

        warn "# summary";
        $mech->follow_link( url_regex => qr/summary/ );
        save_mech $mech;
}

sub get_results {
        my $from = 1;

        while ( 1 ) {

                my $to = $from + $range_size;

                $mech->submit_form(
                        form_name => 'summary_output_form',
                        fields => {
                                record_select_type => 'range',
                                mark_from => $from,
                                mark_to => $to,
                                mark_id => 'WOS',

                                qo_fields => 'fullrecord',
                                citedref => 'citedref',

                                save_options => 'plain_text',

                                fields => 'Full',
                                format => 'save',
                        },
                        button => 'save',
                );
                save_mech $mech;

                if ( $mech->content =~ m{invalid API call} ) {
                        $mech->back;
                        last;
                }

                warn "range $from - $to [$q]\n";
                $mech->follow_link( url_regex => qr/save_file/ );
                save_mech $mech => "/tmp/isi.$q.$from-$to.txt";

                $from += $range_size;

                $mech->back;
                $mech->back;
                #save_mech $mech;

        }

}


sub citations {
        save_mech $mech;
        warn "# citation report";
        $mech->follow_link( url_regex => qr/search_mode=CitationReport/ );
        save_mech $mech;

        warn "view citing articles";
        $mech->follow_link( url_regex => qr/search_mode=TotalCitingArticles/ );
        save_mech $mech;
}

sub years {
        my $years_url = $mech->find_link( text_regex => qr/more options/ );
        if ( ! $years_url ) {
                warn "W: can't find years\n";
                return;
        }
        $years_url = $years_url->url_abs;
        warn "## $years_url";
        if ( $years_url !~ s{ra_name=\w+}{ra_name=PublicationYear} ) {
                warn "W: no ra_name\n";
                return;
        }
        warn "# refine years (hidden by javascript)";
        warn "http://apps.isiknowledge.com/RAMore.do?product=WOS&search_mode=TotalCitingArticles&SID=T1o6bChdN9PGP1LN1Nh&qid=3&ra_mode=more&ra_name=PublicationYear&db_id=WOS&viewType=raMore\n$years_url\n";
        $mech->get( $years_url );
        save_mech $mech;

        my $html = $mech->content;
        my $years;
        while ( $html =~ s{>(\d\d\d\d)\s\((\d+)\)</label.+?value="PublicationYear_}{} ) {
                $years->{$1} = $2;
        }
        warn "# years ",dump $years;
        $mech->back;
        return $years;
}

search;
years;
get_results;


citations;
years;
$q .= '.citing';
get_results;

1	dpavlin	1280	#!/usr/bin/perl
2
3			use warnings;
4			use strict;
5
6	dpavlin	1291	our $q = 'AD=Croatia';
7	dpavlin	1289	my $range_size = 500;
8	dpavlin	1280
9	dpavlin	1282	my $dump = @ARGV ? 1 : 0;
10
11	dpavlin	1289	$q = 'TS=psychology AND AD=Croatia';
12	dpavlin	1282
13	dpavlin	1280	use WWW::Mechanize;
14	dpavlin	1291	use Data::Dump qw(dump);
15			use File::Path;
16	dpavlin	1280
17	dpavlin	1289	our $mech = WWW::Mechanize->new(
18	dpavlin	1280	autocheck => 1,
19			cookie_jar => undef,
20			);
21
22	dpavlin	1281	our $step = 0;
23	dpavlin	1280
24	dpavlin	1291	my $dir = '/tmp/isi/';
25			rmtree $dir if -e $dir;
26			mkdir $dir;
27
28	dpavlin	1280	sub save_mech {
29	dpavlin	1281	my ( $mech, $path ) = @_;
30			$step++;
31	dpavlin	1291	my $base_path = sprintf('%s/%04d', $dir,$step);
32	dpavlin	1290	$path \|\|= $base_path . ( $mech->{ct} =~ m{html}i ? '.html' : '.txt' );
33	dpavlin	1281	$mech->save_content( $path );
34	dpavlin	1290	warn "# [$step] $path ", -s $path, " ", $mech->ct, "\n";
35			open(my $dump, '>', "$base_path.dump.txt");
36			$mech->dump_all($dump);
37	dpavlin	1280	}
38
39			warn "# get session";
40			$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
41			save_mech $mech;
42
43	dpavlin	1290	sub search {
44			warn "# advanced serach";
45			$mech->follow_link( url_regex => qr/AdvancedSearch/ );
46			save_mech $mech;
47	dpavlin	1280
48	dpavlin	1290	warn "# cookie_jar ", dump $mech->cookie_jar;
49	dpavlin	1280
50	dpavlin	1290	$mech->submit_form(
51			fields => {
52			'value(input1)' => $q,
53			}
54			);
55			save_mech $mech;
56	dpavlin	1280
57	dpavlin	1290	warn "# summary";
58			$mech->follow_link( url_regex => qr/summary/ );
59			save_mech $mech;
60			}
61
62	dpavlin	1289	sub get_results {
63			my $from = 1;
64	dpavlin	1280
65	dpavlin	1289	while ( 1 ) {
66	dpavlin	1280
67	dpavlin	1289	my $to = $from + $range_size;
68	dpavlin	1280
69	dpavlin	1289	$mech->submit_form(
70			form_name => 'summary_output_form',
71			fields => {
72			record_select_type => 'range',
73			mark_from => $from,
74			mark_to => $to,
75			mark_id => 'WOS',
76	dpavlin	1280
77	dpavlin	1289	qo_fields => 'fullrecord',
78			citedref => 'citedref',
79	dpavlin	1280
80	dpavlin	1289	save_options => 'plain_text',
81	dpavlin	1280
82	dpavlin	1289	fields => 'Full',
83			format => 'save',
84			},
85			button => 'save',
86			);
87			save_mech $mech;
88	dpavlin	1281
89	dpavlin	1289	if ( $mech->content =~ m{invalid API call} ) {
90			$mech->back;
91	dpavlin	1290	last;
92	dpavlin	1289	}
93	dpavlin	1281
94	dpavlin	1290	warn "range $from - $to [$q]\n";
95	dpavlin	1289	$mech->follow_link( url_regex => qr/save_file/ );
96			save_mech $mech => "/tmp/isi.$q.$from-$to.txt";
97	dpavlin	1281
98	dpavlin	1289	$from += $range_size;
99	dpavlin	1281
100	dpavlin	1289	$mech->back;
101			$mech->back;
102			#save_mech $mech;
103	dpavlin	1281
104	dpavlin	1290	}
105	dpavlin	1289
106	dpavlin	1281	}
107	dpavlin	1289
108
109	dpavlin	1290	sub citations {
110			save_mech $mech;
111			warn "# citation report";
112			$mech->follow_link( url_regex => qr/search_mode=CitationReport/ );
113			save_mech $mech;
114	dpavlin	1289
115	dpavlin	1290	warn "view citing articles";
116			$mech->follow_link( url_regex => qr/search_mode=TotalCitingArticles/ );
117			save_mech $mech;
118			}
119	dpavlin	1289
120	dpavlin	1290	sub years {
121	dpavlin	1291	my $years_url = $mech->find_link( text_regex => qr/more options/ );
122			if ( ! $years_url ) {
123			warn "W: can't find years\n";
124			return;
125			}
126			$years_url = $years_url->url_abs;
127	dpavlin	1290	warn "## $years_url";
128	dpavlin	1291	if ( $years_url !~ s{ra_name=\w+}{ra_name=PublicationYear} ) {
129			warn "W: no ra_name\n";
130			return;
131			}
132	dpavlin	1290	warn "# refine years (hidden by javascript)";
133			warn "http://apps.isiknowledge.com/RAMore.do?product=WOS&search_mode=TotalCitingArticles&SID=T1o6bChdN9PGP1LN1Nh&qid=3&ra_mode=more&ra_name=PublicationYear&db_id=WOS&viewType=raMore\n$years_url\n";
134			$mech->get( $years_url );
135			save_mech $mech;
136
137			my $html = $mech->content;
138	dpavlin	1291	my $years;
139	dpavlin	1290	while ( $html =~ s{>(\d\d\d\d)\s\((\d+)\)</label.+?value="PublicationYear_}{} ) {
140	dpavlin	1291	$years->{$1} = $2;
141	dpavlin	1290	}
142	dpavlin	1291	warn "# years ",dump $years;
143	dpavlin	1290	$mech->back;
144	dpavlin	1291	return $years;
145	dpavlin	1290	}
146
147	dpavlin	1291	search;
148	dpavlin	1290	years;
149	dpavlin	1291	get_results;
150	dpavlin	1290
151	dpavlin	1291
152	dpavlin	1290	citations;
153			years;
154	dpavlin	1291	$q .= '.citing';
155			get_results;
156	dpavlin	1289