/[webpac2]/trunk/bin/isi-download-results.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/bin/isi-download-results.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1370 - (show annotations)
Sun Aug 12 12:38:51 2012 UTC (11 years, 7 months ago) by dpavlin
File MIME type: text/plain
File size: 7007 byte(s)
correctly save citedref pages
1 #!/usr/bin/perl
2
3 use warnings;
4 use strict;
5
6 use WWW::Mechanize;
7 use Data::Dump qw(dump);
8 use File::Path;
9 use Text::Unaccent;
10
11 # Advanced search syntax:
12 # http://images.isiknowledge.com/WOK46/help/WOS/h_advanced_examples.html
13
14 our $q = 'AD=Croatia';
15 #$q = 'AU=BRATKO, D';
16 $q = 'CA=BRATKO, D';
17 #$q = 'AD=(croat* OR hrvat*)';
18
19 my $range_size = 100;
20 my $overlap = 3; # between previous and this range
21
22 my $results = 0;
23 my $citations = 0;
24 my $cited_reference = 1; # html tables
25 my $citing_articles = 1; # as many files as cited articles
26
27 my $cites_by_year = 0;
28
29 my $max_cites = 5000; # ISI limit to get cites
30
31 $q = unac_string( 'utf-8', join(' ', @ARGV) ) if @ARGV;
32
33 warn "QUERY: $q\n";
34
35 our $mech = WWW::Mechanize->new(
36 autocheck => 0, # it dies in reference download with it!
37 cookie_jar => undef,
38 );
39
40 our $step = 0;
41 our @ranges;
42
43 my $dir = '/tmp/isi/';
44 #rmtree $dir if -e $dir;
45 mkdir $dir unless -d $dir;
46
47 sub save_mech {
48 my $path = shift;
49 $step++;
50 my $base_path = sprintf('%s/%04d', $dir,$step);
51 $path ||= $base_path;
52 $path .= $mech->{ct} =~ m{html}i ? '.html' : '.txt';
53 $mech->save_content( $path );
54 warn "# [$step] $path ", -s $path, " ", $mech->ct, "\n";
55 open(my $dump, '>', "$base_path.dump.txt");
56 $mech->dump_all($dump);
57 }
58
59 warn "# get session";
60 $mech->get( 'http://www.webofknowledge.com/?DestApp=WOS' );
61 save_mech;
62
63 sub search {
64 warn "# advanced serach";
65 $mech->follow_link( url_regex => qr/AdvancedSearch/ );
66 save_mech;
67
68 warn "# cookie_jar ", dump $mech->cookie_jar;
69
70 my $q_this = $q;
71
72 if ( @ranges ) {
73 $q_this .= ' AND (' . join(' OR ', map { "PY=$_" } @{ shift @ranges } ) . ')';
74 }
75
76 warn "# submit_form search: $q_this\n";
77 $mech->submit_form(
78 fields => {
79 'value(input1)' => $q_this,
80 },
81 );
82 save_mech;
83
84 warn "# summary";
85 $mech->follow_link( url_regex => qr/summary/ );
86 save_mech;
87 }
88
89 sub get_results {
90 my $desc = shift;
91 my $from = 1;
92
93 while ( 1 ) {
94
95 my $to = $from + $range_size;
96
97 warn "# submit_form results $from - $to\n";
98 save_mech;
99
100 $mech->submit_form(
101 form_name => 'output_form',
102 fields => {
103 'value(record_select_type)' => 'range',
104 markFrom => $from,
105 markTo => $to,
106
107 mark_from => $from,
108 mark_to => $to,
109 mark_id => 'WOS',
110
111 fields_selection => 'ACCESSION_NUM FUNDING SUBJECT_CATEGORY JCR_CATEGORY LANG IDS PAGEC SABBR CITREFC ISSN PUBINFO KEYWORDS CITTIMES ADDRS CONFERENCE_SPONSORS DOCTYPE ABSTRACT CONFERENCE_INFO SOURCE TITLE AUTHORS',
112 filters => 'ACCESSION_NUM FUNDING SUBJECT_CATEGORY JCR_CATEGORY LANG IDS PAGEC SABBR CITREFC ISSN PUBINFO KEYWORDS CITTIMES ADDRS CONFERENCE_SPONSORS DOCTYPE ABSTRACT CONFERENCE_INFO SOURCE TITLE AUTHORS CITREF',
113 fullrec_fields_option => 'CITREF',
114
115 save_options => 'fieldtagged',
116 format => 'saveToFile',
117
118 },
119 button => 'saveToFile',
120 );
121 save_mech;
122
123
124 if ( $mech->content =~ m{invalid API call} ) {
125 $mech->back;
126 last;
127 }
128
129 if ( $mech->content !~ m{Please wait while your request is processed} ) {
130 warn "WARNING: expecting processing request";
131 }
132
133
134 my $path = "/tmp/isi.$q.$from-$to";
135 $path .= '.' . $desc if $desc;
136
137 warn "save $from - $to into $path\n";
138 $mech->submit_form(
139 form_name => 'etsForm',
140 );
141 save_mech $path;
142
143 $from += $range_size - $overlap;
144
145 $mech->back;
146 $mech->back;
147 save_mech;
148 }
149 }
150
151
152 sub citations {
153 warn "# citation report";
154 $mech->follow_link( url_regex => qr/search_mode=CitationReport/ );
155 save_mech;
156
157 warn "view citing articles";
158 $mech->follow_link( url_regex => qr/search_mode=TotalCitingArticles/ );
159 save_mech;
160 }
161
162 sub years {
163 my $years_url = $mech->find_link( url_regex => qr/ra_name=/ );
164 if ( ! $years_url ) {
165 warn "W: can't find ra_name link\n";
166 return;
167 }
168 $years_url = $years_url->url_abs;
169 warn "## $years_url";
170 if ( $years_url !~ s{ra_name=\w+}{ra_name=PublicationYear} ) {
171 warn "W: no ra_name in $years_url\n";
172 return;
173 }
174 warn "# refine years (hidden by javascript)";
175 # warn "http://apps.isiknowledge.com/RAMore.do?product=WOS&search_mode=TotalCitingArticles&SID=T1o6bChdN9PGP1LN1Nh&qid=3&ra_mode=more&ra_name=PublicationYear&db_id=WOS&viewType=raMore\n$years_url\n";
176 $mech->get( $years_url );
177 save_mech;
178
179 my $html = $mech->content;
180 my $years;
181 while ( $html =~ s{<label.+?PublicationYear.+?>(\d{4})\s\(([\d,]+)\)</label>}{} ) {
182 my ( $year, $count ) = ( $1, $2 );
183 $count =~ s{,}{}g;
184 $years->{$year} = $count;
185 }
186 warn "# years ",dump $years;
187 $mech->back;
188
189 my @y = sort keys %$years;
190
191 @ranges = ();
192
193 if ( $cites_by_year ) {
194 push @ranges, [ $_ ] foreach @y;
195 warn "# cites_by_year ranges ", dump @ranges;
196 return;
197 }
198
199 my $y = shift @y;
200 my $size = $years->{$y};
201
202 my $cites_range;
203 $cites_range = [$y] if $y;
204
205 foreach my $y ( @y ) {
206 if ( $size + $years->{$y} > $max_cites ) {
207 push @ranges, $cites_range;
208 warn "# cites_range $size years ",dump( $cites_range ),$/;
209
210 $cites_range = [];
211 $size = 0;
212 }
213 $size += $years->{$y};
214 push @$cites_range, $y;
215 }
216
217 if ( $cites_range ) {
218 push @ranges, $cites_range;
219 warn "# cites_range $size years ",dump( $cites_range ), " FINAL\n"
220 }
221
222 warn '# ranges ', dump @ranges;
223 @ranges = () if $#ranges == 1; # just take all
224
225 return $years;
226 }
227
228
229 our $page = 1;
230 sub next_page {
231 $page++;
232 warn "next_page $page\n";
233
234 $mech->submit_form(
235 form_name => 'summary_navigation',
236 fields => {
237 'page' => $page,
238 },
239 );
240
241 save_mech;
242
243 $mech->form_name( 'summary_navigation' );
244 my $is_next_page = $mech->value('page') == $page;
245 warn "no next_page" unless $is_next_page;
246 return $is_next_page;
247 }
248
249 if ( $results ) {
250 search;
251 years;
252 get_results;
253 }
254
255 if ( $citations ) {
256
257 citations;
258 years unless @ranges;
259
260 do {
261 my $part;
262 if ( @ranges ) {
263 $part .= $ranges[0]->[0] . '.';
264 search;
265 citations;
266 }
267 $part .= 'citing';
268 get_results $part;
269 } while ( @ranges );
270
271 }
272
273
274
275 if ( $q =~ m{CA=(.+)} && $cited_reference ) {
276
277 my $CA = $1;
278
279 warn "# citated reference search";
280 $mech->follow_link( url_regex => qr/CitedReferenceSearch/ );
281 save_mech;
282
283
284 $mech->submit_form(
285 form_name => 'WOS_CitedReferenceSearch_input_form',
286 fields => {
287 'value(input1)' => $CA,
288 },
289 );
290
291 my $page = 1;
292 my $records = $1 if $mech->content =~ m/(\d+)\s+records/;
293 warn "# found $records records\n";
294 my $last_span = 'fake';
295
296 while (1) {
297 save_mech "/tmp/isi.$q.citedref.$page";
298
299 last unless next_page();
300 $page++;
301
302 if ( $mech->content =~ m/(\d+\s*-\s*(\d+))/ ) {
303 warn "span: $1\n";
304 last if $2 == $records;
305 last if $1 == $last_span;
306 $last_span = $1;
307 } elsif ( $page > 5 ) {
308 warn "ARTIFICALLY LIMITED TO 5 PAGES WITHOUT VALID SPAN!";
309 last;
310 }
311
312 }
313
314 }
315
316 if ( $q =~ m{CA=(.+)} && $citing_articles ) {
317
318 search;
319
320 my $orig_q = $q;
321 my $nr = 0;
322
323 do {
324
325 foreach my $link ( $mech->find_all_links( url_regex => qr/CitingArticles.do/ ) ) {
326 $nr++;
327 warn "link $nr\n";
328 $mech->get( $link->url );
329 save_mech;
330 $q = $orig_q . '.citing_article.' . $nr;
331 get_results;
332 $mech->back;
333 $mech->back;
334
335 #last if $nr > 3; # FIXME only for development
336 }
337
338 } while next_page;
339
340 $q = $orig_q;
341 }
342
343 warn "OVER\n";

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26