/[webpac2]/trunk/run.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/run.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1184 - (show annotations)
Mon May 18 18:51:40 2009 UTC (14 years, 10 months ago) by dpavlin
File MIME type: text/plain
File size: 19217 byte(s)
create out/report and out/marc if they doesn't exist

1 #!/usr/bin/perl -w
2
3 use strict;
4
5 use Cwd qw/abs_path/;
6 use File::Temp qw/tempdir/;
7 use lib './lib';
8
9 use WebPAC::Common 0.02;
10 use WebPAC::Parser 0.08;
11 use WebPAC::Input 0.16;
12 use WebPAC::Store 0.15;
13 use WebPAC::Normalize 0.22;
14 #use WebPAC::Output::TT;
15 use WebPAC::Validate 0.11;
16 use WebPAC::Output::MARC;
17 use WebPAC::Config;
18 use Getopt::Long;
19 use File::Path;
20 use Time::HiRes qw/time/;
21 use File::Slurp;
22 use Data::Dump qw/dump/;
23 use Storable qw/dclone/;
24 use Pod::Usage qw/pod2usage/;
25 use LWP::Simple;
26
27 use POSIX ":sys_wait_h"; # imports WNOHANG
28
29 =head1 NAME
30
31 run.pl - start WebPAC indexing
32
33 B<this command will probably go away. Don't get used to it!>
34
35 =head1 OPTIONS
36
37 =over 4
38
39 =item --offset 42
40
41 start loading (all) databases at offset 42
42
43 =item --limit 100
44
45 limit loading to 100 records
46
47 =item --clean
48
49 remove database and Hyper Estraier index before indexing
50
51 =item --only=database_name/input_filter
52
53 reindex just single database (legacy name is --one)
54
55 C</input_filter> is optional part which can be C<name>
56 or C<type> from input
57
58 =item --config conf/config.yml
59
60 path to YAML configuration file
61
62 =item --stats
63
64 disable indexing, modify_* in configuration and dump statistics about field
65 and subfield usage for each input
66
67 =item --validate path/to/validation_file
68
69 turn on extra validation of imput records, see L<WebPAC::Validation>
70
71 You can use special variables C<$database> and $C<$input> in this parametar
72 like C<--validate 'conf/validate/$database-$input'> to construct filename
73
74 =item --validate-delimiters path/to/validate_delimiters_file
75
76 this option is used with C<--validate> to turn on extra validation of
77 delimiters. If file is non existant, it will be created on first run.
78
79 =item --marc-generate
80
81 Generate MARC file. This will automatically be on if file contains C<marc*> directives.
82 You can use this option as C<--no-marc-generate> to disable MARC generation.
83
84 =item --marc-lint
85
86 By default turned on if normalisation file has C<marc*> directives. You can disable lint
87 messages with C<--no-marc-lint>.
88
89 =item --marc-dump
90
91 Force dump or input and marc record for debugging.
92
93 =item --parallel 4
94
95 Run databases in parallel (aproximatly same as number of processors in
96 machine if you want to use full load)
97
98 =item --only-links
99
100 Create just links
101
102 =item --merge
103
104 Create merged index of databases which have links
105
106 =item --mirror http://www.example.com
107
108 Tries to download input path files from mirror URI
109
110 =back
111
112 =cut
113
114 my $offset;
115 my $limit;
116
117 my $clean = 0;
118 my $config_path;
119 my $debug = 0;
120 my $only_filter;
121 my $stats = 0;
122 my $validate_path;
123 my $validate_delimiters_path;
124 my $marc_generate = 1;
125 my $marc_lint = 1;
126 my $marc_dump = 0;
127 my $parallel = 0;
128 my $only_links = 0;
129 my $merge = 0;
130 my $mirror;
131 my $help;
132
133 my $log = _new WebPAC::Common()->_get_logger();
134
135 GetOptions(
136 "limit=i" => \$limit,
137 "offset=i" => \$offset,
138 "clean" => \$clean,
139 "one=s" => \$only_filter,
140 "only=s" => \$only_filter,
141 "config=s" => \$config_path,
142 "debug+" => \$debug,
143 "stats" => \$stats,
144 "validate=s" => \$validate_path,
145 "validate-delimiters=s" => \$validate_delimiters_path,
146 "marc-generate!" => \$marc_generate,
147 "marc-lint!" => \$marc_lint,
148 "marc-dump!" => \$marc_dump,
149 "parallel=i" => \$parallel,
150 "only-links!" => \$only_links,
151 "merge" => \$merge,
152 "mirror=s" => \$mirror,
153 "help" => \$help,
154 );
155
156 $marc_generate = 0 if ( $validate_delimiters_path );
157
158 pod2usage(-verbose => 2) if ($help);
159
160 my $config = new WebPAC::Config( path => $config_path );
161
162 WebPAC::Normalize::_debug( $debug - 1 ) if $debug > 1;
163
164 #print "config = ",dump($config) if ($debug);
165
166 die "no databases in config file!\n" unless ($config->databases);
167
168 $log->info( "-" x 79 );
169
170 my $log_file = 'log';
171
172 if (-e $log_file ) { # && -s $log_file > 5 * 1024 * 1024) {
173 $log->info("moved old log with ", -s $log_file, " bytes to '${log_file}.old'");
174 rename $log_file, "${log_file}.old" || $log->logwarn("can't rename $log_file to ${log_file}.old: $!");
175 }
176
177 my $estcmd_fh;
178 my $estcmd_path = './estcmd-merge.sh';
179 if ($merge) {
180 open($estcmd_fh, '>', $estcmd_path) || $log->logdie("can't open $estcmd_path: $!");
181 print $estcmd_fh 'cd /data/estraier/_node/ || exit 1',$/;
182 print $estcmd_fh 'sudo /etc/init.d/hyperestraier stop',$/;
183 $log->info("created merge batch file $estcmd_path");
184 }
185
186 my $validate;
187 $validate = new WebPAC::Validate(
188 delimiters => $config->webpac('delimiters'),
189 ) if ($validate_path || $validate_delimiters_path);
190
191 my $use_indexer = $config->use_indexer;
192 $stats ||= $validate;
193 if ($stats) {
194 $log->debug("disabled indexing for stats collection");
195 $use_indexer = undef;
196 } elsif ( $use_indexer ) {
197 $log->info("using $use_indexer indexing engine...");
198 }
199
200 # parse normalize files and create source files for lookup and normalization
201
202 my ($only_database,$only_input) = split(m#/#, $only_filter) if $only_filter;
203
204 my $parser = new WebPAC::Parser(
205 config => $config,
206 only_database => $only_database,
207 only_input => $only_input,
208 );
209
210 my $total_rows = 0;
211 my $start_t = time();
212
213 my @links;
214
215 if ($parallel) {
216 eval 'use Proc::Queue size => 1;';
217 die $@ if $@;
218 $log->info("Using $parallel processes for speedup");
219 Proc::Queue::size($parallel);
220 }
221
222 sub create_ds_config {
223 my ($db_config, $database, $input, $mfn) = @_;
224 my $c = dclone( $db_config );
225 $c->{_} = $database || $log->logconfess("need database");
226 $c->{_mfn} = $mfn || $log->logconfess("need mfn");
227 $c->{input} = $input || $log->logconfess("need input");
228 return $c;
229 }
230
231 foreach my $database ( sort keys %{ $config->databases } ) {
232 my $db_config = $config->databases->{$database};
233
234 next if ($only_database && $database !~ m/$only_database/i);
235
236 if ($parallel) {
237 my $f=fork;
238 if(defined ($f) and $f==0) {
239 $log->info("Created processes $$ for speedup");
240 } else {
241 next;
242 }
243 }
244
245 my $indexer;
246 if ($use_indexer && $parser->have_rules( 'search', $database )) {
247
248 my $cfg_name = $use_indexer;
249 $cfg_name =~ s/\-.*$//;
250
251 my $indexer_config = $config->get( $cfg_name ) || $log->logdie("can't find '$cfg_name' part in confguration");
252 $indexer_config->{database} = $database;
253 $indexer_config->{clean} = $clean;
254 $indexer_config->{label} = $db_config->{name};
255
256 # force clean if database has links
257 $indexer_config->{clean} = 1 if ($db_config->{links});
258
259 if ($use_indexer eq 'hyperestraier') {
260
261 # open Hyper Estraier database
262 require WebPAC::Output::Estraier;
263 $indexer = new WebPAC::Output::Estraier( %{ $indexer_config } );
264
265 } elsif ($use_indexer eq 'hyperestraier-native') {
266
267 # open Hyper Estraier database
268 require WebPAC::Output::EstraierNative;
269 $indexer = new WebPAC::Output::EstraierNative( %{ $indexer_config } );
270
271 } elsif ($use_indexer eq 'kinosearch') {
272
273 die "no longer supported";
274
275 } else {
276 $log->logdie("unknown use_indexer: $use_indexer");
277 }
278
279 $log->logdie("can't continue without valid indexer") unless ($indexer);
280 }
281
282
283 #
284 # store Hyper Estraier links to other databases
285 #
286 if (ref($db_config->{links}) eq 'ARRAY' && $use_indexer) {
287 foreach my $link (@{ $db_config->{links} }) {
288 if ($use_indexer eq 'hyperestraier') {
289 if ($merge) {
290 print $estcmd_fh 'sudo -u www-data estcmd merge ' . $database . ' ' . $link->{to},$/;
291 } else {
292 $log->info("saving link $database -> $link->{to} [$link->{credit}]");
293 push @links, sub {
294 $log->info("adding link $database -> $link->{to} [$link->{credit}]");
295 $indexer->add_link(
296 from => $database,
297 to => $link->{to},
298 credit => $link->{credit},
299 );
300 };
301 }
302 } else {
303 $log->warn("NOT IMPLEMENTED WITH $use_indexer: adding link $database -> $link->{to} [$link->{credit}]");
304 }
305 }
306 }
307 next if ($only_links);
308
309
310 #
311 # now WebPAC::Store
312 #
313 my $store = new WebPAC::Store({
314 debug => $debug,
315 });
316
317
318 #
319 # prepare output
320 #
321 my @outputs = force_array( $db_config->{output}, sub {
322 $log->error("Database $database doesn't have any outputs defined. Do you want to remove it from configuration?" );
323 } );
324
325 my @output_modules;
326
327 foreach my $output ( @outputs ) {
328
329 #warn '## output = ',dump( $output );
330
331 my $module = $output->{module} || $log->logdie("need module in output section of $database");
332 $module = 'WebPAC::Output::' . $module unless $module =~ m/::/;
333
334 $log->debug("loading output module $module");
335 eval "require $module";
336
337 # add database to arugemnts for output filter
338 $output->{database} = $database;
339 $output->{clean} = $clean;
340
341 $log->debug("calling $module->new(",dump( $output ),")");
342 my $out = new $module->new( $output );
343 if ( $out->init ) {
344 push @output_modules, $out;
345 } else {
346 $log->warn("SKIPPED $module");
347 }
348 }
349
350
351 #
352 # now, iterate through input formats
353 #
354
355
356 my @inputs = force_array( $db_config->{input}, sub {
357 $log->info("database $database doesn't have inputs defined");
358 } );
359
360 if ( -e 'out/debug' ) { # FIXME flag?
361 my $out;
362 foreach my $i ( @inputs ) {
363 warn dump( $i );
364 next unless defined $i->{normalize};
365 warn dump( $i->{normalize} );
366 foreach my $normalize ( @{ $i->{normalize} } ) {
367 my $path = $normalize->{path};
368 $out .= qq/\n##\n## $path\n##\n\n/;
369 $out .= read_file( $path );
370 }
371 }
372 my $all = "out/debug/all-normalize.pl";
373 write_file( $all, $out );
374 warn "### all normalize for this input saved to: $all";
375 };
376
377 foreach my $input (@inputs) {
378
379 my $input_name = $input->{name} || $log->logdie("input without a name isn't valid: ",dump($input));
380
381 if ( $input->{skip} ) {
382 $log->info("skip $input_name");
383 next;
384 }
385
386 next if ($only_input && ($input_name !~ m#$only_input#i && $input->{type} !~ m#$only_input#i));
387
388 my $type = lc($input->{type});
389
390 # FIXME check if input module exists
391 my $input_module = $input->{module};
392
393 if ( ! $input_module ) {
394 if ( grep(/$type/, $config->webpac('inputs')) ) {
395 $input_module = $config->webpac('inputs')->{$type};
396 } else {
397 $log->logdie("I know only how to handle input types ", join(",", $config->webpac('inputs') ), " not '$type'!" );
398 }
399 }
400
401 my @lookups = $parser->have_lookup_create($database, $input);
402
403 $log->info("working on $database/$input_name with $input_module on $input->{path}",
404 @lookups ? " creating lookups: ".join(", ", @lookups) : ""
405 );
406
407 if ($stats) {
408 # disable modification of records if --stats is in use
409 delete($input->{modify_records});
410 delete($input->{modify_file});
411 }
412
413 if ( $mirror ) {
414 my $path = $input->{path} || die "no input path in ",dump( $input );
415
416 my $base = $path;
417 $base =~ s{/[^/]+$}{};
418 mkpath $base unless -e $base;
419
420 my $rc = mirror( "$mirror/$path", $path );
421 if (is_error( $rc )) {
422 die "can't mirror $mirror/$path -> $path [$rc]";
423 } else {
424 $log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" );
425 }
426
427 }
428
429 my $input_db = new WebPAC::Input(
430 module => $input_module,
431 limit => $limit || $input->{limit},
432 offset => $offset,
433 recode => $input->{recode},
434 stats => $stats,
435 modify_records => $input->{modify_records},
436 modify_file => $input->{modify_file},
437 input_config => $input,
438 );
439 $log->logdie("can't create input using $input_module") unless ($input);
440
441 if (defined( $input->{lookup} )) {
442 $log->warn("$database/$input_name has depriciated lookup definition, removing it...");
443 delete( $input->{lookup} );
444 }
445
446 my $lookup_coderef;
447
448 if (@lookups) {
449
450 my $rules = $parser->lookup_create_rules($database, $input) || $log->logdie("no rules found for $database/$input");
451
452 $lookup_coderef = sub {
453 my $rec = shift || die "need rec!";
454 my $mfn = $rec->{'000'}->[0] || die "need mfn in 000";
455
456 WebPAC::Normalize::data_structure(
457 row => $rec,
458 rules => $rules,
459 config => create_ds_config( $db_config, $database, $input, $mfn ),
460 );
461
462 #warn "current lookup: ", dump(WebPAC::Normalize::_get_lookup());
463 };
464
465 WebPAC::Normalize::_set_lookup( undef );
466
467 $log->debug("created lookup_coderef using:\n$rules");
468
469 };
470
471 my $lookup_jar;
472
473 my $maxmfn = $input_db->open(
474 path => $input->{path},
475 input_encoding => $input->{encoding}, # database encoding
476 lookup_coderef => $lookup_coderef,
477 lookup => $lookup_jar,
478 %{ $input },
479 load_row => sub {
480 my $a = shift;
481 return $store->load_row(
482 database => $database,
483 input => $input_name,
484 id => $a->{id},
485 );
486 },
487 save_row => sub {
488 my $a = shift;
489 return $store->save_row(
490 database => $database,
491 input => $input_name,
492 id => $a->{id},
493 row => $a->{row},
494 );
495 },
496
497 );
498
499 my $lookup_data = WebPAC::Normalize::_get_lookup();
500
501 if (defined( $lookup_data->{$database}->{$input_name} )) {
502 $log->debug("created following lookups: ", sub { dump( $lookup_data ) } );
503
504 foreach my $key (keys %{ $lookup_data->{$database}->{$input_name} }) {
505 $store->save_lookup(
506 database => $database,
507 input => $input_name,
508 key => $key,
509 data => $lookup_data->{$database}->{$input_name}->{$key},
510 );
511 }
512 }
513
514 my $report_fh;
515 if ($stats || $validate) {
516 my $out_report = 'out/report'; # FIXME move to config
517 mkpath $out_report unless -e $out_report;
518 my $path = "$out_report/${database}-${input_name}.txt";
519 open($report_fh, '>', $path) || $log->logdie("can't open $path: $!");
520
521 print $report_fh "Report for database '$database' input '$input_name' records ",
522 $offset || 1, "-", $limit || $input->{limit} || $maxmfn, "\n\n";
523 $log->info("Generating report file $path");
524
525 if ( $validate ) {
526 $validate->read_validate_file( $validate->fill_in( $validate_path, database => $database, input => $input_name ) ) if ( $validate_path );
527 $validate->read_validate_delimiters_file( $validate->fill_in( $validate_delimiters_path, database => $database, input => $input_name ) ) if ( $validate_delimiters_path );
528 }
529 }
530
531 my $marc;
532 if ($marc_generate && $parser->have_rules( 'marc', $database, $input_name )) {
533
534 my $out_marc = 'out/marc'; # FIXME move to config
535 mkpath $out_marc unless -e $out_marc;
536
537 $marc = new WebPAC::Output::MARC(
538 path => "$out_marc/${database}-${input_name}.marc",
539 lint => $marc_lint,
540 dump => $marc_dump,
541 );
542 }
543
544 my $rules = $parser->normalize_rules($database,$input_name);
545 $log->logwarn("no normalize rules for $database/$input_name") unless $rules;
546
547 $log->debug("parsed normalize rules:\n$rules");
548
549 # reset position in database
550 $input_db->seek(1);
551
552 # generate name of config key for indexer (strip everything after -)
553 my $indexer_config = $use_indexer;
554 $indexer_config =~ s/^(\w+)-?.*$/$1/g if ($indexer_config);
555
556 my $lookup_hash;
557 my $depends = $parser->depends($database,$input_name);
558
559 if ($depends) {
560 $log->debug("$database/$input_name depends on: ", dump($depends)) if ($depends);
561 $log->logdie("parser->depends didn't return HASH") unless (ref($depends) eq 'HASH');
562
563 foreach my $db (keys %$depends) {
564 foreach my $i (keys %{$depends->{$db}}) {
565 foreach my $k (keys %{$depends->{$db}->{$i}}) {
566 my $t = time();
567 $log->debug("loading lookup $db/$i");
568 $lookup_hash->{$db}->{$i}->{$k} = $store->load_lookup(
569 database => $db,
570 input => $i,
571 key => $k,
572 );
573 $log->debug(sprintf("lookup $db/$i took %.2fs", time() - $t));
574 }
575 }
576 }
577
578 $log->debug("lookup_hash = ", sub { dump( $lookup_hash ) });
579 }
580
581
582 # setup input name for all output filters
583 foreach my $out ( @output_modules ) {
584 if ( $out->can('input') ) {
585 $out->input( $input_name );
586 } else {
587 $log->warn("output filter ",ref($out)," doesn't support input name");
588 }
589 }
590
591
592 foreach my $pos ( 0 ... $input_db->size ) {
593
594 my $row = $input_db->fetch || next;
595
596 $total_rows++;
597
598 my $mfn = $row->{'000'}->[0];
599
600 if (! $mfn || $mfn !~ m{^\d+$}) {
601 $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos");
602 $mfn = $pos;
603 push @{ $row->{'000'} }, $pos;
604 }
605
606
607 if ($validate) {
608 if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) {
609 $log->error( "MFN $mfn validation error:\n",
610 $validate->report_error( $errors )
611 );
612 }
613 next; # validation doesn't create any output
614 }
615
616 if ($rules) {
617
618 my $ds = WebPAC::Normalize::data_structure(
619 row => $row,
620 rules => $rules,
621 lookup => $lookup_hash,
622 config => create_ds_config( $db_config, $database, $input, $mfn ),
623 marc_encoding => 'utf-8',
624 load_row_coderef => sub {
625 my ($database,$input,$mfn) = @_;
626 #warn "### load_row($database,$input,$mfn) from data_structure\n";
627 return $store->load_row(
628 database => $database,
629 input => $input,
630 id => $mfn,
631 );
632 },
633 );
634
635 $log->debug("ds = ", sub { dump($ds) });
636
637 if ( $ds ) {
638
639 $store->save_ds(
640 database => $database,
641 input => $input_name,
642 id => $mfn,
643 ds => $ds,
644 ) if !$stats;
645
646 $indexer->add(
647 id => "${input_name}/${mfn}",
648 ds => $ds,
649 type => $config->get($indexer_config)->{type},
650 ) if $indexer;
651
652 foreach my $out ( @output_modules ) {
653 $out->add( $mfn, $ds ) if $out->can('add');
654 }
655
656 } else {
657 $log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
658 }
659 }
660
661 if ($marc) {
662 my $i = 0;
663
664 while (my $fields = WebPAC::Normalize::MARC::_get_marc_fields( fetch_next => 1 ) ) {
665 $marc->add(
666 id => $mfn . ( $i ? "/$i" : '' ),
667 fields => $fields,
668 leader => WebPAC::Normalize::MARC::_get_marc_leader(),
669 row => $row,
670 );
671 $i++;
672 }
673
674 $log->info("Created $i instances of MFN $mfn\n") if ($i > 1);
675 }
676
677 }
678
679 if ($validate) {
680 my $errors = $validate->report;
681 if ($errors) {
682 $log->info("validation errors:\n$errors\n" );
683 print $report_fh "$errors\n" if ($report_fh);
684 }
685
686 print $report_fh "\nAll possible subfields/delimiter templates:\n", $validate->delimiters_templates( report => 1, current_input => 1 ), "\n\n";
687
688 # must be last thing that touches $validate for this input
689 $validate->reset;
690 }
691
692 if ($stats) {
693 my $s = $input_db->stats;
694 $log->info("statistics of fields usage:\n$s");
695 print $report_fh "Statistics of fields usage:\n$s" if ($report_fh);
696 }
697
698 # close MARC file
699 $marc->finish if ($marc);
700
701 # close report
702 close($report_fh) if ($report_fh);
703 }
704
705 $indexer->finish if $indexer && $indexer->can('finish');
706
707 foreach my $out ( @output_modules ) {
708 $out->finish if $out->can('finish');
709 }
710
711 my $dt = time() - $start_t;
712 $log->info("$total_rows records ", $indexer ? "indexed " : "",
713 sprintf("in %.2f sec [%.2f rec/sec]",
714 $dt, ($total_rows / $dt)
715 )
716 );
717
718
719 # end forked process
720 if ($parallel) {
721 $log->info("parallel process $$ finished");
722 exit(0);
723 }
724
725 }
726
727 if ($parallel) {
728 # wait all children to finish
729 sleep(1) while wait != -1;
730 $log->info("all parallel processes finished");
731 }
732
733 # save new delimiters if needed
734 $validate->save_delimiters_templates if ( $validate_delimiters_path );
735
736 #
737 # handle links or merge after indexing
738 #
739
740 if ($merge) {
741 print $estcmd_fh 'sudo /etc/init.d/hyperestraier start',$/;
742 close($estcmd_fh);
743 chmod 0700, $estcmd_path || $log->warn("can't chmod 0700 $estcmd_path: $!");
744 system $estcmd_path;
745 } else {
746 foreach my $link (@links) {
747 $log->logdie("coderef in link ", Dumper($link), " is ", ref($link), " and not CODE") unless (ref($link) eq 'CODE');
748 $link->();
749 }
750 }

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26