/[webpac2]/trunk/run.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/run.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1075 - (show annotations)
Wed Nov 28 11:14:44 2007 UTC (16 years, 4 months ago) by dpavlin
File MIME type: text/plain
File size: 17865 byte(s)
 r1679@llin:  dpavlin | 2007-11-28 12:14:40 +0100
 better output

1 #!/usr/bin/perl -w
2
3 use strict;
4
5 use Cwd qw/abs_path/;
6 use File::Temp qw/tempdir/;
7 use lib './lib';
8
9 use WebPAC::Common 0.02;
10 use WebPAC::Parser 0.08;
11 use WebPAC::Input 0.16;
12 use WebPAC::Store 0.15;
13 use WebPAC::Normalize 0.22;
14 use WebPAC::Output::TT;
15 use WebPAC::Validate 0.11;
16 use WebPAC::Output::MARC;
17 use WebPAC::Config;
18 use Getopt::Long;
19 use File::Path;
20 use Time::HiRes qw/time/;
21 use File::Slurp;
22 use Data::Dump qw/dump/;
23 use Storable qw/dclone/;
24 use Pod::Usage qw/pod2usage/;
25
26 use Proc::Queue size => 1;
27 use POSIX ":sys_wait_h"; # imports WNOHANG
28
29 =head1 NAME
30
31 run.pl - start WebPAC indexing
32
33 B<this command will probably go away. Don't get used to it!>
34
35 =head1 OPTIONS
36
37 =over 4
38
39 =item --offset 42
40
41 start loading (all) databases at offset 42
42
43 =item --limit 100
44
45 limit loading to 100 records
46
47 =item --clean
48
49 remove database and Hyper Estraier index before indexing
50
51 =item --only=database_name/input_filter
52
53 reindex just single database (legacy name is --one)
54
55 C</input_filter> is optional part which can be C<name>
56 or C<type> from input
57
58 =item --config conf/config.yml
59
60 path to YAML configuration file
61
62 =item --stats
63
64 disable indexing, modify_* in configuration and dump statistics about field
65 and subfield usage for each input
66
67 =item --validate path/to/validation_file
68
69 turn on extra validation of imput records, see L<WebPAC::Validation>
70
71 You can use special variables C<$database> and $C<$input> in this parametar
72 like C<--validate 'conf/validate/$database-$input'> to construct filename
73
74 =item --validate-delimiters path/to/validate_delimiters_file
75
76 this option is used with C<--validate> to turn on extra validation of
77 delimiters. If file is non existant, it will be created on first run.
78
79 =item --marc-generate
80
81 Generate MARC file. This will automatically be on if file contains C<marc*> directives.
82 You can use this option as C<--no-marc-generate> to disable MARC generation.
83
84 =item --marc-lint
85
86 By default turned on if normalisation file has C<marc*> directives. You can disable lint
87 messages with C<--no-marc-lint>.
88
89 =item --marc-dump
90
91 Force dump or input and marc record for debugging.
92
93 =item --parallel 4
94
95 Run databases in parallel (aproximatly same as number of processors in
96 machine if you want to use full load)
97
98 =item --only-links
99
100 Create just links
101
102 =item --merge
103
104 Create merged index of databases which have links
105
106 =back
107
108 =cut
109
110 my $offset;
111 my $limit;
112
113 my $clean = 0;
114 my $config_path;
115 my $debug = 0;
116 my $only_filter;
117 my $stats = 0;
118 my $validate_path;
119 my $validate_delimiters_path;
120 my $marc_generate = 1;
121 my $marc_lint = 1;
122 my $marc_dump = 0;
123 my $parallel = 0;
124 my $only_links = 0;
125 my $merge = 0;
126 my $help;
127
128 my $log = _new WebPAC::Common()->_get_logger();
129
130 GetOptions(
131 "limit=i" => \$limit,
132 "offset=i" => \$offset,
133 "clean" => \$clean,
134 "one=s" => \$only_filter,
135 "only=s" => \$only_filter,
136 "config=s" => \$config_path,
137 "debug+" => \$debug,
138 "stats" => \$stats,
139 "validate=s" => \$validate_path,
140 "validate-delimiters=s" => \$validate_delimiters_path,
141 "marc-generate!" => \$marc_generate,
142 "marc-lint!" => \$marc_lint,
143 "marc-dump!" => \$marc_dump,
144 "parallel=i" => \$parallel,
145 "only-links!" => \$only_links,
146 "merge" => \$merge,
147 "help" => \$help,
148 );
149
150 $marc_generate = 0 if ( $validate_delimiters_path );
151
152 pod2usage(-verbose => 2) if ($help);
153
154 my $config = new WebPAC::Config( path => $config_path );
155
156 WebPAC::Normalize::_debug( $debug - 1 ) if $debug > 1;
157
158 #print "config = ",dump($config) if ($debug);
159
160 die "no databases in config file!\n" unless ($config->databases);
161
162 $log->info( "-" x 79 );
163
164 my $log_file = 'log';
165
166 if (-e $log_file ) { # && -s $log_file > 5 * 1024 * 1024) {
167 $log->info("moved old log with ", -s $log_file, " bytes to '${log_file}.old'");
168 rename $log_file, "${log_file}.old" || $log->logwarn("can't rename $log_file to ${log_file}.old: $!");
169 }
170
171 my $estcmd_fh;
172 my $estcmd_path = './estcmd-merge.sh';
173 if ($merge) {
174 open($estcmd_fh, '>', $estcmd_path) || $log->logdie("can't open $estcmd_path: $!");
175 print $estcmd_fh 'cd /data/estraier/_node/ || exit 1',$/;
176 print $estcmd_fh 'sudo /etc/init.d/hyperestraier stop',$/;
177 $log->info("created merge batch file $estcmd_path");
178 }
179
180 my $validate;
181 $validate = new WebPAC::Validate(
182 delimiters => $config->webpac('delimiters'),
183 ) if ($validate_path || $validate_delimiters_path);
184
185 my $use_indexer = $config->use_indexer;
186 $stats ||= $validate;
187 if ($stats) {
188 $log->debug("disabled indexing for stats collection");
189 $use_indexer = undef;
190 } elsif ( $use_indexer ) {
191 $log->info("using $use_indexer indexing engine...");
192 }
193
194 # parse normalize files and create source files for lookup and normalization
195
196 my ($only_database,$only_input) = split(m#/#, $only_filter) if $only_filter;
197
198 my $parser = new WebPAC::Parser(
199 config => $config,
200 only_database => $only_database,
201 only_input => $only_input,
202 );
203
204 my $total_rows = 0;
205 my $start_t = time();
206
207 my @links;
208
209 if ($parallel) {
210 $log->info("Using $parallel processes for speedup");
211 Proc::Queue::size($parallel);
212 }
213
214 sub create_ds_config {
215 my ($db_config, $database, $input, $mfn) = @_;
216 my $c = dclone( $db_config );
217 $c->{_} = $database || $log->logconfess("need database");
218 $c->{_mfn} = $mfn || $log->logconfess("need mfn");
219 $c->{input} = $input || $log->logconfess("need input");
220 return $c;
221 }
222
223 foreach my $database ( sort keys %{ $config->databases } ) {
224 my $db_config = $config->databases->{$database};
225
226 next if ($only_database && $database !~ m/$only_database/i);
227
228 if ($parallel) {
229 my $f=fork;
230 if(defined ($f) and $f==0) {
231 $log->info("Created processes $$ for speedup");
232 } else {
233 next;
234 }
235 }
236
237 my $indexer;
238 if ($use_indexer && $parser->have_rules( 'search', $database )) {
239
240 my $cfg_name = $use_indexer;
241 $cfg_name =~ s/\-.*$//;
242
243 my $indexer_config = $config->get( $cfg_name ) || $log->logdie("can't find '$cfg_name' part in confguration");
244 $indexer_config->{database} = $database;
245 $indexer_config->{clean} = $clean;
246 $indexer_config->{label} = $db_config->{name};
247
248 # force clean if database has links
249 $indexer_config->{clean} = 1 if ($db_config->{links});
250
251 if ($use_indexer eq 'hyperestraier') {
252
253 # open Hyper Estraier database
254 require WebPAC::Output::Estraier;
255 $indexer = new WebPAC::Output::Estraier( %{ $indexer_config } );
256
257 } elsif ($use_indexer eq 'hyperestraier-native') {
258
259 # open Hyper Estraier database
260 require WebPAC::Output::EstraierNative;
261 $indexer = new WebPAC::Output::EstraierNative( %{ $indexer_config } );
262
263 } elsif ($use_indexer eq 'kinosearch') {
264
265 die "no longer supported";
266
267 } else {
268 $log->logdie("unknown use_indexer: $use_indexer");
269 }
270
271 $log->logdie("can't continue without valid indexer") unless ($indexer);
272 }
273
274
275 #
276 # store Hyper Estraier links to other databases
277 #
278 if (ref($db_config->{links}) eq 'ARRAY' && $use_indexer) {
279 foreach my $link (@{ $db_config->{links} }) {
280 if ($use_indexer eq 'hyperestraier') {
281 if ($merge) {
282 print $estcmd_fh 'sudo -u www-data estcmd merge ' . $database . ' ' . $link->{to},$/;
283 } else {
284 $log->info("saving link $database -> $link->{to} [$link->{credit}]");
285 push @links, sub {
286 $log->info("adding link $database -> $link->{to} [$link->{credit}]");
287 $indexer->add_link(
288 from => $database,
289 to => $link->{to},
290 credit => $link->{credit},
291 );
292 };
293 }
294 } else {
295 $log->warn("NOT IMPLEMENTED WITH $use_indexer: adding link $database -> $link->{to} [$link->{credit}]");
296 }
297 }
298 }
299 next if ($only_links);
300
301
302 #
303 # now WebPAC::Store
304 #
305 my $store = new WebPAC::Store({
306 debug => $debug,
307 });
308
309
310 #
311 # prepare output
312 #
313 my @outputs = force_array( $db_config->{output}, sub {
314 $log->error("Database $database doesn't have any outputs defined. Do you want to remove it from configuration?" );
315 } );
316
317 my @output_modules;
318
319 foreach my $output ( @outputs ) {
320
321 #warn '## output = ',dump( $output );
322
323 my $module = $output->{module} || $log->logdie("need module in output section of $database");
324 $module = 'WebPAC::Output::' . $module unless $module =~ m/::/;
325
326 $log->debug("loading output module $module");
327 eval "require $module";
328
329 # add database to arugemnts for output filter
330 $output->{database} = $database;
331
332 $log->debug("calling $module->new(",dump( $output ),")");
333 my $out = new $module->new( $output );
334 if ( $out->init ) {
335 push @output_modules, $out;
336 } else {
337 $log->warn("SKIPPED $module");
338 }
339 }
340
341
342 #
343 # now, iterate through input formats
344 #
345
346
347 my @inputs = force_array( $db_config->{input}, sub {
348 $log->info("database $database doesn't have inputs defined");
349 } );
350
351 foreach my $input (@inputs) {
352
353 my $input_name = $input->{name} || $log->logdie("input without a name isn't valid: ",dump($input));
354
355 next if ($only_input && ($input_name !~ m#$only_input#i && $input->{type} !~ m#$only_input#i));
356
357 my $type = lc($input->{type});
358
359 # FIXME check if input module exists
360 my $input_module = $input->{module};
361
362 if ( ! $input_module ) {
363 if ( grep(/$type/, $config->webpac('inputs')) ) {
364 $input_module = $config->webpac('inputs')->{$type};
365 } else {
366 $log->logdie("I know only how to handle input types ", join(",", $config->webpac('inputs') ), " not '$type'!" );
367 }
368 }
369
370 my @lookups = $parser->have_lookup_create($database, $input);
371
372 $log->info("working on $database/$input_name with $input_module on $input->{path}",
373 @lookups ? " creating lookups: ".join(", ", @lookups) : ""
374 );
375
376 if ($stats) {
377 # disable modification of records if --stats is in use
378 delete($input->{modify_records});
379 delete($input->{modify_file});
380 }
381
382 my $input_db = new WebPAC::Input(
383 module => $input_module,
384 encoding => $config->webpac('webpac_encoding'),
385 limit => $limit || $input->{limit},
386 offset => $offset,
387 recode => $input->{recode},
388 stats => $stats,
389 modify_records => $input->{modify_records},
390 modify_file => $input->{modify_file},
391 input_config => $input,
392 );
393 $log->logdie("can't create input using $input_module") unless ($input);
394
395 if (defined( $input->{lookup} )) {
396 $log->warn("$database/$input_name has depriciated lookup definition, removing it...");
397 delete( $input->{lookup} );
398 }
399
400 my $lookup_coderef;
401
402 if (@lookups) {
403
404 my $rules = $parser->lookup_create_rules($database, $input) || $log->logdie("no rules found for $database/$input");
405
406 $lookup_coderef = sub {
407 my $rec = shift || die "need rec!";
408 my $mfn = $rec->{'000'}->[0] || die "need mfn in 000";
409
410 WebPAC::Normalize::data_structure(
411 row => $rec,
412 rules => $rules,
413 config => create_ds_config( $db_config, $database, $input, $mfn ),
414 );
415
416 #warn "current lookup: ", dump(WebPAC::Normalize::_get_lookup());
417 };
418
419 WebPAC::Normalize::_set_lookup( undef );
420
421 $log->debug("created lookup_coderef using:\n$rules");
422
423 };
424
425 my $lookup_jar;
426
427 my $maxmfn = $input_db->open(
428 path => $input->{path},
429 code_page => $input->{encoding}, # database encoding
430 lookup_coderef => $lookup_coderef,
431 lookup => $lookup_jar,
432 %{ $input },
433 load_row => sub {
434 my $a = shift;
435 return $store->load_row(
436 database => $database,
437 input => $input_name,
438 id => $a->{id},
439 );
440 },
441 save_row => sub {
442 my $a = shift;
443 return $store->save_row(
444 database => $database,
445 input => $input_name,
446 id => $a->{id},
447 row => $a->{row},
448 );
449 },
450
451 );
452
453 my $lookup_data = WebPAC::Normalize::_get_lookup();
454
455 if (defined( $lookup_data->{$database}->{$input_name} )) {
456 $log->debug("created following lookups: ", sub { dump( $lookup_data ) } );
457
458 foreach my $key (keys %{ $lookup_data->{$database}->{$input_name} }) {
459 $store->save_lookup(
460 database => $database,
461 input => $input_name,
462 key => $key,
463 data => $lookup_data->{$database}->{$input_name}->{$key},
464 );
465 }
466 }
467
468 my $report_fh;
469 if ($stats || $validate) {
470 my $path = "out/report/${database}-${input_name}.txt";
471 open($report_fh, '>', $path) || $log->logdie("can't open $path: $!");
472
473 print $report_fh "Report for database '$database' input '$input_name' records ",
474 $offset || 1, "-", $limit || $input->{limit} || $maxmfn, "\n\n";
475 $log->info("Generating report file $path");
476
477 if ( $validate ) {
478 $validate->read_validate_file( $validate->fill_in( $validate_path, database => $database, input => $input_name ) ) if ( $validate_path );
479 $validate->read_validate_delimiters_file( $validate->fill_in( $validate_delimiters_path, database => $database, input => $input_name ) ) if ( $validate_delimiters_path );
480 }
481 }
482
483 my $marc;
484 if ($marc_generate && $parser->have_rules( 'marc', $database, $input_name )) {
485 $marc = new WebPAC::Output::MARC(
486 path => "out/marc/${database}-${input_name}.marc",
487 lint => $marc_lint,
488 dump => $marc_dump,
489 );
490 }
491
492 my $rules = $parser->normalize_rules($database,$input_name) || $log->logdie("no normalize rules found for $database/$input_name");
493 $log->debug("parsed normalize rules:\n$rules");
494
495 # reset position in database
496 $input_db->seek(1);
497
498 # generate name of config key for indexer (strip everything after -)
499 my $indexer_config = $use_indexer;
500 $indexer_config =~ s/^(\w+)-?.*$/$1/g if ($indexer_config);
501
502 my $lookup_hash;
503 my $depends = $parser->depends($database,$input_name);
504
505 if ($depends) {
506 $log->debug("$database/$input_name depends on: ", dump($depends)) if ($depends);
507 $log->logdie("parser->depends didn't return HASH") unless (ref($depends) eq 'HASH');
508
509 foreach my $db (keys %$depends) {
510 foreach my $i (keys %{$depends->{$db}}) {
511 foreach my $k (keys %{$depends->{$db}->{$i}}) {
512 my $t = time();
513 $log->debug("loading lookup $db/$i");
514 $lookup_hash->{$db}->{$i}->{$k} = $store->load_lookup(
515 database => $db,
516 input => $i,
517 key => $k,
518 );
519 $log->debug(sprintf("lookup $db/$i took %.2fs", time() - $t));
520 }
521 }
522 }
523
524 $log->debug("lookup_hash = ", sub { dump( $lookup_hash ) });
525 }
526
527
528 # setup input name for all output filters
529 foreach my $out ( @output_modules ) {
530 if ( $out->can('input') ) {
531 $out->input( $input_name );
532 } else {
533 $log->warn("output filter ",ref($out)," doesn't support input name");
534 }
535 }
536
537
538 foreach my $pos ( 0 ... $input_db->size ) {
539
540 my $row = $input_db->fetch || next;
541
542 $total_rows++;
543
544 my $mfn = $row->{'000'}->[0];
545
546 if (! $mfn || $mfn !~ m{^\d+$}) {
547 $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos");
548 $mfn = $pos;
549 push @{ $row->{'000'} }, $pos;
550 }
551
552
553 if ($validate) {
554 if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) {
555 $log->error( "MFN $mfn validation error:\n",
556 $validate->report_error( $errors )
557 );
558 }
559 next; # validation doesn't create any output
560 }
561
562 my $ds = WebPAC::Normalize::data_structure(
563 row => $row,
564 rules => $rules,
565 lookup => $lookup_hash,
566 config => create_ds_config( $db_config, $database, $input, $mfn ),
567 marc_encoding => 'utf-8',
568 load_row_coderef => sub {
569 my ($database,$input,$mfn) = @_;
570 #warn "### load_row($database,$input,$mfn) from data_structure\n";
571 return $store->load_row(
572 database => $database,
573 input => $input,
574 id => $mfn,
575 );
576 },
577 );
578
579 $log->debug("ds = ", sub { dump($ds) });
580
581 if ( $ds ) {
582
583 $store->save_ds(
584 database => $database,
585 input => $input_name,
586 id => $mfn,
587 ds => $ds,
588 ) if !$stats;
589
590 $indexer->add(
591 id => "${input_name}/${mfn}",
592 ds => $ds,
593 type => $config->get($indexer_config)->{type},
594 ) if $indexer;
595
596 foreach my $out ( @output_modules ) {
597 $out->add( $mfn, $ds ) if $out->can('add');
598 }
599
600 } else {
601 $log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
602 }
603
604 if ($marc) {
605 my $i = 0;
606
607 while (my $fields = WebPAC::Normalize::MARC::_get_marc_fields( fetch_next => 1 ) ) {
608 $marc->add(
609 id => $mfn . ( $i ? "/$i" : '' ),
610 fields => $fields,
611 leader => WebPAC::Normalize::MARC::_get_marc_leader(),
612 row => $row,
613 );
614 $i++;
615 }
616
617 $log->info("Created $i instances of MFN $mfn\n") if ($i > 1);
618 }
619
620 }
621
622 if ($validate) {
623 my $errors = $validate->report;
624 if ($errors) {
625 $log->info("validation errors:\n$errors\n" );
626 print $report_fh "$errors\n" if ($report_fh);
627 }
628
629 print $report_fh "\nAll possible subfields/delimiter templates:\n", $validate->delimiters_templates( report => 1, current_input => 1 ), "\n\n";
630
631 # must be last thing that touches $validate for this input
632 $validate->reset;
633 }
634
635 if ($stats) {
636 my $s = $input_db->stats;
637 $log->info("statistics of fields usage:\n$s");
638 print $report_fh "Statistics of fields usage:\n$s" if ($report_fh);
639 }
640
641 # close MARC file
642 $marc->finish if ($marc);
643
644 # close report
645 close($report_fh) if ($report_fh);
646 }
647
648 eval { $indexer->finish } if ($indexer && $indexer->can('finish'));
649
650 foreach my $out ( @output_modules ) {
651 $out->finish if $out->can('finish');
652 }
653
654 my $dt = time() - $start_t;
655 $log->info("$total_rows records ", $indexer ? "indexed " : "",
656 sprintf("in %.2f sec [%.2f rec/sec]",
657 $dt, ($total_rows / $dt)
658 )
659 );
660
661
662 # end forked process
663 if ($parallel) {
664 $log->info("parallel process $$ finished");
665 exit(0);
666 }
667
668 }
669
670 if ($parallel) {
671 # wait all children to finish
672 sleep(1) while wait != -1;
673 $log->info("all parallel processes finished");
674 }
675
676 # save new delimiters if needed
677 $validate->save_delimiters_templates if ( $validate_delimiters_path );
678
679 #
680 # handle links or merge after indexing
681 #
682
683 if ($merge) {
684 print $estcmd_fh 'sudo /etc/init.d/hyperestraier start',$/;
685 close($estcmd_fh);
686 chmod 0700, $estcmd_path || $log->warn("can't chmod 0700 $estcmd_path: $!");
687 system $estcmd_path;
688 } else {
689 foreach my $link (@links) {
690 $log->logdie("coderef in link ", Dumper($link), " is ", ref($link), " and not CODE") unless (ref($link) eq 'CODE');
691 $link->();
692 }
693 }

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26