22 |
use Data::Dump qw/dump/; |
use Data::Dump qw/dump/; |
23 |
use Storable qw/dclone/; |
use Storable qw/dclone/; |
24 |
use Pod::Usage qw/pod2usage/; |
use Pod::Usage qw/pod2usage/; |
25 |
use LWP::Simple; |
use LWP::Simple qw//; |
26 |
|
|
27 |
use POSIX ":sys_wait_h"; # imports WNOHANG |
use POSIX ":sys_wait_h"; # imports WNOHANG |
28 |
|
|
417 |
$base =~ s{/[^/]+$}{}; |
$base =~ s{/[^/]+$}{}; |
418 |
mkpath $base unless -e $base; |
mkpath $base unless -e $base; |
419 |
|
|
420 |
my $rc = mirror( "$mirror/$path", $path ); |
my $rc = LWP::Simple::mirror( "$mirror/$path", $path ); |
421 |
if (is_error( $rc )) { |
if (LWP::Simple::is_error( $rc )) { |
422 |
die "can't mirror $mirror/$path -> $path [$rc]"; |
die "can't mirror $mirror/$path -> $path [$rc]"; |
423 |
} else { |
} else { |
424 |
$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" ); |
$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" ); |
513 |
|
|
514 |
my $report_fh; |
my $report_fh; |
515 |
if ($stats || $validate) { |
if ($stats || $validate) { |
516 |
my $path = "out/report/${database}-${input_name}.txt"; |
my $out_report = 'out/report'; # FIXME move to config |
517 |
|
mkpath $out_report unless -e $out_report; |
518 |
|
my $path = "$out_report/${database}-${input_name}.txt"; |
519 |
open($report_fh, '>', $path) || $log->logdie("can't open $path: $!"); |
open($report_fh, '>', $path) || $log->logdie("can't open $path: $!"); |
520 |
|
|
521 |
print $report_fh "Report for database '$database' input '$input_name' records ", |
print $report_fh "Report for database '$database' input '$input_name' records ", |
530 |
|
|
531 |
my $marc; |
my $marc; |
532 |
if ($marc_generate && $parser->have_rules( 'marc', $database, $input_name )) { |
if ($marc_generate && $parser->have_rules( 'marc', $database, $input_name )) { |
533 |
|
|
534 |
|
my $out_marc = 'out/marc'; # FIXME move to config |
535 |
|
mkpath $out_marc unless -e $out_marc; |
536 |
|
|
537 |
$marc = new WebPAC::Output::MARC( |
$marc = new WebPAC::Output::MARC( |
538 |
path => "out/marc/${database}-${input_name}.marc", |
path => "$out_marc/${database}-${input_name}.marc", |
539 |
lint => $marc_lint, |
lint => $marc_lint, |
540 |
dump => $marc_dump, |
dump => $marc_dump, |
541 |
); |
); |
542 |
} |
} |
543 |
|
|
544 |
my $rules = $parser->normalize_rules($database,$input_name); |
my $rules = $parser->normalize_rules($database,$input_name); |
545 |
$log->logwarn("no normalize rules for $database/$input_name") unless $rules; |
$log->logwarn("no normalize rules for $database/$input_name", $input_db->input_module->can('normalize') ? " using normalize from input module" : '') unless $rules; |
546 |
|
|
547 |
$log->debug("parsed normalize rules:\n$rules"); |
$log->debug("parsed normalize rules:\n$rules"); |
548 |
|
|
603 |
push @{ $row->{'000'} }, $pos; |
push @{ $row->{'000'} }, $pos; |
604 |
} |
} |
605 |
|
|
606 |
|
foreach my $out ( @output_modules ) { |
607 |
|
$out->add_row( $mfn, $row ) if $out->can('add_row'); |
608 |
|
} |
609 |
|
|
610 |
if ($validate) { |
if ($validate) { |
611 |
if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) { |
if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) { |
616 |
next; # validation doesn't create any output |
next; # validation doesn't create any output |
617 |
} |
} |
618 |
|
|
619 |
|
my $ds; |
620 |
|
|
621 |
if ($rules) { |
if ($rules) { |
622 |
|
|
623 |
my $ds = WebPAC::Normalize::data_structure( |
$ds = WebPAC::Normalize::data_structure( |
624 |
row => $row, |
row => $row, |
625 |
rules => $rules, |
rules => $rules, |
626 |
lookup => $lookup_hash, |
lookup => $lookup_hash, |
637 |
}, |
}, |
638 |
); |
); |
639 |
|
|
640 |
$log->debug("ds = ", sub { dump($ds) }); |
} elsif ( $input_db->input_module->can('normalize') ) { |
641 |
|
$ds = $input_db->input_module->normalize( $mfn ); |
642 |
if ( $ds ) { |
} |
643 |
|
|
644 |
$store->save_ds( |
if ( $ds ) { |
645 |
database => $database, |
$log->debug("ds = ", sub { dump($ds) }); |
|
input => $input_name, |
|
|
id => $mfn, |
|
|
ds => $ds, |
|
|
) if !$stats; |
|
|
|
|
|
$indexer->add( |
|
|
id => "${input_name}/${mfn}", |
|
|
ds => $ds, |
|
|
type => $config->get($indexer_config)->{type}, |
|
|
) if $indexer; |
|
646 |
|
|
647 |
foreach my $out ( @output_modules ) { |
$store->save_ds( |
648 |
$out->add( $mfn, $ds ) if $out->can('add'); |
database => $database, |
649 |
} |
input => $input_name, |
650 |
|
id => $mfn, |
651 |
|
ds => $ds, |
652 |
|
) if !$stats; |
653 |
|
|
654 |
|
$indexer->add( |
655 |
|
id => "${input_name}/${mfn}", |
656 |
|
ds => $ds, |
657 |
|
type => $config->get($indexer_config)->{type}, |
658 |
|
) if $indexer; |
659 |
|
|
660 |
} else { |
foreach my $out ( @output_modules ) { |
661 |
$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc; |
$out->add( $mfn, $ds ) if $out->can('add'); |
662 |
} |
} |
|
} |
|
663 |
|
|
664 |
|
} else { |
665 |
|
$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc; |
666 |
|
} |
667 |
if ($marc) { |
if ($marc) { |
668 |
my $i = 0; |
my $i = 0; |
669 |
|
|