22 |
use Data::Dump qw/dump/; |
use Data::Dump qw/dump/; |
23 |
use Storable qw/dclone/; |
use Storable qw/dclone/; |
24 |
use Pod::Usage qw/pod2usage/; |
use Pod::Usage qw/pod2usage/; |
25 |
use LWP::Simple; |
use LWP::Simple qw//; |
26 |
|
|
27 |
use POSIX ":sys_wait_h"; # imports WNOHANG |
use POSIX ":sys_wait_h"; # imports WNOHANG |
28 |
|
|
417 |
$base =~ s{/[^/]+$}{}; |
$base =~ s{/[^/]+$}{}; |
418 |
mkpath $base unless -e $base; |
mkpath $base unless -e $base; |
419 |
|
|
420 |
my $rc = mirror( "$mirror/$path", $path ); |
my $rc = LWP::Simple::mirror( "$mirror/$path", $path ); |
421 |
if (is_error( $rc )) { |
if (LWP::Simple::is_error( $rc )) { |
422 |
die "can't mirror $mirror/$path -> $path [$rc]"; |
die "can't mirror $mirror/$path -> $path [$rc]"; |
423 |
} else { |
} else { |
424 |
$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" ); |
$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" ); |
542 |
} |
} |
543 |
|
|
544 |
my $rules = $parser->normalize_rules($database,$input_name); |
my $rules = $parser->normalize_rules($database,$input_name); |
545 |
$log->logwarn("no normalize rules for $database/$input_name") unless $rules; |
$log->logwarn("no normalize rules for $database/$input_name", $input_db->input_module->can('normalize') ? " using normalize from input module" : '') unless $rules; |
546 |
|
|
547 |
$log->debug("parsed normalize rules:\n$rules"); |
$log->debug("parsed normalize rules:\n$rules"); |
548 |
|
|
603 |
push @{ $row->{'000'} }, $pos; |
push @{ $row->{'000'} }, $pos; |
604 |
} |
} |
605 |
|
|
606 |
|
foreach my $out ( @output_modules ) { |
607 |
|
$out->add_row( $mfn, $row ) if $out->can('add_row'); |
608 |
|
} |
609 |
|
|
610 |
if ($validate) { |
if ($validate) { |
611 |
if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) { |
if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) { |
616 |
next; # validation doesn't create any output |
next; # validation doesn't create any output |
617 |
} |
} |
618 |
|
|
619 |
|
my $ds; |
620 |
|
|
621 |
if ($rules) { |
if ($rules) { |
622 |
|
|
623 |
my $ds = WebPAC::Normalize::data_structure( |
my $ds = WebPAC::Normalize::data_structure( |
637 |
}, |
}, |
638 |
); |
); |
639 |
|
|
640 |
$log->debug("ds = ", sub { dump($ds) }); |
} elsif ( $input_db->input_module->can('normalize') ) { |
641 |
|
$ds = $input_db->input_module->normalize( $mfn ); |
642 |
if ( $ds ) { |
} |
643 |
|
|
644 |
$store->save_ds( |
if ( $ds ) { |
645 |
database => $database, |
$log->debug("ds = ", sub { dump($ds) }); |
|
input => $input_name, |
|
|
id => $mfn, |
|
|
ds => $ds, |
|
|
) if !$stats; |
|
|
|
|
|
$indexer->add( |
|
|
id => "${input_name}/${mfn}", |
|
|
ds => $ds, |
|
|
type => $config->get($indexer_config)->{type}, |
|
|
) if $indexer; |
|
646 |
|
|
647 |
foreach my $out ( @output_modules ) { |
$store->save_ds( |
648 |
$out->add( $mfn, $ds ) if $out->can('add'); |
database => $database, |
649 |
} |
input => $input_name, |
650 |
|
id => $mfn, |
651 |
|
ds => $ds, |
652 |
|
) if !$stats; |
653 |
|
|
654 |
|
$indexer->add( |
655 |
|
id => "${input_name}/${mfn}", |
656 |
|
ds => $ds, |
657 |
|
type => $config->get($indexer_config)->{type}, |
658 |
|
) if $indexer; |
659 |
|
|
660 |
} else { |
foreach my $out ( @output_modules ) { |
661 |
$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc; |
$out->add( $mfn, $ds ) if $out->can('add'); |
662 |
} |
} |
|
} |
|
663 |
|
|
664 |
|
} else { |
665 |
|
$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc; |
666 |
|
} |
667 |
if ($marc) { |
if ($marc) { |
668 |
my $i = 0; |
my $i = 0; |
669 |
|
|