191 |
|
|
192 |
# parse normalize files and create source files for lookup and normalization |
# parse normalize files and create source files for lookup and normalization |
193 |
|
|
194 |
my $parser = new WebPAC::Parser( config => $config ); |
my ($only_database,$only_input) = split(m#/#, $only_filter) if $only_filter; |
195 |
|
|
196 |
|
my $parser = new WebPAC::Parser( |
197 |
|
config => $config, |
198 |
|
only_database => $only_database, |
199 |
|
only_input => $only_input, |
200 |
|
); |
201 |
|
|
202 |
my $total_rows = 0; |
my $total_rows = 0; |
203 |
my $start_t = time(); |
my $start_t = time(); |
221 |
foreach my $database ( sort keys %{ $config->databases } ) { |
foreach my $database ( sort keys %{ $config->databases } ) { |
222 |
my $db_config = $config->databases->{$database}; |
my $db_config = $config->databases->{$database}; |
223 |
|
|
|
my ($only_database,$only_input) = split(m#/#, $only_filter) if ($only_filter); |
|
224 |
next if ($only_database && $database !~ m/$only_database/i); |
next if ($only_database && $database !~ m/$only_database/i); |
225 |
|
|
226 |
if ($parallel) { |
if ($parallel) { |
352 |
|
|
353 |
my $type = lc($input->{type}); |
my $type = lc($input->{type}); |
354 |
|
|
355 |
die "I know only how to handle input types ", join(",", $config->webpac('inputs') ), " not '$type'!\n" unless (grep(/$type/, $config->webpac('inputs'))); |
# FIXME check if input module exists |
356 |
|
my $input_module = $input->{module}; |
357 |
|
|
358 |
my $input_module = $config->webpac('inputs')->{$type}; |
if ( ! $input_module ) { |
359 |
|
if ( grep(/$type/, $config->webpac('inputs')) ) { |
360 |
|
$input_module = $config->webpac('inputs')->{$type}; |
361 |
|
} else { |
362 |
|
$log->logdie("I know only how to handle input types ", join(",", $config->webpac('inputs') ), " not '$type'!" ); |
363 |
|
} |
364 |
|
} |
365 |
|
|
366 |
my @lookups = $parser->have_lookup_create($database, $input); |
my @lookups = $parser->have_lookup_create($database, $input); |
367 |
|
|
555 |
next; # validation doesn't create any output |
next; # validation doesn't create any output |
556 |
} |
} |
557 |
|
|
558 |
if ( my $ds = WebPAC::Normalize::data_structure( |
my $ds = WebPAC::Normalize::data_structure( |
559 |
row => $row, |
row => $row, |
560 |
rules => $rules, |
rules => $rules, |
561 |
lookup => $lookup_hash, |
lookup => $lookup_hash, |
563 |
marc_encoding => 'utf-8', |
marc_encoding => 'utf-8', |
564 |
load_row_coderef => sub { |
load_row_coderef => sub { |
565 |
my ($database,$input,$mfn) = @_; |
my ($database,$input,$mfn) = @_; |
566 |
|
#warn "### load_row($database,$input,$mfn) from data_structure\n"; |
567 |
return $store->load_row( |
return $store->load_row( |
568 |
database => $database, |
database => $database, |
569 |
input => $input, |
input => $input, |
570 |
id => $mfn, |
id => $mfn, |
571 |
); |
); |
572 |
}, |
}, |
573 |
) ) { # if |
); |
574 |
|
|
575 |
$log->debug("ds = ", sub { dump($ds) }); |
$log->debug("ds = ", sub { dump($ds) }); |
576 |
|
|
577 |
|
if ( $ds ) { |
578 |
|
|
579 |
$store->save_ds( |
$store->save_ds( |
580 |
database => $database, |
database => $database, |
589 |
type => $config->get($indexer_config)->{type}, |
type => $config->get($indexer_config)->{type}, |
590 |
) if $indexer; |
) if $indexer; |
591 |
|
|
|
if ($marc) { |
|
|
my $i = 0; |
|
|
|
|
|
while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) { |
|
|
$marc->add( |
|
|
id => $mfn . ( $i ? "/$i" : '' ), |
|
|
fields => $fields, |
|
|
leader => WebPAC::Normalize::_get_marc_leader(), |
|
|
row => $row, |
|
|
); |
|
|
$i++; |
|
|
} |
|
|
|
|
|
$log->info("Created $i instances of MFN $mfn\n") if ($i > 1); |
|
|
} |
|
|
|
|
592 |
foreach my $out ( @output_modules ) { |
foreach my $out ( @output_modules ) { |
593 |
$out->add( $mfn, $ds ) if $out->can('add'); |
$out->add( $mfn, $ds ) if $out->can('add'); |
594 |
} |
} |
595 |
|
|
596 |
} else { |
} else { |
597 |
$log->warn("record $pos didn't produce any output after normalization rules!"); |
$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc; |
598 |
|
} |
599 |
|
|
600 |
|
if ($marc) { |
601 |
|
my $i = 0; |
602 |
|
|
603 |
|
while (my $fields = WebPAC::Normalize::MARC::_get_marc_fields( fetch_next => 1 ) ) { |
604 |
|
$marc->add( |
605 |
|
id => $mfn . ( $i ? "/$i" : '' ), |
606 |
|
fields => $fields, |
607 |
|
leader => WebPAC::Normalize::MARC::_get_marc_leader(), |
608 |
|
row => $row, |
609 |
|
); |
610 |
|
$i++; |
611 |
|
} |
612 |
|
|
613 |
|
$log->info("Created $i instances of MFN $mfn\n") if ($i > 1); |
614 |
} |
} |
615 |
|
|
616 |
} |
} |