299 |
debug => $debug, |
debug => $debug, |
300 |
}); |
}); |
301 |
|
|
|
sub iterate_over { |
|
|
my ( $what, $error ) = @_; |
|
|
my @result; |
|
|
if ( ref( $what ) eq 'ARRAY' ) { |
|
|
@result = @{ $what }; |
|
|
} elsif ($db_config->{input}) { |
|
|
@result = ( $what ); |
|
|
} else { |
|
|
$error->() if ref($error) eq 'CODE'; |
|
|
} |
|
|
return @result; |
|
|
} |
|
302 |
|
|
303 |
# |
# |
304 |
# prepare output |
# prepare output |
305 |
# |
# |
306 |
my @outputs = iterate_over( $db_config->{output}, sub { |
my @outputs = force_array( $db_config->{output}, sub { |
307 |
$log->error("Database $database doesn't have any outputs defined. Do you want to remove it from configuration?" ); |
$log->error("Database $database doesn't have any outputs defined. Do you want to remove it from configuration?" ); |
308 |
} ); |
} ); |
309 |
|
|
318 |
|
|
319 |
$log->debug("loading output module $module"); |
$log->debug("loading output module $module"); |
320 |
eval "require $module"; |
eval "require $module"; |
321 |
|
|
322 |
|
# add database to arugemnts for output filter |
323 |
$output->{database} = $database; |
$output->{database} = $database; |
324 |
|
|
325 |
$log->debug("calling $module->new(",dump( $output ),")"); |
$log->debug("calling $module->new(",dump( $output ),")"); |
335 |
# |
# |
336 |
|
|
337 |
|
|
338 |
my @inputs = iterate_over( $db_config->{input}, sub { |
my @inputs = force_array( $db_config->{input}, sub { |
339 |
$log->info("database $database doesn't have inputs defined"); |
$log->info("database $database doesn't have inputs defined"); |
340 |
} ); |
} ); |
341 |
|
|
509 |
} |
} |
510 |
|
|
511 |
|
|
512 |
|
# setup input name for all output filters |
513 |
|
foreach my $out ( @output_modules ) { |
514 |
|
if ( $out->can('input') ) { |
515 |
|
$out->input( $input_name ); |
516 |
|
} else { |
517 |
|
$log->warn("output filter ",ref($out)," doesn't support input name"); |
518 |
|
} |
519 |
|
} |
520 |
|
|
521 |
|
|
522 |
foreach my $pos ( 0 ... $input_db->size ) { |
foreach my $pos ( 0 ... $input_db->size ) { |
523 |
|
|
524 |
my $row = $input_db->fetch || next; |
my $row = $input_db->fetch || next; |
543 |
next; # validation doesn't create any output |
next; # validation doesn't create any output |
544 |
} |
} |
545 |
|
|
546 |
my $ds = WebPAC::Normalize::data_structure( |
if ( my $ds = WebPAC::Normalize::data_structure( |
547 |
row => $row, |
row => $row, |
548 |
rules => $rules, |
rules => $rules, |
549 |
lookup => $lookup_hash, |
lookup => $lookup_hash, |
557 |
id => $mfn, |
id => $mfn, |
558 |
); |
); |
559 |
}, |
}, |
560 |
); |
) ) { # if |
561 |
|
|
562 |
$log->debug("ds = ", sub { dump($ds) }) if ($ds); |
$log->debug("ds = ", sub { dump($ds) }); |
563 |
|
|
564 |
$store->save_ds( |
$store->save_ds( |
565 |
database => $database, |
database => $database, |
566 |
input => $input_name, |
input => $input_name, |
567 |
id => $mfn, |
id => $mfn, |
568 |
ds => $ds, |
ds => $ds, |
569 |
) if ($ds && !$stats); |
) if !$stats; |
570 |
|
|
571 |
$indexer->add( |
$indexer->add( |
572 |
id => "${input_name}/${mfn}", |
id => "${input_name}/${mfn}", |
573 |
ds => $ds, |
ds => $ds, |
574 |
type => $config->get($indexer_config)->{type}, |
type => $config->get($indexer_config)->{type}, |
575 |
) if ($indexer && $ds); |
) if $indexer; |
576 |
|
|
577 |
if ($marc) { |
if ($marc) { |
578 |
my $i = 0; |
my $i = 0; |
579 |
|
|
580 |
while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) { |
while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) { |
581 |
$marc->add( |
$marc->add( |
582 |
id => $mfn . ( $i ? "/$i" : '' ), |
id => $mfn . ( $i ? "/$i" : '' ), |
583 |
fields => $fields, |
fields => $fields, |
584 |
leader => WebPAC::Normalize::_get_marc_leader(), |
leader => WebPAC::Normalize::_get_marc_leader(), |
585 |
row => $row, |
row => $row, |
586 |
); |
); |
587 |
$i++; |
$i++; |
588 |
|
} |
589 |
|
|
590 |
|
$log->info("Created $i instances of MFN $mfn\n") if ($i > 1); |
591 |
} |
} |
592 |
|
|
593 |
$log->info("Created $i instances of MFN $mfn\n") if ($i > 1); |
foreach my $out ( @output_modules ) { |
594 |
} |
$out->add( $mfn, $ds ) if $out->can('add'); |
595 |
|
} |
596 |
|
|
597 |
foreach my $out ( @output_modules ) { |
} else { |
598 |
$out->add( $mfn, $ds ) if $out->can('add'); |
$log->warn("record $pos didn't produce any output after normalization rules!"); |
599 |
} |
} |
600 |
|
|
601 |
} |
} |