--- trunk/run.pl 2006/09/26 12:42:49 713 +++ trunk/run.pl 2006/09/29 20:18:39 731 @@ -10,7 +10,7 @@ use WebPAC::Parser 0.04; use WebPAC::Input 0.13; use WebPAC::Store 0.11; -use WebPAC::Normalize 0.11; +use WebPAC::Normalize 0.21; use WebPAC::Output::TT; use WebPAC::Validate 0.06; use WebPAC::Output::MARC; @@ -354,7 +354,6 @@ delete( $input->{lookup} ); } - my $lookup; my $lookup_coderef; if (@lookups) { @@ -367,12 +366,11 @@ WebPAC::Normalize::data_structure( row => $rec, - rules => $rules, - lookup => $lookup, + rules => "_debug(2); $rules _debug(0)", config => create_ds_config( $db_config, $database, $input, $mfn ), ); - warn "current lookup = ", dump($lookup) if ($lookup); + warn "current lookup: ", dump(WebPAC::Normalize::_get_lookup()); }; WebPAC::Normalize::_set_lookup( undef ); @@ -381,10 +379,13 @@ }; + my $lookup_jar; + my $maxmfn = $input_db->open( path => $input->{path}, code_page => $input->{encoding}, # database encoding lookup_coderef => $lookup_coderef, + lookup => $lookup_jar, %{ $input }, ); @@ -413,121 +414,138 @@ $log->info("Generating report file $path"); } - my @norm_array = ref($input->{normalize}) eq 'ARRAY' ? - @{ $input->{normalize} } : ( $input->{normalize} ); - + my $marc; if ($marc_normalize) { - @norm_array = ( { - path => $marc_normalize, - output => $marc_output || "out/marc/${database}-${input_name}.marc", - } ); - } - - foreach my $normalize (@norm_array) { - - my $normalize_path = $normalize->{path} || $log->logdie("can't find normalize path in config"); - - $log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i ); - - my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!"; - - $log->info("Using $normalize_path for normalization..."); - - my $marc = new WebPAC::Output::MARC( - path => $normalize->{output}, + $marc = new WebPAC::Output::MARC( + path => $marc_output || "out/marc/${database}-${input_name}.marc", lint => $marc_lint, dump => $marc_dump, - ) if ($normalize->{output}); - - # reset position in database - $input_db->seek(1); - - # generate name of config key for indexer (strip everything after -) - my $indexer_config = $use_indexer; - $indexer_config =~ s/^(\w+)-?.*$/$1/g if ($indexer_config); - - foreach my $pos ( 0 ... $input_db->size ) { - - my $row = $input_db->fetch || next; + ); + } - my $mfn = $row->{'000'}->[0]; + my $rules = $parser->normalize_rules($database,$input_name) || $log->logdie("no normalize rules found for $database/$input_name"); + $log->debug("parsed normalize rules:\n$rules"); - if (! $mfn || $mfn !~ m#^\d+$#) { - $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); - $mfn = $pos; - push @{ $row->{'000'} }, $pos; - } + # reset position in database + $input_db->seek(1); - - if ($validate) { - if ( my $errors = $validate->validate_errors( $row, $input_db->dump ) ) { - $log->error( "MFN $mfn validation error:\n", - $validate->report_error( $errors ) + # generate name of config key for indexer (strip everything after -) + my $indexer_config = $use_indexer; + $indexer_config =~ s/^(\w+)-?.*$/$1/g if ($indexer_config); + + my $lookup_hash; + my $depends = $parser->depends($database,$input_name); + + if ($depends) { + $log->debug("$database/$input_name depends on: ", dump($depends)) if ($depends); + $log->logdie("parser->depends didn't return HASH") unless (ref($depends) eq 'HASH'); + + foreach my $db (keys %$depends) { + foreach my $i (keys %{$depends->{$db}}) { + foreach my $k (keys %{$depends->{$db}->{$i}}) { + $log->debug("loading lookup $db/$i"); + $lookup_hash->{$db}->{$i}->{$k} = $store->load_lookup( + database => $db, + input => $i, + key => $k, ); } } + } - my $ds = WebPAC::Normalize::data_structure( - row => $row, - rules => $rules, - lookup => $lookup ? $lookup->lookup_hash : undef, - config => create_ds_config( $db_config, $database, $input, $mfn ), - marc_encoding => 'utf-8', - ); + $log->debug("lookup_hash = ", dump( $lookup_hash )); + } - $store->save_ds( - database => $database, - input => $input_name, - id => $mfn, - ds => $ds, - ) if ($ds && !$stats); - - $indexer->add( - id => "${input_name}/${mfn}", - ds => $ds, - type => $config->get($indexer_config)->{type}, - ) if ($indexer && $ds); - - if ($marc) { - my $i = 0; - - while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) { - $marc->add( - id => $mfn . ( $i ? "/$i" : '' ), - fields => $fields, - leader => WebPAC::Normalize::marc_leader(), - row => $row, - ); - $i++; - } - $log->info("Created $i instances of MFN $mfn\n") if ($i > 1); - } + foreach my $pos ( 0 ... $input_db->size ) { + + my $row = $input_db->fetch || next; - $total_rows++; + my $mfn = $row->{'000'}->[0]; + + if (! $mfn || $mfn !~ m#^\d+$#) { + $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); + $mfn = $pos; + push @{ $row->{'000'} }, $pos; } + if ($validate) { - my $errors = $validate->report; - if ($errors) { - $log->info("validation errors:\n$errors\n" ); - print $report_fh "$errors\n" if ($report_fh); + if ( my $errors = $validate->validate_errors( $row, $input_db->dump ) ) { + $log->error( "MFN $mfn validation error:\n", + $validate->report_error( $errors ) + ); } } - if ($stats) { - my $s = $input_db->stats; - $log->info("statistics of fields usage:\n$s"); - print $report_fh "Statistics of fields usage:\n$s" if ($report_fh); + my $ds = WebPAC::Normalize::data_structure( + row => $row, + rules => $rules, + lookup => $lookup_hash, + config => create_ds_config( $db_config, $database, $input, $mfn ), + marc_encoding => 'utf-8', + load_ds_coderef => sub { + my ($database,$input,$mfn) = @_; + return $store->load_ds( + database => $database, + input => $input, + id => $mfn, + ); + }, + ); + + $store->save_ds( + database => $database, + input => $input_name, + id => $mfn, + ds => $ds, + ) if ($ds && !$stats); + + $indexer->add( + id => "${input_name}/${mfn}", + ds => $ds, + type => $config->get($indexer_config)->{type}, + ) if ($indexer && $ds); + + if ($marc) { + my $i = 0; + + while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) { + $marc->add( + id => $mfn . ( $i ? "/$i" : '' ), + fields => $fields, + leader => WebPAC::Normalize::marc_leader(), + row => $row, + ); + $i++; + } + + $log->info("Created $i instances of MFN $mfn\n") if ($i > 1); } - # close MARC file - $marc->finish if ($marc); + $total_rows++; + } - # close report - close($report_fh) if ($report_fh) + if ($validate) { + my $errors = $validate->report; + if ($errors) { + $log->info("validation errors:\n$errors\n" ); + print $report_fh "$errors\n" if ($report_fh); + } } + if ($stats) { + my $s = $input_db->stats; + $log->info("statistics of fields usage:\n$s"); + print $report_fh "Statistics of fields usage:\n$s" if ($report_fh); + } + + # close MARC file + $marc->finish if ($marc); + + # close report + close($report_fh) if ($report_fh) + } eval { $indexer->finish } if ($indexer && $indexer->can('finish'));