--- trunk/run.pl 2006/06/26 16:39:51 536 +++ trunk/run.pl 2006/06/29 15:29:32 539 @@ -220,56 +220,68 @@ %{ $input }, ); - my $rules; - my $normalize_path = $input->{normalize}->{path}; + my @norm_array = ref($input->{normalize}) eq 'ARRAY' ? + @{ $input->{normalize} } : ( $input->{normalize} ); - $log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i ); + foreach my $normalize (@norm_array) { - my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!"; + my $rules; + my $normalize_path = $normalize->{path} || $log->logdie("can't find normalize path in config"); - foreach my $pos ( 0 ... $input_db->size ) { + $log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i ); - my $row = $input_db->fetch || next; + my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!"; - my $mfn = $row->{'000'}->[0]; + $log->info("Using $normalize_path for normalization..."); - if (! $mfn || $mfn !~ m#^\d+$#) { - $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); - $mfn = $pos; - push @{ $row->{'000'} }, $pos; - } + # reset position in database + $input_db->seek(1); + foreach my $pos ( 0 ... $input_db->size ) { + + my $row = $input_db->fetch || next; + + my $mfn = $row->{'000'}->[0]; + + if (! $mfn || $mfn !~ m#^\d+$#) { + $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); + $mfn = $pos; + push @{ $row->{'000'} }, $pos; + } - if ($validate) { - my @errors = $validate->validate_errors( $row ); - $log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors); - } - - my $ds = WebPAC::Normalize::data_structure( - row => $row, - rules => $rules, - lookup => $lookup ? $lookup->lookup_hash : undef, - ); - - $db->save_ds( - id => $mfn, - ds => $ds, - prefix => $input->{name}, - ) if ($ds && !$stats); - - $indexer->add( - id => $input->{name} . "/" . $mfn, - ds => $ds, - type => $config->{$use_indexer}->{type}, - ) if ($indexer); + if ($validate) { + my @errors = $validate->validate_errors( $row ); + $log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors); + } + + + my $ds = WebPAC::Normalize::data_structure( + row => $row, + rules => $rules, + lookup => $lookup ? $lookup->lookup_hash : undef, + ); + + $db->save_ds( + id => $mfn, + ds => $ds, + prefix => $input->{name}, + ) if ($ds && !$stats); + + $indexer->add( + id => $input->{name} . "/" . $mfn, + ds => $ds, + type => $config->{$use_indexer}->{type}, + ) if ($indexer); + + $total_rows++; + } - $total_rows++; - } + $log->info("statistics of fields usage:\n", $input_db->stats) if ($stats); - $log->info("statistics of fields usage:\n", $input_db->stats) if ($stats); + }; - }; + } eval { $indexer->finish } if ($indexer && $indexer->can('finish'));