--- trunk/run.pl 2007/05/27 22:24:28 863 +++ trunk/run.pl 2007/11/02 10:54:44 957 @@ -9,7 +9,7 @@ use WebPAC::Common 0.02; use WebPAC::Parser 0.08; use WebPAC::Input 0.16; -use WebPAC::Store 0.14; +use WebPAC::Store 0.15; use WebPAC::Normalize 0.22; use WebPAC::Output::TT; use WebPAC::Validate 0.11; @@ -185,7 +185,7 @@ if ($stats) { $log->debug("disabled indexing for stats collection"); $use_indexer = undef; -} else { +} elsif ( $use_indexer ) { $log->info("using $use_indexer indexing engine..."); } @@ -244,27 +244,24 @@ if ($use_indexer eq 'hyperestraier') { # open Hyper Estraier database - use WebPAC::Output::Estraier '0.10'; + require WebPAC::Output::Estraier; $indexer = new WebPAC::Output::Estraier( %{ $indexer_config } ); } elsif ($use_indexer eq 'hyperestraier-native') { # open Hyper Estraier database - use WebPAC::Output::EstraierNative; + require WebPAC::Output::EstraierNative; $indexer = new WebPAC::Output::EstraierNative( %{ $indexer_config } ); } elsif ($use_indexer eq 'kinosearch') { - # open KinoSearch - use WebPAC::Output::KinoSearch; - $indexer_config->{clean} = 1 unless (-e $indexer_config->{index_path}); - $indexer = new WebPAC::Output::KinoSearch( %{ $indexer_config } ); + die "no longer supported"; } else { $log->logdie("unknown use_indexer: $use_indexer"); } - $log->logide("can't continue without valid indexer") unless ($indexer); + $log->logdie("can't continue without valid indexer") unless ($indexer); } @@ -298,36 +295,49 @@ # # now WebPAC::Store # - my $abs_path = abs_path($0); - $abs_path =~ s#/[^/]*$#/#; + my $store = new WebPAC::Store({ + debug => $debug, + }); - my $db_path = $config->webpac('db_path'); - if ($clean) { - $log->info("creating new database '$database' in $db_path"); - rmtree( $db_path ) || $log->warn("can't remove $db_path: $!"); - } else { - $log->info("working on database '$database' in $db_path"); - } + # + # prepare output + # + my @outputs = force_array( $db_config->{output}, sub { + $log->error("Database $database doesn't have any outputs defined. Do you want to remove it from configuration?" ); + } ); - my $store = new WebPAC::Store( - path => $db_path, - debug => $debug, - ); + my @output_modules; + + foreach my $output ( @outputs ) { + +#warn '## output = ',dump( $output ); + + my $module = $output->{module} || $log->logdie("need module in output section of $database"); + $module = 'WebPAC::Output::' . $module unless $module =~ m/::/; + + $log->debug("loading output module $module"); + eval "require $module"; + + # add database to arugemnts for output filter + $output->{database} = $database; + + $log->debug("calling $module->new(",dump( $output ),")"); + my $out = new $module->new( $output ); + $out->init; + + push @output_modules, $out; + } # # now, iterate through input formats # - my @inputs; - if (ref($db_config->{input}) eq 'ARRAY') { - @inputs = @{ $db_config->{input} }; - } elsif ($db_config->{input}) { - push @inputs, $db_config->{input}; - } else { + + my @inputs = force_array( $db_config->{input}, sub { $log->info("database $database doesn't have inputs defined"); - } + } ); foreach my $input (@inputs) { @@ -362,6 +372,7 @@ stats => $stats, modify_records => $input->{modify_records}, modify_file => $input->{modify_file}, + input_config => $input, ); $log->logdie("can't create input using $input_module") unless ($input); @@ -498,6 +509,16 @@ } + # setup input name for all output filters + foreach my $out ( @output_modules ) { + if ( $out->can('input') ) { + $out->input( $input_name ); + } else { + $log->warn("output filter ",ref($out)," doesn't support input name"); + } + } + + foreach my $pos ( 0 ... $input_db->size ) { my $row = $input_db->fetch || next; @@ -506,7 +527,7 @@ my $mfn = $row->{'000'}->[0]; - if (! $mfn || $mfn !~ m#^\d+$#) { + if (! $mfn || $mfn !~ m{^\d+$}) { $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); $mfn = $pos; push @{ $row->{'000'} }, $pos; @@ -522,7 +543,7 @@ next; # validation doesn't create any output } - my $ds = WebPAC::Normalize::data_structure( + if ( my $ds = WebPAC::Normalize::data_structure( row => $row, rules => $rules, lookup => $lookup_hash, @@ -536,38 +557,47 @@ id => $mfn, ); }, - ); + ) ) { # if - $log->debug("ds = ", sub { dump($ds) }) if ($ds); + $log->debug("ds = ", sub { dump($ds) }); - $store->save_ds( - database => $database, - input => $input_name, - id => $mfn, - ds => $ds, - ) if ($ds && !$stats); - - $indexer->add( - id => "${input_name}/${mfn}", - ds => $ds, - type => $config->get($indexer_config)->{type}, - ) if ($indexer && $ds); - - if ($marc) { - my $i = 0; - - while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) { - $marc->add( - id => $mfn . ( $i ? "/$i" : '' ), - fields => $fields, - leader => WebPAC::Normalize::_get_marc_leader(), - row => $row, - ); - $i++; + $store->save_ds( + database => $database, + input => $input_name, + id => $mfn, + ds => $ds, + ) if !$stats; + + $indexer->add( + id => "${input_name}/${mfn}", + ds => $ds, + type => $config->get($indexer_config)->{type}, + ) if $indexer; + + if ($marc) { + my $i = 0; + + while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) { + $marc->add( + id => $mfn . ( $i ? "/$i" : '' ), + fields => $fields, + leader => WebPAC::Normalize::_get_marc_leader(), + row => $row, + ); + $i++; + } + + $log->info("Created $i instances of MFN $mfn\n") if ($i > 1); + } + + foreach my $out ( @output_modules ) { + $out->add( $mfn, $ds ) if $out->can('add'); } - $log->info("Created $i instances of MFN $mfn\n") if ($i > 1); + } else { + $log->warn("record $pos didn't produce any output after normalization rules!"); } + } if ($validate) { @@ -598,6 +628,10 @@ eval { $indexer->finish } if ($indexer && $indexer->can('finish')); + foreach my $out ( @output_modules ) { + $out->finish if $out->can('finish'); + } + my $dt = time() - $start_t; $log->info("$total_rows records ", $indexer ? "indexed " : "", sprintf("in %.2f sec [%.2f rec/sec]", @@ -621,7 +655,7 @@ } # save new delimiters if needed -$validate->save_delimiters_templates if ( $validate ); +$validate->save_delimiters_templates if ( $validate_delimiters_path ); # # handle links or merge after indexing