--- trunk/run.pl	2009/05/18 18:47:22	1183
+++ trunk/run.pl	2009/06/10 08:24:01	1225
@@ -22,7 +22,7 @@
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
-use LWP::Simple;
+use LWP::Simple qw//;
 
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
@@ -417,8 +417,8 @@
 			$base =~ s{/[^/]+$}{};
 			mkpath $base unless -e $base;
 
-			my $rc = mirror( "$mirror/$path", $path );
-			if (is_error( $rc )) {
+			my $rc = LWP::Simple::mirror( "$mirror/$path", $path );
+			if (LWP::Simple::is_error( $rc )) {
 				die "can't mirror $mirror/$path -> $path [$rc]";
 			} else {
 				$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" );
@@ -513,7 +513,9 @@
 
 		my $report_fh;
 		if ($stats || $validate) {
-			my $path = "out/report/${database}-${input_name}.txt";
+			my $out_report = 'out/report'; # FIXME move to config
+			mkpath $out_report unless -e $out_report;
+			my $path = "$out_report/${database}-${input_name}.txt";
 			open($report_fh, '>', $path) || $log->logdie("can't open $path: $!");
 
 			print $report_fh "Report for database '$database' input '$input_name' records ",
@@ -528,15 +530,19 @@
 
 		my $marc;
 		if ($marc_generate && $parser->have_rules( 'marc', $database, $input_name )) {
+			
+			my $out_marc = 'out/marc'; # FIXME move to config
+			mkpath $out_marc unless -e $out_marc;
+
 			$marc = new WebPAC::Output::MARC(
-				path => "out/marc/${database}-${input_name}.marc",
+				path => "$out_marc/${database}-${input_name}.marc",
 				lint => $marc_lint,
 				dump => $marc_dump,
 			);
 		}
 
 		my $rules = $parser->normalize_rules($database,$input_name);
-		$log->logwarn("no normalize rules for $database/$input_name") unless $rules;
+		$log->logwarn("no normalize rules for $database/$input_name", $input_db->input_module->can('normalize') ? " using normalize from input module" : '') unless $rules;
 
 		$log->debug("parsed normalize rules:\n$rules");
 
@@ -597,6 +603,9 @@
 				push @{ $row->{'000'} }, $pos;
 			}
 
+			foreach my $out ( @output_modules ) {
+				$out->add_row( $mfn, $row ) if $out->can('add_row');
+			}
 
 			if ($validate) {
 				if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) {
@@ -607,9 +616,11 @@
 				next;	# validation doesn't create any output
 			}
 
+			my $ds;
+
 			if ($rules) {
 
-				my $ds = WebPAC::Normalize::data_structure(
+				$ds = WebPAC::Normalize::data_structure(
 					row => $row,
 					rules => $rules,
 					lookup => $lookup_hash,
@@ -626,32 +637,33 @@
 					},
 				);
 
-				$log->debug("ds = ", sub { dump($ds) });
-
-				if ( $ds ) {
+			} elsif ( $input_db->input_module->can('normalize') ) {
+				$ds = $input_db->input_module->normalize( $mfn );
+			}
 
-					$store->save_ds(
-						database => $database,
-						input => $input_name,
-						id => $mfn,
-						ds => $ds,
-					) if !$stats;
-
-					$indexer->add(
-						id => "${input_name}/${mfn}",
-						ds => $ds,
-						type => $config->get($indexer_config)->{type},
-					) if $indexer;
+			if ( $ds ) {
+				$log->debug("ds = ", sub { dump($ds) });
 
-					foreach my $out ( @output_modules ) {
-						$out->add( $mfn, $ds ) if $out->can('add');
-					}
+				$store->save_ds(
+					database => $database,
+					input => $input_name,
+					id => $mfn,
+					ds => $ds,
+				) if !$stats;
+
+				$indexer->add(
+					id => "${input_name}/${mfn}",
+					ds => $ds,
+					type => $config->get($indexer_config)->{type},
+				) if $indexer;
 
-				} else {
-					$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
+				foreach my $out ( @output_modules ) {
+					$out->add( $mfn, $ds ) if $out->can('add');
 				}
-			}
 
+			} else {
+				$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
+			}
 			if ($marc) {
 				my $i = 0;