--- trunk/run.pl	2009/05/18 18:51:40	1184
+++ trunk/run.pl	2009/07/22 09:23:36	1246
@@ -22,7 +22,7 @@
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
-use LWP::Simple;
+use LWP::Simple qw//;
 
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
@@ -383,7 +383,7 @@
 			next;
 		}
 
-		next if ($only_input && ($input_name !~ m#$only_input#i && $input->{type} !~ m#$only_input#i));
+		next if defined $only_input && $input_name !~ m#$only_input#i;
 
 		my $type = lc($input->{type});
 
@@ -417,8 +417,8 @@
 			$base =~ s{/[^/]+$}{};
 			mkpath $base unless -e $base;
 
-			my $rc = mirror( "$mirror/$path", $path );
-			if (is_error( $rc )) {
+			my $rc = LWP::Simple::mirror( "$mirror/$path", $path );
+			if (LWP::Simple::is_error( $rc )) {
 				die "can't mirror $mirror/$path -> $path [$rc]";
 			} else {
 				$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" );
@@ -542,7 +542,10 @@
 		}
 
 		my $rules = $parser->normalize_rules($database,$input_name);
-		$log->logwarn("no normalize rules for $database/$input_name") unless $rules;
+		if ( ! $rules ) {
+			$log->logwarn("no normalize rules for $database/$input_name", $input_db->input_module->can('normalize') ? " using normalize from input module" : '');
+			next;
+		}
 
 		$log->debug("parsed normalize rules:\n$rules");
 
@@ -603,6 +606,9 @@
 				push @{ $row->{'000'} }, $pos;
 			}
 
+			foreach my $out ( @output_modules ) {
+				$out->add_row( $mfn, $row ) if $out->can('add_row');
+			}
 
 			if ($validate) {
 				if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) {
@@ -613,9 +619,11 @@
 				next;	# validation doesn't create any output
 			}
 
+			my $ds;
+
 			if ($rules) {
 
-				my $ds = WebPAC::Normalize::data_structure(
+				$ds = WebPAC::Normalize::data_structure(
 					row => $row,
 					rules => $rules,
 					lookup => $lookup_hash,
@@ -632,32 +640,33 @@
 					},
 				);
 
-				$log->debug("ds = ", sub { dump($ds) });
-
-				if ( $ds ) {
+			} elsif ( $input_db->input_module->can('normalize') ) {
+				$ds = $input_db->input_module->normalize( $mfn );
+			}
 
-					$store->save_ds(
-						database => $database,
-						input => $input_name,
-						id => $mfn,
-						ds => $ds,
-					) if !$stats;
-
-					$indexer->add(
-						id => "${input_name}/${mfn}",
-						ds => $ds,
-						type => $config->get($indexer_config)->{type},
-					) if $indexer;
+			if ( $ds ) {
+				$log->debug("ds = ", sub { dump($ds) });
 
-					foreach my $out ( @output_modules ) {
-						$out->add( $mfn, $ds ) if $out->can('add');
-					}
+				$store->save_ds(
+					database => $database,
+					input => $input_name,
+					id => $mfn,
+					ds => $ds,
+				) if !$stats;
+
+				$indexer->add(
+					id => "${input_name}/${mfn}",
+					ds => $ds,
+					type => $config->get($indexer_config)->{type},
+				) if $indexer;
 
-				} else {
-					$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
+				foreach my $out ( @output_modules ) {
+					$out->add( $mfn, $ds ) if $out->can('add');
 				}
-			}
 
+			} else {
+				$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
+			}
 			if ($marc) {
 				my $i = 0;