--- trunk/run.pl	2007/11/03 12:34:26	977
+++ trunk/run.pl	2009/04/25 14:17:27	1160
@@ -23,7 +23,6 @@
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
 
-use Proc::Queue size => 1;
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
 =head1 NAME
@@ -153,6 +152,8 @@
 
 my $config = new WebPAC::Config( path => $config_path );
 
+WebPAC::Normalize::_debug( $debug - 1 ) if $debug > 1;
+
 #print "config = ",dump($config) if ($debug);
 
 die "no databases in config file!\n" unless ($config->databases);
@@ -205,6 +206,8 @@
 my @links;
 
 if ($parallel) {
+	eval 'use Proc::Queue size => 1;';
+	die $@ if $@;
 	$log->info("Using $parallel processes for speedup");
 	Proc::Queue::size($parallel);
 }
@@ -326,12 +329,15 @@
 
 		# add database to arugemnts for output filter
 		$output->{database} = $database;
+		$output->{clean} = $clean;
 
 		$log->debug("calling $module->new(",dump( $output ),")");
 		my $out = new $module->new( $output );
-		$out->init;
-
-		push @output_modules, $out;
+		if ( $out->init ) {
+			push @output_modules, $out;
+		} else {
+			$log->warn("SKIPPED $module");
+		}
 	}
 
 
@@ -344,10 +350,32 @@
 		$log->info("database $database doesn't have inputs defined");
 	} );
 
+	if ( -e 'out/debug' ) {	# FIXME flag?
+		my $out;
+		foreach my $i ( @inputs ) {
+			warn dump( $i );
+			next unless defined $i->{normalize};
+			warn dump( $i->{normalize} );
+			foreach my $normalize ( @{ $i->{normalize} } ) {
+				my $path = $normalize->{path};
+				$out .= qq/\n##\n## $path\n##\n\n/;
+				$out .= read_file( $path );
+			}
+		}
+		my $all = "out/debug/all-normalize.pl";
+		write_file( $all, $out );
+		warn "### all normalize for this input saved to: $all";
+	};
+
 	foreach my $input (@inputs) {
 
 		my $input_name = $input->{name} || $log->logdie("input without a name isn't valid: ",dump($input));
 
+		if ( $input->{skip} ) {
+			$log->info("skip $input_name");
+			next;
+		}
+
 		next if ($only_input && ($input_name !~ m#$only_input#i && $input->{type} !~ m#$only_input#i));
 
 		my $type = lc($input->{type});
@@ -365,7 +393,7 @@
 
 		my @lookups = $parser->have_lookup_create($database, $input);
 
-		$log->info("working on input '$input_name' in $input->{path} [type: $input->{type}] using $input_module",
+		$log->info("working on $database/$input_name with $input_module on $input->{path}",
 			@lookups ? " creating lookups: ".join(", ", @lookups) : ""
 		);
 
@@ -377,7 +405,6 @@
 
 		my $input_db = new WebPAC::Input(
 			module => $input_module,
-			encoding => $config->webpac('webpac_encoding'),
 			limit => $limit || $input->{limit},
 			offset => $offset,
 			recode => $input->{recode},
@@ -422,7 +449,7 @@
 
 		my $maxmfn = $input_db->open(
 			path => $input->{path},
-			code_page => $input->{encoding},	# database encoding
+			input_encoding => $input->{encoding},	# database encoding
 			lookup_coderef => $lookup_coderef,
 			lookup => $lookup_jar,
 			%{ $input },
@@ -485,7 +512,9 @@
 			);
 		}
 
-		my $rules = $parser->normalize_rules($database,$input_name) || $log->logdie("no normalize rules found for $database/$input_name");
+		my $rules = $parser->normalize_rules($database,$input_name);
+		$log->logwarn("no normalize rules for $database/$input_name") unless $rules;
+
 		$log->debug("parsed normalize rules:\n$rules");
 
 		# reset position in database
@@ -555,56 +584,59 @@
 				next;	# validation doesn't create any output
 			}
 
-			my $ds = WebPAC::Normalize::data_structure(
-				row => $row,
-				rules => $rules,
-				lookup => $lookup_hash,
-				config => create_ds_config( $db_config, $database, $input, $mfn ),
-				marc_encoding => 'utf-8',
-				load_row_coderef => sub {
-					my ($database,$input,$mfn) = @_;
-warn "### load_row($database,$input,$mfn) from data_structure\n";
-					return $store->load_row(
+			if ($rules) {
+
+				my $ds = WebPAC::Normalize::data_structure(
+					row => $row,
+					rules => $rules,
+					lookup => $lookup_hash,
+					config => create_ds_config( $db_config, $database, $input, $mfn ),
+					marc_encoding => 'utf-8',
+					load_row_coderef => sub {
+						my ($database,$input,$mfn) = @_;
+#warn "### load_row($database,$input,$mfn) from data_structure\n";
+						return $store->load_row(
+							database => $database,
+							input => $input,
+							id => $mfn,
+						);
+					},
+				);
+
+				$log->debug("ds = ", sub { dump($ds) });
+
+				if ( $ds ) {
+
+					$store->save_ds(
 						database => $database,
-						input => $input,
+						input => $input_name,
 						id => $mfn,
-					);
-				},
-			);
-
-			$log->debug("ds = ", sub { dump($ds) });
+						ds => $ds,
+					) if !$stats;
 
-			if ( $ds ) {
+					$indexer->add(
+						id => "${input_name}/${mfn}",
+						ds => $ds,
+						type => $config->get($indexer_config)->{type},
+					) if $indexer;
 
-				$store->save_ds(
-					database => $database,
-					input => $input_name,
-					id => $mfn,
-					ds => $ds,
-				) if !$stats;
-
-				$indexer->add(
-					id => "${input_name}/${mfn}",
-					ds => $ds,
-					type => $config->get($indexer_config)->{type},
-				) if $indexer;
+					foreach my $out ( @output_modules ) {
+						$out->add( $mfn, $ds ) if $out->can('add');
+					}
 
-				foreach my $out ( @output_modules ) {
-					$out->add( $mfn, $ds ) if $out->can('add');
+				} else {
+					$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
 				}
-
-			} else {
-				$log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
 			}
 
 			if ($marc) {
 				my $i = 0;
 
-				while (my $fields = WebPAC::Normalize::_get_marc_fields( fetch_next => 1 ) ) {
+				while (my $fields = WebPAC::Normalize::MARC::_get_marc_fields( fetch_next => 1 ) ) {
 					$marc->add(
 						id => $mfn . ( $i ? "/$i" : '' ),
 						fields => $fields,
-						leader => WebPAC::Normalize::_get_marc_leader(),
+						leader => WebPAC::Normalize::MARC::_get_marc_leader(),
 						row => $row,
 					);
 					$i++;
@@ -641,7 +673,7 @@
 		close($report_fh) if ($report_fh);
 	}
 
-	eval { $indexer->finish } if ($indexer && $indexer->can('finish'));
+	$indexer->finish if $indexer && $indexer->can('finish');
 
 	foreach my $out ( @output_modules ) {
 		$out->finish if $out->can('finish');