--- trunk/run.pl	2007/12/23 22:06:08	1084
+++ trunk/run.pl	2009/05/28 10:15:20	1198
@@ -11,7 +11,7 @@
 use WebPAC::Input 0.16;
 use WebPAC::Store 0.15;
 use WebPAC::Normalize 0.22;
-use WebPAC::Output::TT;
+#use WebPAC::Output::TT;
 use WebPAC::Validate 0.11;
 use WebPAC::Output::MARC;
 use WebPAC::Config;
@@ -22,8 +22,8 @@
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
+use LWP::Simple qw//;
 
-use Proc::Queue size => 1;
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
 =head1 NAME
@@ -103,6 +103,10 @@
 
 Create merged index of databases which have links
 
+=item --mirror http://www.example.com
+
+Tries to download input path files from mirror URI
+
 =back
 
 =cut
@@ -123,6 +127,7 @@
 my $parallel = 0;
 my $only_links = 0;
 my $merge = 0;
+my $mirror;
 my $help;
 
 my $log = _new WebPAC::Common()->_get_logger();
@@ -144,6 +149,7 @@
 	"parallel=i" => \$parallel,
 	"only-links!" => \$only_links,
 	"merge" => \$merge,
+	"mirror=s" => \$mirror,
 	"help" => \$help,
 );
 
@@ -207,6 +213,8 @@
 my @links;
 
 if ($parallel) {
+	eval 'use Proc::Queue size => 1;';
+	die $@ if $@;
 	$log->info("Using $parallel processes for speedup");
 	Proc::Queue::size($parallel);
 }
@@ -349,10 +357,32 @@
 		$log->info("database $database doesn't have inputs defined");
 	} );
 
+	if ( -e 'out/debug' ) {	# FIXME flag?
+		my $out;
+		foreach my $i ( @inputs ) {
+			warn dump( $i );
+			next unless defined $i->{normalize};
+			warn dump( $i->{normalize} );
+			foreach my $normalize ( @{ $i->{normalize} } ) {
+				my $path = $normalize->{path};
+				$out .= qq/\n##\n## $path\n##\n\n/;
+				$out .= read_file( $path );
+			}
+		}
+		my $all = "out/debug/all-normalize.pl";
+		write_file( $all, $out );
+		warn "### all normalize for this input saved to: $all";
+	};
+
 	foreach my $input (@inputs) {
 
 		my $input_name = $input->{name} || $log->logdie("input without a name isn't valid: ",dump($input));
 
+		if ( $input->{skip} ) {
+			$log->info("skip $input_name");
+			next;
+		}
+
 		next if ($only_input && ($input_name !~ m#$only_input#i && $input->{type} !~ m#$only_input#i));
 
 		my $type = lc($input->{type});
@@ -380,9 +410,24 @@
 			delete($input->{modify_file});
 		}
 
+		if ( $mirror ) {
+			my $path = $input->{path} || die "no input path in ",dump( $input );
+
+			my $base = $path;
+			$base =~ s{/[^/]+$}{};
+			mkpath $base unless -e $base;
+
+			my $rc = LWP::Simple::mirror( "$mirror/$path", $path );
+			if (LWP::Simple::is_error( $rc )) {
+				die "can't mirror $mirror/$path -> $path [$rc]";
+			} else {
+				$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" );
+			}
+			
+		}
+
 		my $input_db = new WebPAC::Input(
 			module => $input_module,
-			encoding => $config->webpac('webpac_encoding'),
 			limit => $limit || $input->{limit},
 			offset => $offset,
 			recode => $input->{recode},
@@ -427,7 +472,7 @@
 
 		my $maxmfn = $input_db->open(
 			path => $input->{path},
-			code_page => $input->{encoding},	# database encoding
+			input_encoding => $input->{encoding},	# database encoding
 			lookup_coderef => $lookup_coderef,
 			lookup => $lookup_jar,
 			%{ $input },
@@ -468,7 +513,9 @@
 
 		my $report_fh;
 		if ($stats || $validate) {
-			my $path = "out/report/${database}-${input_name}.txt";
+			my $out_report = 'out/report'; # FIXME move to config
+			mkpath $out_report unless -e $out_report;
+			my $path = "$out_report/${database}-${input_name}.txt";
 			open($report_fh, '>', $path) || $log->logdie("can't open $path: $!");
 
 			print $report_fh "Report for database '$database' input '$input_name' records ",
@@ -483,8 +530,12 @@
 
 		my $marc;
 		if ($marc_generate && $parser->have_rules( 'marc', $database, $input_name )) {
+			
+			my $out_marc = 'out/marc'; # FIXME move to config
+			mkpath $out_marc unless -e $out_marc;
+
 			$marc = new WebPAC::Output::MARC(
-				path => "out/marc/${database}-${input_name}.marc",
+				path => "$out_marc/${database}-${input_name}.marc",
 				lint => $marc_lint,
 				dump => $marc_dump,
 			);
@@ -552,6 +603,9 @@
 				push @{ $row->{'000'} }, $pos;
 			}
 
+			foreach my $out ( @output_modules ) {
+				$out->add_row( $mfn, $row ) if $out->can('add_row');
+			}
 
 			if ($validate) {
 				if ( my $errors = $validate->validate_rec( $row, $input_db->dump_ascii ) ) {
@@ -651,7 +705,7 @@
 		close($report_fh) if ($report_fh);
 	}
 
-	eval { $indexer->finish } if ($indexer && $indexer->can('finish'));
+	$indexer->finish if $indexer && $indexer->can('finish');
 
 	foreach my $out ( @output_modules ) {
 		$out->finish if $out->can('finish');