--- trunk/run.pl	2009/04/21 21:03:53	1129
+++ trunk/run.pl	2009/05/18 18:47:22	1183
@@ -11,7 +11,7 @@
 use WebPAC::Input 0.16;
 use WebPAC::Store 0.15;
 use WebPAC::Normalize 0.22;
-use WebPAC::Output::TT;
+#use WebPAC::Output::TT;
 use WebPAC::Validate 0.11;
 use WebPAC::Output::MARC;
 use WebPAC::Config;
@@ -22,6 +22,7 @@
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
+use LWP::Simple;
 
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
@@ -102,6 +103,10 @@
 
 Create merged index of databases which have links
 
+=item --mirror http://www.example.com
+
+Tries to download input path files from mirror URI
+
 =back
 
 =cut
@@ -122,6 +127,7 @@
 my $parallel = 0;
 my $only_links = 0;
 my $merge = 0;
+my $mirror;
 my $help;
 
 my $log = _new WebPAC::Common()->_get_logger();
@@ -143,6 +149,7 @@
 	"parallel=i" => \$parallel,
 	"only-links!" => \$only_links,
 	"merge" => \$merge,
+	"mirror=s" => \$mirror,
 	"help" => \$help,
 );
 
@@ -371,6 +378,11 @@
 
 		my $input_name = $input->{name} || $log->logdie("input without a name isn't valid: ",dump($input));
 
+		if ( $input->{skip} ) {
+			$log->info("skip $input_name");
+			next;
+		}
+
 		next if ($only_input && ($input_name !~ m#$only_input#i && $input->{type} !~ m#$only_input#i));
 
 		my $type = lc($input->{type});
@@ -398,6 +410,22 @@
 			delete($input->{modify_file});
 		}
 
+		if ( $mirror ) {
+			my $path = $input->{path} || die "no input path in ",dump( $input );
+
+			my $base = $path;
+			$base =~ s{/[^/]+$}{};
+			mkpath $base unless -e $base;
+
+			my $rc = mirror( "$mirror/$path", $path );
+			if (is_error( $rc )) {
+				die "can't mirror $mirror/$path -> $path [$rc]";
+			} else {
+				$log->info( "mirror ", $path, " [$rc] ", -s $path, " bytes" );
+			}
+			
+		}
+
 		my $input_db = new WebPAC::Input(
 			module => $input_module,
 			limit => $limit || $input->{limit},