--- trunk/scripts/est-spider	2006/01/25 23:38:57	89
+++ trunk/scripts/est-spider	2006/01/26 01:53:29	90
@@ -49,7 +49,14 @@
 #$basedir =~ s,/[^/]+$,/,;
 #require "$basedir/filter.pm";
 
-my $pdftotext = which('pdftotext');
+my $filter;
+foreach my $f (qw/pdftotext pstotext/) {
+	my $w = which($f);
+	if ($f) {
+		$filter->{$f} = $w;
+		print STDERR "using $f filter at $w\n" if ($verbose);
+	}
+}
 
 #my $mm = new File::MMagic('/usr/share/misc/file/magic');
 my $mm = new File::MMagic::XS();
@@ -59,8 +66,6 @@
 select(STDERR); $|=1;
 select(STDOUT); $|=1;
 
-print STDERR "using $pdftotext to convert pdf into html\n" if ($pdftotext && $verbose);
-
 my $db = new Search::Estraier::Node;
 $db->set_url($node_url);
 $db->set_auth('admin', 'admin');
@@ -74,10 +79,10 @@
 
 exit;
 
-sub dump_contents($$$$) {
-	my ($db,$contents,$mtime,$path) = @_;
+sub dump_contents {
+	my ($db,$contents,$mtime,$path,$size) = @_;
 
-	return unless ($contents);	# don't die on empty files
+	return unless (defined($contents));	# don't die on empty files
 
 	if ($exclude && $path =~ m/$exclude/i) {
 		print STDERR "skip: $path\n" if ($verbose);
@@ -85,7 +90,9 @@
 	}
 
 	use bytes;
-	my $size = length $contents;
+	if (! $size) {
+		$size = length $contents;
+	}
 
 	print STDERR " [$size]" if ($verbose);
 
@@ -105,11 +112,17 @@
 	$doc->add_attr('@size', $size);
 	$doc->add_attr('@mtime', $mtime);
 
-	# html2text
-	$contents =~ s#<[^>]+/*>##gs;
-	$contents =~ s#\s\s+# #gs;
+	if ($contents) {
+		# html2text
+		$contents =~ s#<[^>]+/*>##gs;
+		$contents =~ s#\s\s+# #gs;
 
-	$doc->add_text($iconv->convert($contents));
+		$doc->add_text($iconv->convert($contents));
+	}
+	# store path
+	$doc->add_hidden_text($path);
+	# boost title
+	$doc->add_hidden_text($title);
 
 #	print $doc->dump_draft if ($verbose);
 
@@ -118,41 +131,14 @@
 
 }
 
-sub file {
-
-	my $path = $_;
-	my $contents;
-
-	return if (! $force && -l $path || $path =~ m#/.svn# || $path =~ m/(~|.bak)$/);
-
-	my $mtime = (stat($path))[9] || -1;
-	my $mtime_db = $db->get_doc_attr_by_uri("file:///$path", '@mtime') || -2;
-
-	if ($mtime == $mtime_db) {
-		print STDERR "# same: $path $mtime\n" if ($verbose);
-		return unless($force);
-	} else {
-		print STDERR "# changed: $path $mtime != $mtime_db\n" if ($debug);
-	}
-
-	# skip files on which File::MMagic::XS croaks
-	return if ($path =~ m#\.au$#);
-
-	my $type = $mm->checktype_filename($path);
-	$type =~ s/\s+/ /gs;
-
-	print STDERR "# $path $type\n" if ($debug);
-
-	if ($pdftotext && -f $path && $type =~ m/pdf/i) {
+sub filter_to_pages {
+		my ($path, $mtime, $command) = @_;
 
 		print STDERR "$path {converting}" if ($verbose);
 
-		open(F,"$pdftotext -htmlmeta \"$path\" - |") || die "can't open $pdftotext with '$path'";
+		open(F,"$command |") || die "can't open $command with '$path': $!";
 		my $html;
 		while(<F>) {
-			# XXX why pdftotext barks if I try to use this is beyond me.
-			#$contents .= $_;
-
 			$html .= $_;
 		}
 		close(F);
@@ -185,13 +171,60 @@
 			$page_nr++;
 		}
 
+
+
+}
+
+sub file {
+
+	my $path = $_;
+	my $contents;
+
+	return if (! $force && -l $path || $path =~ m#/.svn# || $path =~ m/(~|.bak)$/);
+
+	my $mtime = (stat($path))[9] || -1;
+	my $mtime_db = $db->get_doc_attr_by_uri("file:///$path", '@mtime') || -2;
+
+	if ($mtime == $mtime_db) {
+		print STDERR "# same: $path $mtime\n" if ($verbose);
+		return unless($force);
+	} else {
+		print STDERR "# changed: $path $mtime != $mtime_db\n" if ($debug);
+	}
+
+	# skip files on which File::MMagic::XS croaks
+	if ($path =~ m#\.au$#) {
+		warn "skipped '$path' to prevent File::MMagic::XS bug\n" if ($debug);
+		return;
+	}
+
+	my $type = $mm->checktype_filename($path);
+	$type =~ s/\s+/ /gs;
+
+	print STDERR "# $path $type\n" if ($debug);
+
+	if ($type =~ m/pdf/i) {
+		if ($filter->{pdftotext}) {
+			filter_to_pages($path, $mtime, qq( $filter->{pdftotext} -htmlmeta "$path" - ));
+		} else {
+			warn "skipping '$path', no pdftotext filter\n" if ($verbose);
+			return;
+		}
+	} elsif ($type eq 'application/postscript') {
+		if ($filter->{pstotext}) {
+			filter_to_pages($path, $mtime, qq( $filter->{pstotext} "$path" ));
+		} else {
+			warn "skipping '$path', no pstotext filter\n" if ($verbose);
+			return;
+		}
 	} else {
 
 #		return if (! -f $path || ! m/\.(html*|php|pl|txt|info|log|text)$/i);
 		if (-f $path && $type =~ m/html/ ||
-			($type !~ m#text# && $path =~ m/\.(php|pl|txt|info|log|text)$/io)
+			($path !~ m/\.(php|pl|txt|info|log|text)$/io)
 		) {
-			dump_contents($db, "<title>$path</title> $type", $mtime, $path);
+			dump_contents($db, '', $mtime, $path, -s $path);
+			return;
 		}
 
 		# skip index files