--- trunk/scripts/est-spider	2006/01/07 23:46:10	64
+++ trunk/scripts/est-spider	2006/08/25 11:59:04	179
@@ -3,14 +3,13 @@
 use File::Find;
 use Getopt::Long;
 use File::Which;
-use HyperEstraier;
 use Search::Estraier;
 use Text::Iconv;
 #use File::MMagic;
 use File::MMagic::XS qw/:compat/;
-
-# do we use Node API?
-my $node_url;
+use Time::HiRes qw/time/;
+use HTML::TreeBuilder;
+use Data::Dump qw/dump/;
 
 my $collection;		# name which will be inserted
 my $path_add;		# add additional info in path
@@ -20,7 +19,7 @@
 #$verbose = 1;
 my $debug = 0;
 my $force = 0;
-my $native = 0;
+my $all = 0;
 
 my $result = GetOptions(
 	"collection=s" => \$collection,
@@ -28,12 +27,24 @@
 	"verbose!" => \$verbose,
 	"debug!" => \$debug,
 	"exclude=s" => \$exclude,
-	"node=s" => \$node_url,
 	"force!" => \$force,
-	"native!" => \$native,
+	"all!" => \$all,
 );
 
-my $dir = shift @ARGV || die "usage: $0 [dir]";
+my ($node_url,$dir) = @ARGV;
+
+die <<"_END_OF_USAGE_" if (! $node_url || ! $dir);
+usage: $0 http://localhost:1978/node/my_dir /path/to/directory
+
+options:
+	--collection="name of collection"
+	--path=/path/to/add/at/end
+	--exclude=regex_to_exclude
+	--verbose
+	--force
+	--debug
+	--all	save placeholders for all files
+_END_OF_USAGE_
 
 if (! -e $dir) {
 	warn "directory $dir doesn't exist, skipping\n";
@@ -44,7 +55,17 @@
 #$basedir =~ s,/[^/]+$,/,;
 #require "$basedir/filter.pm";
 
-my $pdftotext = which('pdftotext');
+my $docs = 0;
+my $start_t = time();
+
+my $filter;
+foreach my $f (qw/pdftotext pstotext/) {
+	my $w = which($f);
+	if ($f) {
+		$filter->{$f} = $w;
+		print STDERR "using $f filter at $w\n" if ($verbose);
+	}
+}
 
 #my $mm = new File::MMagic('/usr/share/misc/file/magic');
 my $mm = new File::MMagic::XS();
@@ -54,52 +75,128 @@
 select(STDERR); $|=1;
 select(STDOUT); $|=1;
 
-print STDERR "using $pdftotext to convert pdf into html\n" if ($pdftotext && $verbose);
+my $db = new Search::Estraier::Node(
+	url => $node_url,
+	user => 'admin',
+	passwd => 'admin',
+	croak_on_error => 1,
+	create => 1,
+);
+
+#
+# check if hhc file exists, and if it does, extract information from it
+#
+
+my $hhc_file;
+# try to find hhc
+find({ wanted => sub {
+		return unless( m!\.hhc$!i );
+		$hhc_file = $_;
+		warn "using $hhc_file for tree structure\n";
+	},
+	follow => 1,
+	follow_skip => 2,
+	no_chdir => 1,
+}, $dir);
 
-my $db;
-if ($node_url) {
-	if ($native) {
-		$db = HyperEstraier::Node->new($node_url);
-	} else {
-		$db = new Search::Estraier::Node;
-		$db->set_url($node_url);
+my $meta;
+
+if ($hhc_file) {
+
+	sub param {
+		my ($el) = @_;
+
+		my $n;
+		foreach my $p ( $el->find('param') ) {
+			$n->{ $p->attr('name') } = $p->attr('value');
+		}
+
+		if ( ! defined($n->{Local}) ) {
+			warn "### skipped = ",dump($n),$/;
+			return;
+		}
+
+		my $path = $dir . '/' . $n->{Local};
+		$path =~ s!//!/!g;
+		$path = lc($path);
+
+		$n->{path} = $path;
+
+		my $nr = $n->{ImageNumber} || next;
+
+		if ($nr == 27) {
+			$meta->{title} = $n->{Name};
+			$meta->{index_path} = $path;
+		} elsif ($nr == 21) {
+			$meta->{toc_path} = $path;
+		} elsif ($nr == 1) {
+			$meta->{foreword_path} = $path;
+		} elsif ($nr == 11) {
+			# nop
+		} else {
+			warn "unknown ImageNumber: $nr\n";
+		}
+
+		return $n;
 	}
-	$db->set_auth('admin', 'admin');
-} else {
-	# open the database
-	$db = HyperEstraier::Database->new();
-	$db->open('/tmp/casket', $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
-
-	sub signal {
-		my($sig) = @_;
-		print "\nCaught a SIG$sig--syncing database and shutting down\n";
-		$db->sync();
-		exit(0);
+
+	my $tree = HTML::TreeBuilder->new;
+	$tree->parse_file($hhc_file);
+
+	my $prefix = $collection ? ( $collection . ' :: ' ) : '';
+
+	my @prefix;
+	my $depth = 0;
+
+	foreach my $e ( $tree->look_down( sub { $_[0]->tag =~ m/(object)/ } ) ) {
+
+#		printf("%05s %s\n", $e->parent->address(), $e->as_HTML() );
+
+		my $l = ($e->depth() / 2) - 1;
+
+		$prefix[ 0 ] = $meta->{title} || '';
+
+		my $n = param($e);
+		$prefix[ $l ] = $n->{Name};
+
+		next unless ($n->{path});
+
+		my $t = '';
+		my @p;
+		foreach my $i ( 0 .. $l ) {
+			push @p, $prefix[ $i ] if ($prefix[ $i ]);
+		}
+		$t = join(' :: ', @p ) if (@p);
+
+		$meta->{path2title}->{ $n->{path} } = $t;
+
 	}
 
-	$SIG{'INT'}  = \&signal;
-	$SIG{'QUIT'} = \&signal;
+	$tree->delete;
+
 }
 
+
 find({ wanted => \&file, 
 	follow => 1,
 	follow_skip => 2,
 	no_chdir => 1,
 }, $dir);
 
-unless ($node_url) {
-	print "--- sync\n";
-	$db->sync();
+my $dur = (time() - $start_t) || 1;
+printf STDERR "%d documents in %.2fs [%.2f docs/s]\n", $docs, $dur, ($docs / $dur);
+
+$db->master(
+	action => 'sync'
+);
+
 
-	print "--- optimize...\n";
-	$db->optimize(0);
-}
 exit;
 
-sub dump_contents($$$$) {
-	my ($db,$contents,$mtime,$path) = @_;
+sub dump_contents {
+	my ($db,$contents,$mtime,$path,$size) = @_;
 
-	return unless ($contents);	# don't die on empty files
+	return unless (defined($contents));	# don't die on empty files
 
 	if ($exclude && $path =~ m/$exclude/i) {
 		print STDERR "skip: $path\n" if ($verbose);
@@ -107,24 +204,30 @@
 	}
 
 	use bytes;
-	my $size = length $contents;
+	if (! $size) {
+		$size = length $contents;
+	}
 
 	print STDERR " [$size]" if ($verbose);
 
 	# create a document object 
-	my $doc;
-	if ($native) {
-		$doc = HyperEstraier::Document->new;
+	my $doc = new Search::Estraier::Document;
+
+	my $title;
+
+	if ( defined($meta->{path2title}->{lc($path)}) ) {
+		$title = $meta->{path2title}->{lc($path)};
+		warn " $title\n";
 	} else {
-		$doc = new Search::Estraier::Document;
-	}
 
-	my $title = $1 if ($contents =~ m#<title>(.+?)</title>#is);
+		$title = $1 if ($contents =~ m#<title>(.+?)</title>#is);
 
-	# chop long titles to 100 chars
-	$title = substr($title, 0, 100) . '...' if ($title && length($title) > 100);
-	# use path if no title is found
-	$title ||= $path;
+		# chop long titles to 100 chars
+		$title = substr($title, 0, 100) . '...' if ($title && length($title) > 100);
+		# use path if no title is found
+		$title ||= $path;
+	
+	}
 
 	# add attributes to the document object 
 	$doc->add_attr('@uri', "file:///$path");
@@ -132,58 +235,35 @@
 	$doc->add_attr('@size', $size);
 	$doc->add_attr('@mtime', $mtime);
 
-	# html2text
-	$contents =~ s#<[^>]+/*>##gs;
-	$contents =~ s#\s\s+# #gs;
-
-	$doc->add_text($iconv->convert($contents));
+	if ($contents) {
+		# html2text
+		$contents =~ s#<[^>]+/*>##gs;
+		$contents =~ s#\s\s+# #gs;
 
-#	print $doc->dump_draft if ($verbose);
-
-	# register the document object to the database
-	if ($node_url) {
-		$db->put_doc($doc);
-	} else {
-		$db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
+		$doc->add_text($iconv->convert($contents));
 	}
+	# store path
+	$doc->add_hidden_text($path);
+	# boost title
+	$doc->add_hidden_text($title);
 
-}
-
-sub file {
+	print $doc->dump_draft if ($debug);
 
-	my $path = $_;
-	my $contents;
-
-	return if (! $force && -l $path || $path =~ m#/.svn# || $path =~ m/(~|.bak)$/);
-
-	my $mtime = (stat($path))[9] || -1;
-	my $mtime_db = $db->get_doc_attr_by_uri("file:///$path", '@mtime') || -2;
-
-	if ($mtime == $mtime_db) {
-		print STDERR "# same: $path $mtime\n" if ($verbose);
-		return unless($force);
-	} else {
-		print STDERR "# changed: $path $mtime != $mtime_db\n" if ($debug);
-	}
-
-	# skip files on which File::MMagic::XS croaks
-	return if ($path =~ m#\.au$#);
+	# register the document object to the database
+	$db->put_doc($doc);
 
-	my $type = $mm->checktype_filename($path);
-	$type =~ s/\s+/ /gs;
+	$docs++;
 
-	print STDERR "# $path $type\n" if ($debug);
+}
 
-	if ($pdftotext && -f $path && $type =~ m/pdf/i) {
+sub filter_to_pages {
+		my ($path, $mtime, $command) = @_;
 
 		print STDERR "$path {converting}" if ($verbose);
 
-		open(F,"$pdftotext -htmlmeta \"$path\" - |") || die "can't open $pdftotext with '$path'";
+		open(F,"$command |") || die "can't open $command with '$path': $!";
 		my $html;
 		while(<F>) {
-			# XXX why pdftotext barks if I try to use this is beyond me.
-			#$contents .= $_;
-
 			$html .= $_;
 		}
 		close(F);
@@ -216,15 +296,67 @@
 			$page_nr++;
 		}
 
+
+
+}
+
+sub file {
+
+	my $path = $_;
+	my $contents;
+
+	return if (! $force && -l $path || $path =~ m#/.svn# || $path =~ m/(~|.bak|.gif)$/);
+	return if (! $all && -d $path);
+
+	my $mtime = (stat($path))[9] || -1;
+	my $mtime_db = eval { $db->get_doc_attr_by_uri("file:///$path", '@mtime') };
+	$mtime_db ||= -2;
+
+	if ($mtime == $mtime_db) {
+		print STDERR "# same: $path $mtime\n" if ($verbose);
+		return unless($force);
+	} else {
+		print STDERR "# changed: $path $mtime != $mtime_db\n" if ($debug);
+	}
+
+	# skip files on which File::MMagic::XS croaks
+	if ($path =~ m#\.au$#) {
+		warn "skipped '$path' to prevent File::MMagic::XS bug\n" if ($debug);
+		return;
+	}
+
+	my $type = $mm->checktype_filename($path);
+	$type =~ s/\s+/ /gs;
+
+	print STDERR "# $path $type\n" if ($debug);
+
+	if ($type =~ m/pdf/i) {
+		if ($filter->{pdftotext}) {
+			filter_to_pages($path, $mtime, qq( $filter->{pdftotext} -htmlmeta "$path" - ));
+		} else {
+			warn "skipping '$path', no pdftotext filter\n" if ($verbose);
+			return;
+		}
+	} elsif ($type eq 'application/postscript') {
+		if ($filter->{pstotext}) {
+			filter_to_pages($path, $mtime, qq( $filter->{pstotext} "$path" ));
+		} else {
+			warn "skipping '$path', no pstotext filter\n" if ($verbose);
+			return;
+		}
 	} else {
 
 #		return if (! -f $path || ! m/\.(html*|php|pl|txt|info|log|text)$/i);
-		return unless (-f $path && $type =~ m/html/ ||
-			($type =~ m#text# && $path =~ m/\.(php|pl|txt|info|log|text)$/io)
-		);
+		if (-f $path &&
+			$type !~ m/html/ &&
+			$path !~ m/\.(php|pl|txt|info|log|text)$/io
+		) {
+			dump_contents($db, '', $mtime, $path, -s $path) if ($all);
+			return;
+		}
 
 		# skip index files
-		return if (m/index_[a-z]\.html*/i || m/index_symbol\.html*/i);
+		return if ($path =~ m/index_(?:[a-z]+|symbol)\.html*/i);
 
 		open(F,"$path") || die "can't open file: $path";
 		print STDERR "$path ($type)" if ($verbose);