--- trunk/scripts/est-spider	2006/01/16 21:34:14	77
+++ trunk/scripts/est-spider	2006/01/25 23:38:57	89
@@ -3,15 +3,11 @@
 use File::Find;
 use Getopt::Long;
 use File::Which;
-use HyperEstraier;
 use Search::Estraier;
 use Text::Iconv;
 #use File::MMagic;
 use File::MMagic::XS qw/:compat/;
 
-# do we use Node API?
-my $node_url;
-
 my $collection;		# name which will be inserted
 my $path_add;		# add additional info in path
 my $verbose;
@@ -20,7 +16,6 @@
 #$verbose = 1;
 my $debug = 0;
 my $force = 0;
-my $native = 0;
 
 my $result = GetOptions(
 	"collection=s" => \$collection,
@@ -28,12 +23,22 @@
 	"verbose!" => \$verbose,
 	"debug!" => \$debug,
 	"exclude=s" => \$exclude,
-	"node=s" => \$node_url,
 	"force!" => \$force,
-	"native!" => \$native,
 );
 
-my $dir = shift @ARGV || die "usage: $0 [dir]";
+my ($node_url,$dir) = @ARGV;
+
+die <<"_END_OF_USAGE_" if (! $node_url || ! $dir);
+usage: $0 http://localhost:1978/node/my_dir /path/to/directory
+
+options:
+	--collection="name of collection"
+	--path=/path/to/add/at/end
+	--exclude=regex_to_exclude
+	--verbose
+	--force
+	--debug
+_END_OF_USAGE_
 
 if (! -e $dir) {
 	warn "directory $dir doesn't exist, skipping\n";
@@ -56,30 +61,9 @@
 
 print STDERR "using $pdftotext to convert pdf into html\n" if ($pdftotext && $verbose);
 
-my $db;
-if ($node_url) {
-	if ($native) {
-		$db = HyperEstraier::Node->new($node_url);
-	} else {
-		$db = new Search::Estraier::Node;
-		$db->set_url($node_url);
-	}
-	$db->set_auth('admin', 'admin');
-} else {
-	# open the database
-	$db = HyperEstraier::Database->new();
-	$db->open('/tmp/casket', $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
-
-	sub signal {
-		my($sig) = @_;
-		print "\nCaught a SIG$sig--syncing database and shutting down\n";
-		$db->sync();
-		exit(0);
-	}
-
-	$SIG{'INT'}  = \&signal;
-	$SIG{'QUIT'} = \&signal;
-}
+my $db = new Search::Estraier::Node;
+$db->set_url($node_url);
+$db->set_auth('admin', 'admin');
 
 find({ wanted => \&file, 
 	follow => 1,
@@ -87,13 +71,7 @@
 	no_chdir => 1,
 }, $dir);
 
-unless ($node_url) {
-	print "--- sync\n";
-	$db->sync();
 
-	print "--- optimize...\n";
-	$db->optimize(0);
-}
 exit;
 
 sub dump_contents($$$$) {
@@ -112,12 +90,7 @@
 	print STDERR " [$size]" if ($verbose);
 
 	# create a document object 
-	my $doc;
-	if ($native) {
-		$doc = HyperEstraier::Document->new;
-	} else {
-		$doc = new Search::Estraier::Document;
-	}
+	my $doc = new Search::Estraier::Document;
 
 	my $title = $1 if ($contents =~ m#<title>(.+?)</title>#is);
 
@@ -141,11 +114,7 @@
 #	print $doc->dump_draft if ($verbose);
 
 	# register the document object to the database
-	if ($node_url) {
-		$db->put_doc($doc);
-	} else {
-		$db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
-	}
+	$db->put_doc($doc);
 
 }