--- trunk/Estraier.pm	2006/01/07 16:19:31	63
+++ trunk/Estraier.pm	2006/05/08 21:33:37	132
@@ -4,7 +4,7 @@
 use strict;
 use warnings;
 
-our $VERSION = '0.01';
+our $VERSION = '0.06_1';
 
 =head1 NAME
 
@@ -12,8 +12,65 @@
 
 =head1 SYNOPSIS
 
-  use Search::Estraier;
-  my $est = new Search::Estraier();
+=head2 Simple indexer
+
+	use Search::Estraier;
+
+	# create and configure node
+	my $node = new Search::Estraier::Node(
+		url => 'http://localhost:1978/node/test',
+		user => 'admin',
+		passwd => 'admin'
+	);
+
+	# create document
+	my $doc = new Search::Estraier::Document;
+
+	# add attributes
+	$doc->add_attr('@uri', "http://estraier.gov/example.txt");
+	$doc->add_attr('@title', "Over the Rainbow");
+
+	# add body text to document
+	$doc->add_text("Somewhere over the rainbow.  Way up high.");
+	$doc->add_text("There's a land that I heard of once in a lullaby.");
+
+	die "error: ", $node->status,"\n" unless (eval { $node->put_doc($doc) });
+
+=head2 Simple searcher
+
+	use Search::Estraier;
+
+	# create and configure node
+	my $node = new Search::Estraier::Node(
+		url => 'http://localhost:1978/node/test',
+		user => 'admin',
+		passwd => 'admin',
+		croak_on_error => 1,
+	);
+
+	# create condition
+	my $cond = new Search::Estraier::Condition;
+
+	# set search phrase
+	$cond->set_phrase("rainbow AND lullaby");
+
+	my $nres = $node->search($cond, 0);
+
+	if (defined($nres)) {
+		print "Got ", $nres->hits, " results\n";
+
+		# for each document in results
+		for my $i ( 0 ... $nres->doc_num - 1 ) {
+			# get result document
+			my $rdoc = $nres->get_doc($i);
+			# display attribte
+			print "URI: ", $rdoc->attr('@uri'),"\n";
+			print "Title: ", $rdoc->attr('@title'),"\n";
+			print $rdoc->snippet,"\n";
+		}
+	} else {
+		die "error: ", $node->status,"\n";
+	}
 
 =head1 DESCRIPTION
 
@@ -25,6 +82,8 @@
 It is implemented as multiple packages which closly resamble Ruby
 implementation. It also includes methods to manage nodes.
 
+There are few examples in C<scripts> directory of this distribution.
+
 =cut
 
 =head1 Inheritable common methods
@@ -41,7 +100,8 @@
 =cut
 
 sub _s {
-	my $text = $_[1] || return;
+	my $text = $_[1];
+	return unless defined($text);
 	$text =~ s/\s\s+/ /gs;
 	$text =~ s/^\s+//;
 	$text =~ s/\s+$//;
@@ -106,12 +166,12 @@
 			} elsif ($line =~ m/^$/) {
 				$in_text = 1;
 				next;
-			} elsif ($line =~ m/^(.+)=(.+)$/) {
+			} elsif ($line =~ m/^(.+)=(.*)$/) {
 				$self->{attrs}->{ $1 } = $2;
 				next;
 			}
 
-			warn "draft ignored: $line\n";
+			warn "draft ignored: '$line'\n";
 		}
 	}
 
@@ -269,7 +329,8 @@
 	my $draft;
 
 	foreach my $attr_name (sort keys %{ $self->{attrs} }) {
-		$draft .= $attr_name . '=' . $self->{attrs}->{$attr_name} . "\n";
+		next unless defined(my $v = $self->{attrs}->{$attr_name});
+		$draft .= $attr_name . '=' . $v . "\n";
 	}
 
 	if ($self->{kwords}) {
@@ -317,7 +378,7 @@
 
 package Search::Estraier::Condition;
 
-use Carp qw/confess croak/;
+use Carp qw/carp confess croak/;
 
 use Search::Estraier;
 our @ISA = qw/Search::Estraier/;
@@ -395,30 +456,71 @@
 
 =head2 set_options
 
-  $cond->set_options( SURE => 1 );
+  $cond->set_options( 'SURE' );
+
+  $cond->set_options( qw/AGITO NOIDF SIMPLE/ );
+
+Possible options are:
+
+=over 8
+
+=item SURE
+
+check every N-gram
+
+=item USUAL
+
+check every second N-gram
+
+=item FAST
+
+check every third N-gram
+
+=item AGITO
+
+check every fourth N-gram
+
+=item NOIDF
+
+don't perform TF-IDF tuning
+
+=item SIMPLE
+
+use simplified query phrase
+
+=back
+
+Skipping N-grams will speed up search, but reduce accuracy. Every call to C<set_options> will reset previous
+options;
+
+This option changed in version C<0.04> of this module. It's backwards compatibile.
 
 =cut
 
 my $options = {
-	# check N-gram keys skipping by three
 	SURE => 1 << 0,
-	# check N-gram keys skipping by two
 	USUAL => 1 << 1,
-	# without TF-IDF tuning
 	FAST => 1 << 2,
-	# with the simplified phrase
 	AGITO => 1 << 3,
-	# check every N-gram key
 	NOIDF => 1 << 4,
-	# check N-gram keys skipping by one
 	SIMPLE => 1 << 10,
 };
 
 sub set_options {
 	my $self = shift;
-	my $option = shift;
-	confess "unknown option" unless ($options->{$option});
-	$self->{options} ||= $options->{$option};
+	my $opt = 0;
+	foreach my $option (@_) {
+		my $mask;
+		unless ($mask = $options->{$option}) {
+			if ($option eq '1') {
+				next;
+			} else {
+				croak "unknown option $option";
+			}
+		}
+		$opt += $mask;
+	}
+	$self->{options} = $opt;
 }
 
 
@@ -497,6 +599,35 @@
 }
 
 
+=head2 set_skip
+
+Set number of skipped documents from beginning of results
+
+  $cond->set_skip(42);
+
+Similar to C<offset> in RDBMS.
+
+=cut
+
+sub set_skip {
+	my $self = shift;
+	$self->{skip} = shift;
+}
+
+=head2 skip
+
+Return skip for this condition.
+
+  print $cond->skip;
+
+=cut
+
+sub skip {
+	my $self = shift;
+	return $self->{skip};
+}
+
+
 package Search::Estraier::ResultDocument;
 
 use Carp qw/croak/;
@@ -640,6 +771,9 @@
 
   print $res->doc_num;
 
+This will return real number of documents (limited by C<max>).
+If you want to get total number of hits, see C<hits>.
+
 =cut
 
 sub doc_num {
@@ -671,7 +805,7 @@
 
 Return specific hint from results.
 
-  print $rec->hint( 'VERSION' );
+  print $res->hint( 'VERSION' );
 
 Possible hints are: C<VERSION>, C<NODE>, C<HIT>, C<HINT#n>, C<DOCNUM>, C<WORDNUM>,
 C<TIME>, C<LINK#n>, C<VIEW>.
@@ -684,6 +818,37 @@
 	return $self->{hints}->{$key};
 }
 
+=head2 hints
+
+More perlish version of C<hint>. This one returns hash.
+
+  my %hints = $res->hints;
+
+=cut
+
+sub hints {
+	my $self = shift;
+	return $self->{hints};
+}
+
+=head2 hits
+
+Syntaxtic sugar for total number of hits for this query
+
+  print $res->hits;
+
+It's same as
+
+  print $res->hint('HIT');
+
+but shorter.
+
+=cut
+
+sub hits {
+	my $self = shift;
+	return $self->{hints}->{'HIT'} || 0;
+}
 
 package Search::Estraier::Node;
 
@@ -699,6 +864,48 @@
 
   my $node = new Search::HyperEstraier::Node;
 
+or optionally with C<url> as parametar
+
+  my $node = new Search::HyperEstraier::Node( 'http://localhost:1978/node/test' );
+
+or in more verbose form
+
+  my $node = new Search::HyperEstraier::Node(
+  	url => 'http://localhost:1978/node/test',
+	user => 'admin',
+	passwd => 'admin'
+	debug => 1,
+	croak_on_error => 1
+  );
+
+with following arguments:
+
+=over 4
+
+=item url
+
+URL to node
+
+=item user
+
+specify username for node server authentication
+
+=item passwd
+
+password for authentication
+
+=item debug
+
+dumps a B<lot> of debugging output
+
+=item croak_on_error
+
+very helpful during development. It will croak on all errors instead of
+silently returning C<-1> (which is convention of Hyper Estraier API in other
+languages).
+
+=back
+
 =cut
 
 sub new {
@@ -706,20 +913,29 @@
 	my $self = {
 		pxport => -1,
 		timeout => 0,	# this used to be -1
-		dnum => -1,
-		wnum => -1,
-		size => -1.0,
 		wwidth => 480,
 		hwidth => 96,
 		awidth => 96,
 		status => -1,
 	};
+
 	bless($self, $class);
 
-	my $args = {@_};
+	if ($#_ == 0) {
+		$self->{url} = shift;
+	} else {
+		%$self = ( %$self, @_ );
+
+		$self->set_auth( $self->{user}, $self->{passwd} ) if ($self->{user});
 
-	$self->{debug} = $args->{debug};
-	warn "## Node debug on\n" if ($self->{debug});
+		warn "## Node debug on\n" if ($self->{debug});
+	}
+
+	$self->{inform} = {
+		dnum => -1,
+		wnum => -1,
+		size => -1.0,
+	};
 
 	$self ? return $self : return undef;
 }
@@ -999,12 +1215,14 @@
 
   my $id = $node->uri_to_id( 'file:///document/uri/42' );
 
+This method won't croak, even if using C<croak_on_error>.
+
 =cut
 
 sub uri_to_id {
 	my $self = shift;
 	my $uri = shift || return;
-	return $self->_fetch_doc( uri => $uri, path => '/uri_to_id', chomp_resbody => 1 );
+	return $self->_fetch_doc( uri => $uri, path => '/uri_to_id', chomp_resbody => 1, croak_on_error => 0 );
 }
 
 
@@ -1064,6 +1282,7 @@
 		'application/x-www-form-urlencoded',
 		$arg,
 		\$resbody,
+		$a->{croak_on_error},
 	);
 
 	return if ($rv != 200);
@@ -1094,8 +1313,8 @@
 
 sub name {
 	my $self = shift;
-	$self->_set_info unless ($self->{name});
-	return $self->{name};
+	$self->_set_info unless ($self->{inform}->{name});
+	return $self->{inform}->{name};
 }
 
 
@@ -1107,8 +1326,8 @@
 
 sub label {
 	my $self = shift;
-	$self->_set_info unless ($self->{label});
-	return $self->{label};
+	$self->_set_info unless ($self->{inform}->{label});
+	return $self->{inform}->{label};
 }
 
 
@@ -1120,8 +1339,8 @@
 
 sub doc_num {
 	my $self = shift;
-	$self->_set_info if ($self->{dnum} < 0);
-	return $self->{dnum};
+	$self->_set_info if ($self->{inform}->{dnum} < 0);
+	return $self->{inform}->{dnum};
 }
 
 
@@ -1133,8 +1352,8 @@
 
 sub word_num {
 	my $self = shift;
-	$self->_set_info if ($self->{wnum} < 0);
-	return $self->{wnum};
+	$self->_set_info if ($self->{inform}->{wnum} < 0);
+	return $self->{inform}->{wnum};
 }
 
 
@@ -1146,8 +1365,8 @@
 
 sub size {
 	my $self = shift;
-	$self->_set_info if ($self->{size} < 0);
-	return $self->{size};
+	$self->_set_info if ($self->{inform}->{size} < 0);
+	return $self->{inform}->{size};
 }
 
 
@@ -1180,88 +1399,32 @@
 	);
 	return if ($rv != 200);
 
-	my (@docs, $hints);
-
-	my @lines = split(/\n/, $resbody);
-	return unless (@lines);
-
-	my $border = $lines[0];
-	my $isend = 0;
-	my $lnum = 1;
-
-	while ( $lnum <= $#lines ) {
-		my $line = $lines[$lnum];
-		$lnum++;
-
-		#warn "## $line\n";
-		if ($line && $line =~ m/^\Q$border\E(:END)*$/) {
-			$isend = $1;
-			last;
-		}
-
-		if ($line =~ /\t/) {
-			my ($k,$v) = split(/\t/, $line, 2);
-			$hints->{$k} = $v;
-		}
-	}
-
-	my $snum = $lnum;
-
-	while( ! $isend && $lnum <= $#lines ) {
-		my $line = $lines[$lnum];
-		#warn "# $lnum: $line\n";
-		$lnum++;
-
-		if ($line && $line =~ m/^\Q$border\E/) {
-			if ($lnum > $snum) {
-				my $rdattrs;
-				my $rdvector;
-				my $rdsnippet;
-				
-				my $rlnum = $snum;
-				while ($rlnum < $lnum - 1 ) {
-					#my $rdline = $self->_s($lines[$rlnum]);
-					my $rdline = $lines[$rlnum];
-					$rlnum++;
-					last unless ($rdline);
-					if ($rdline =~ /^%/) {
-						$rdvector = $1 if ($rdline =~ /^%VECTOR\t(.+)$/);
-					} elsif($rdline =~ /=/) {
-						$rdattrs->{$1} = $2 if ($rdline =~ /^(.+)=(.+)$/);
-					} else {
-						confess "invalid format of response";
-					}
-				}
-				while($rlnum < $lnum - 1) {
-					my $rdline = $lines[$rlnum];
-					$rlnum++;
-					$rdsnippet .= "$rdline\n";
-				}
-				#warn Dumper($rdvector, $rdattrs, $rdsnippet);
-				if (my $rduri = $rdattrs->{'@uri'}) {
-					push @docs, new Search::Estraier::ResultDocument(
-						uri => $rduri,
-						attrs => $rdattrs,
-						snippet => $rdsnippet,
-						keywords => $rdvector,
-					);
-				}
-			}
-			$snum = $lnum;
-			#warn "### $line\n";
-			$isend = 1 if ($line =~ /:END$/);
-		}
-
-	}
-
-	if (! $isend) {
-		warn "received result doesn't have :END\n$resbody";
-		return;
+	my @records 	= split /--------\[.*?\]--------(?::END)?\r?\n/, $resbody;
+	my $hintsText	= splice @records, 0, 2; # starts with empty record
+	my $hints		= { $hintsText =~ m/^(.*?)\t(.*?)$/gsm };
+
+	# process records
+	my $docs = [];
+	foreach my $record (@records)
+	{
+		# split into keys and snippets
+		my ($keys, $snippet) = $record =~ m/^(.*?)\n\n(.*?)$/s;
+
+		# create document hash
+		my $doc				= { $keys =~ m/^(.*?)=(.*?)$/gsm };
+		$doc->{'@keywords'}	= $doc->{keywords};
+		($doc->{keywords})	= $keys =~ m/^%VECTOR\t(.*?)$/gm;
+		$doc->{snippet}		= $snippet;
+
+		push @$docs, new Search::Estraier::ResultDocument(
+			attrs 		=> $doc,
+			uri 		=> $doc->{'@uri'},
+			snippet 	=> $snippet,
+			keywords 	=> $doc->{'keywords'},
+		);
 	}
 
-	#warn Dumper(\@docs, $hints);
-
-	return new Search::Estraier::NodeResult( docs => \@docs, hints => $hints );
+	return new Search::Estraier::NodeResult( docs => $docs, hints => $hints );
 }
 
 
@@ -1310,6 +1473,7 @@
 	push @args, 'wwidth=' . $self->{wwidth};
 	push @args, 'hwidth=' . $self->{hwidth};
 	push @args, 'awidth=' . $self->{awidth};
+	push @args, 'skip=' . $self->{skip} if ($self->{skip});
 
 	return join('&', @args);
 }
@@ -1317,7 +1481,7 @@
 
 =head2 shuttle_url
 
-This is method which uses C<IO::Socket::INET> to communicate with Hyper Estraier node
+This is method which uses C<LWP::UserAgent> to communicate with Hyper Estraier node
 master.
 
   my $rv = shuttle_url( $url, $content_type, $req_body, \$resbody );
@@ -1332,7 +1496,9 @@
 sub shuttle_url {
 	my $self = shift;
 
-	my ($url, $content_type, $reqbody, $resbody) = @_;
+	my ($url, $content_type, $reqbody, $resbody, $croak_on_error) = @_;
+
+	$croak_on_error = $self->{croak_on_error} unless defined($croak_on_error);
 
 	$self->{status} = -1;
 
@@ -1359,7 +1525,7 @@
 
 	$req->headers->header( 'Host' => $url->host . ":" . $url->port );
 	$req->headers->header( 'Connection', 'close' );
-	$req->headers->header( 'Authorization', 'Basic ' . $self->{auth} );
+	$req->headers->header( 'Authorization', 'Basic ' . $self->{auth} ) if ($self->{auth});
 	$req->content_type( $content_type );
 
 	warn $req->headers->as_string,"\n" if ($self->{debug});
@@ -1373,10 +1539,16 @@
 
 	warn "## response status: ",$res->status_line,"\n" if ($self->{debug});
 
-	return -1 if (! $res->is_success);
-
 	($self->{status}, $self->{status_message}) = split(/\s+/, $res->status_line, 2);
 
+	if (! $res->is_success) {
+		if ($croak_on_error) {
+			croak("can't get $url: ",$res->status_line);
+		} else {
+			return -1;
+		}
+	}
+
 	$$resbody .= $res->content;
 
 	warn "## response body:\n$$resbody\n" if ($resbody && $self->{debug});
@@ -1475,11 +1647,57 @@
 	my $reqbody = 'url=' . uri_escape($url) . '&label=' . uri_escape($label);
 	$reqbody .= '&credit=' . $credit if ($credit > 0);
 
-	$self->shuttle_url( $self->{url} . '/_set_link',
-		'text/plain',
+	if ($self->shuttle_url( $self->{url} . '/_set_link',
+		'application/x-www-form-urlencoded',
 		$reqbody,
 		undef
-	) == 200;
+	) == 200) {
+		# refresh node info after adding link
+		$self->_set_info;
+		return 1;
+	}
+}
+
+=head2 admins
+
+ my @admins = @{ $node->admins };
+
+Return array of users with admin rights on node
+
+=cut
+
+sub admins {
+	my $self = shift;
+	$self->_set_info unless ($self->{inform}->{name});
+	return $self->{inform}->{admins};
+}
+
+=head2 guests
+
+ my @guests = @{ $node->guests };
+
+Return array of users with guest rights on node
+
+=cut
+
+sub guests {
+	my $self = shift;
+	$self->_set_info unless ($self->{inform}->{name});
+	return $self->{inform}->{guests};
+}
+
+=head2 links
+
+ my $links = @{ $node->links };
+
+Return array of links for this node
+
+=cut
+
+sub links {
+	my $self = shift;
+	$self->_set_info unless ($self->{inform}->{name});
+	return $self->{inform}->{links};
 }
 
 
@@ -1510,11 +1728,30 @@
 
 	return if ($rv != 200 || !$resbody);
 
-	# it seems that response can have multiple line endings
-	$resbody =~ s/[\r\n]+$//;
+	my @lines = split(/[\r\n]/,$resbody);
+
+	$self->{inform} = {};
+
+	( $self->{inform}->{name}, $self->{inform}->{label}, $self->{inform}->{dnum},
+		$self->{inform}->{wnum}, $self->{inform}->{size} ) = split(/\t/, shift @lines, 5);
+
+	return $resbody unless (@lines);
+
+	shift @lines;
+
+	while(my $admin = shift @lines) {
+		push @{$self->{inform}->{admins}}, $admin;
+	}
+
+	while(my $guest = shift @lines) {
+		push @{$self->{inform}->{guests}}, $guest;
+	}
+
+	while(my $link = shift @lines) {
+		push @{$self->{inform}->{links}}, $link;
+	}
 
-	( $self->{name}, $self->{label}, $self->{dnum}, $self->{wnum}, $self->{size} ) =
-		split(/\t/, $resbody, 5);
+	return $resbody;
 
 }
 
@@ -1534,6 +1771,7 @@
 
 Dobrica Pavlinusic, E<lt>dpavlin@rot13.orgE<gt>
 
+Robert Klep E<lt>robert@klep.nameE<gt> contributed refactored search code
 
 =head1 COPYRIGHT AND LICENSE