WebPAC/Output/Estraier.pm

package WebPAC::Output::Estraier;

use warnings;
use strict;

use base qw/WebPAC::Common/;

use Search::Estraier 0.06;
use Encode qw/from_to/;
use Data::Dumper;
use LWP;
use URI::Escape;

=head1 NAME

WebPAC::Output::Estraier - Create Hyper Estraier full text index

=head1 VERSION

Version 0.11

=cut

our $VERSION = '0.11';

=head1 SYNOPSIS

Create full text index using Hyper Estraier index from data with
type C<search>.

=head1 FUNCTIONS

=head2 new

Connect to Hyper Estraier index using HTTP

 my $est = new WebPAC::Output::Estraier(
        masterurl => 'http://localhost:1978/',
        user => 'admin',
        passwd => 'admin',
        database => 'demo',
        label => 'node label',
        encoding => 'iso-8859-2',
        clean => 1,
 );

Options are:

=over 4

=item masterurl

URI to C<estmaster> node

=item user

C<estmaster> user with sufficient rights

=item passwd

password for user

=item database

name of database from which data comes

=item label

label for node (optional)

=item encoding

character encoding of C<data_structure> if it's differenet than C<ISO-8859-2>
(and it probably is). This encoding will be converted to C<UTF-8> for
Hyper Estraier.

=back

Name of database will be used to form URI of documents in index.

=cut

sub new {
        my $class = shift;
        my $self = {@_};
        bless($self, $class);

        my $log = $self->_get_logger;

        #$log->debug("self: ", sub { Dumper($self) });

        foreach my $p (qw/masterurl user passwd database/) {
                $log->logdie("need $p") unless ($self->{$p});
        }

        $self->{encoding} ||= 'ISO-8859-2';

        my $url = $self->{masterurl} . '/node/' . $self->{database};
        $self->{url} = $url;

        $self->{db} = Search::Estraier::Node->new(
                url => $url,
                user => $self->{user},
                passwd => $self->{passwd},
                debug => $self->{debug},
                create => 1,
                label => "WebPAC $self->{database}",
        );

        $log->info("using index $self->{url} with encoding $self->{encoding}");

        if ($self->{clean}) {
                $log->debug("clean $self->{database}");
                $self->master( action => 'nodeclr', name => $self->{database} );
        } else {
                $log->debug("opening index $self->{url}");
        }

        $self ? return $self : return undef;
}


=head2 add

Adds one entry to database.

  $est->add(
        id => 42,
        ds => $ds,
        type => 'display',
        text => 'optional text from which snippet is created',
  );

This function will create  entries in index using following URI format:

  C<file:///type/database%20name/000>

Each tag in C<data_structure> with specified C<type> will create one
attribute and corresponding hidden text (used for search).

=cut

sub add {
        my $self = shift;

        my $args = {@_};

        my $log = $self->_get_logger;

        my $database = $self->{'database'} || $log->logconfess('no database in $self');
        $log->logconfess('need db in object') unless ($self->{'db'});

        foreach my $p (qw/id ds type/) {
                $log->logdie("need $p") unless ($args->{$p});
        }

        my $type = $args->{'type'};
        my $id = $args->{'id'};

        my $uri = "file:///$type/$database/$id";
        $log->debug("creating $uri");

        my $doc = Search::Estraier::Document->new;
        $doc->add_attr('@uri', $self->convert($uri) );

        $log->debug("ds = ", sub { Dumper($args->{'ds'}) } );

        # filter all tags which have type defined
        my @tags = grep {
                ref($args->{'ds'}->{$_}) eq 'HASH' && defined( $args->{'ds'}->{$_}->{$type} )
        } keys %{ $args->{'ds'} };

        $log->debug("tags = ", join(",", @tags));

        return unless (@tags);

        foreach my $tag (@tags) {

                my $vals = join(" ", @{ $args->{'ds'}->{$tag}->{$type} });

                next if (! $vals);

                $vals = $self->convert( $vals ) or
                        $log->logdie("can't convert '$vals' to UTF-8");

                $doc->add_attr( $tag, $vals );
                $doc->add_hidden_text( $vals );
        }

        my $text = $args->{'text'};
        if ( $text ) {
                $text = $self->convert( $text ) or
                        $log->logdie("can't convert '$text' to UTF-8");
                $doc->add_text( $text );
        }

        $log->debug("adding ", sub { $doc->dump_draft } );
        $self->{'db'}->put_doc($doc) || $log->warn("can't add document $uri with draft " . $doc->dump_draft . " to node " . $self->{url} . " status: " . $self->{db}->status());

        return 1;
}

=head2 add_link

  $est->add_link(
        from => 'ps',
        to => 'webpac2',
        credit => 10000,
  );

=cut

sub add_link {
        my $self = shift;

        my $args = {@_};
        my $log = $self->_get_logger;

        my @labels = $self->master( action => 'nodelist' );

        $log->debug("got labels: ", join("|", @labels));

        @labels = grep(/^$args->{to}\t/, @labels);
        my $label = shift @labels;
        (undef,$label) = split(/\t/, $label) if ($label);

        if (! $label) {
                $log->warn("can't find label for $args->{to}, skipping link creaton");
                return;
        }

        $log->debug("using label $label for $args->{to}");

        return $self->estcall(
                validate => 'node',
                action => '_set_link',
                rest_url => $self->{masterurl} . '/node/' . $args->{from} . '/_set_link' ,
                url => $self->{masterurl} . '/node/' . $args->{to},
                label => $label,
                credit => $args->{credit},
        );
}


=head2 master

Issue administrative commands to C<estmaster> process. See documentation for
C<master> in L<Search::Estraier>::Node.

  $self->master(
        action => 'nodeclr',
        name => 'foobar',
  );

=cut

sub master {
        my $self = shift;
        $self->{db}->master( @_ );
}


=head2 convert

 my $utf8_string = $self->convert('string in codepage');

=cut

sub convert {
        my $self = shift;

        my $text = shift || return;
        from_to($text, $self->{encoding}, 'UTF-8');
        return $text;
}

=head1 AUTHOR

Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>

=head1 COPYRIGHT & LICENSE

Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.

This program is free software; you can redistribute it and/or modify it
under the same terms as Perl itself.

=cut

1; # End of WebPAC::Output::Estraier
1	dpavlin	1	package WebPAC::Output::Estraier;
2
3			use warnings;
4			use strict;
5
6	dpavlin	74	use base qw/WebPAC::Common/;
7
8	dpavlin	458	use Search::Estraier 0.06;
9	dpavlin	401	use Encode qw/from_to/;
10	dpavlin	74	use Data::Dumper;
11	dpavlin	211	use LWP;
12	dpavlin	210	use URI::Escape;
13	dpavlin	74
14	dpavlin	1	=head1 NAME
15
16	dpavlin	74	WebPAC::Output::Estraier - Create Hyper Estraier full text index
17	dpavlin	1
18			=head1 VERSION
19
20	dpavlin	458	Version 0.11
21	dpavlin	1
22			=cut
23
24	dpavlin	458	our $VERSION = '0.11';
25	dpavlin	1
26			=head1 SYNOPSIS
27
28	dpavlin	74	Create full text index using Hyper Estraier index from data with
29			type C<search>.
30	dpavlin	1
31	dpavlin	74	=head1 FUNCTIONS
32	dpavlin	1
33	dpavlin	74	=head2 new
34	dpavlin	1
35	dpavlin	74	Connect to Hyper Estraier index using HTTP
36	dpavlin	1
37	dpavlin	74	my $est = new WebPAC::Output::Estraier(
38	dpavlin	210	masterurl => 'http://localhost:1978/',
39	dpavlin	74	user => 'admin',
40			passwd => 'admin',
41			database => 'demo',
42	dpavlin	401	label => 'node label',
43	dpavlin	74	encoding => 'iso-8859-2',
44	dpavlin	307	clean => 1,
45	dpavlin	74	);
46	dpavlin	1
47	dpavlin	74	Options are:
48	dpavlin	1
49	dpavlin	74	=over 4
50	dpavlin	1
51	dpavlin	210	=item masterurl
52	dpavlin	1
53	dpavlin	74	URI to C<estmaster> node
54
55			=item user
56
57			C<estmaster> user with sufficient rights
58
59			=item passwd
60
61			password for user
62
63			=item database
64
65			name of database from which data comes
66
67	dpavlin	401	=item label
68
69			label for node (optional)
70
71	dpavlin	74	=item encoding
72
73			character encoding of C<data_structure> if it's differenet than C<ISO-8859-2>
74			(and it probably is). This encoding will be converted to C<UTF-8> for
75			Hyper Estraier.
76
77			=back
78
79			Name of database will be used to form URI of documents in index.
80
81	dpavlin	1	=cut
82
83	dpavlin	74	sub new {
84			my $class = shift;
85	dpavlin	343	my $self = {@_};
86			bless($self, $class);
87	dpavlin	74
88			my $log = $self->_get_logger;
89
90	dpavlin	211	#$log->debug("self: ", sub { Dumper($self) });
91	dpavlin	210
92			foreach my $p (qw/masterurl user passwd database/) {
93	dpavlin	74	$log->logdie("need $p") unless ($self->{$p});
94			}
95
96	dpavlin	401	$self->{encoding} \|\|= 'ISO-8859-2';
97
98	dpavlin	210	my $url = $self->{masterurl} . '/node/' . $self->{database};
99			$self->{url} = $url;
100	dpavlin	74
101	dpavlin	458	$self->{db} = Search::Estraier::Node->new(
102			url => $url,
103			user => $self->{user},
104			passwd => $self->{passwd},
105			debug => $self->{debug},
106			create => 1,
107			label => "WebPAC $self->{database}",
108			);
109
110			$log->info("using index $self->{url} with encoding $self->{encoding}");
111
112	dpavlin	307	if ($self->{clean}) {
113	dpavlin	458	$log->debug("clean $self->{database}");
114			$self->master( action => 'nodeclr', name => $self->{database} );
115	dpavlin	307	} else {
116			$log->debug("opening index $self->{url}");
117			}
118	dpavlin	74
119			$self ? return $self : return undef;
120	dpavlin	1	}
121
122	dpavlin	75
123	dpavlin	74	=head2 add
124	dpavlin	1
125	dpavlin	74	Adds one entry to database.
126
127			$est->add(
128			id => 42,
129			ds => $ds,
130			type => 'display',
131			text => 'optional text from which snippet is created',
132			);
133
134			This function will create entries in index using following URI format:
135
136	dpavlin	212	C<file:///type/database%20name/000>
137	dpavlin	74
138			Each tag in C<data_structure> with specified C<type> will create one
139			attribute and corresponding hidden text (used for search).
140
141	dpavlin	1	=cut
142
143	dpavlin	74	sub add {
144			my $self = shift;
145
146	dpavlin	456	my $args = {@_};
147	dpavlin	74
148			my $log = $self->_get_logger;
149
150			my $database = $self->{'database'} \|\| $log->logconfess('no database in $self');
151			$log->logconfess('need db in object') unless ($self->{'db'});
152
153			foreach my $p (qw/id ds type/) {
154			$log->logdie("need $p") unless ($args->{$p});
155			}
156
157			my $type = $args->{'type'};
158	dpavlin	213	my $id = $args->{'id'};
159	dpavlin	74
160	dpavlin	213	my $uri = "file:///$type/$database/$id";
161	dpavlin	74	$log->debug("creating $uri");
162
163	dpavlin	343	my $doc = Search::Estraier::Document->new;
164	dpavlin	401	$doc->add_attr('@uri', $self->convert($uri) );
165	dpavlin	74
166			$log->debug("ds = ", sub { Dumper($args->{'ds'}) } );
167
168			# filter all tags which have type defined
169			my @tags = grep {
170	dpavlin	113	ref($args->{'ds'}->{$_}) eq 'HASH' && defined( $args->{'ds'}->{$_}->{$type} )
171	dpavlin	74	} keys %{ $args->{'ds'} };
172
173			$log->debug("tags = ", join(",", @tags));
174
175			return unless (@tags);
176
177			foreach my $tag (@tags) {
178
179			my $vals = join(" ", @{ $args->{'ds'}->{$tag}->{$type} });
180
181	dpavlin	374	next if (! $vals);
182	dpavlin	74
183	dpavlin	401	$vals = $self->convert( $vals ) or
184	dpavlin	85	$log->logdie("can't convert '$vals' to UTF-8");
185	dpavlin	75
186			$doc->add_attr( $tag, $vals );
187			$doc->add_hidden_text( $vals );
188	dpavlin	74	}
189
190			my $text = $args->{'text'};
191	dpavlin	75	if ( $text ) {
192	dpavlin	401	$text = $self->convert( $text ) or
193	dpavlin	85	$log->logdie("can't convert '$text' to UTF-8");
194	dpavlin	75	$doc->add_text( $text );
195			}
196	dpavlin	74
197			$log->debug("adding ", sub { $doc->dump_draft } );
198	dpavlin	319	$self->{'db'}->put_doc($doc) \|\| $log->warn("can't add document $uri with draft " . $doc->dump_draft . " to node " . $self->{url} . " status: " . $self->{db}->status());
199	dpavlin	74
200			return 1;
201	dpavlin	1	}
202
203	dpavlin	255	=head2 add_link
204
205			$est->add_link(
206			from => 'ps',
207			to => 'webpac2',
208			credit => 10000,
209			);
210
211	dpavlin	246	=cut
212
213	dpavlin	255	sub add_link {
214	dpavlin	238	my $self = shift;
215
216	dpavlin	255	my $args = {@_};
217			my $log = $self->_get_logger;
218	dpavlin	238
219	dpavlin	255	my @labels = $self->master( action => 'nodelist' );
220	dpavlin	238
221	dpavlin	255	$log->debug("got labels: ", join("\|", @labels));
222	dpavlin	238
223	dpavlin	303	@labels = grep(/^$args->{to}\t/, @labels);
224			my $label = shift @labels;
225	dpavlin	307	(undef,$label) = split(/\t/, $label) if ($label);
226	dpavlin	255
227			if (! $label) {
228			$log->warn("can't find label for $args->{to}, skipping link creaton");
229			return;
230			}
231
232			$log->debug("using label $label for $args->{to}");
233
234			return $self->estcall(
235			validate => 'node',
236			action => '_set_link',
237			rest_url => $self->{masterurl} . '/node/' . $args->{from} . '/_set_link' ,
238			url => $self->{masterurl} . '/node/' . $args->{to},
239			label => $label,
240			credit => $args->{credit},
241			);
242	dpavlin	238	}
243
244	dpavlin	255
245	dpavlin	458	=head2 master
246	dpavlin	255
247	dpavlin	458	Issue administrative commands to C<estmaster> process. See documentation for
248			C<master> in L<Search::Estraier>::Node.
249
250			$self->master(
251			action => 'nodeclr',
252			name => 'foobar',
253	dpavlin	255	);
254
255			=cut
256
257	dpavlin	458	sub master {
258	dpavlin	210	my $self = shift;
259	dpavlin	458	$self->{db}->master( @_ );
260	dpavlin	210	}
261
262	dpavlin	255
263	dpavlin	401	=head2 convert
264
265			my $utf8_string = $self->convert('string in codepage');
266
267			=cut
268
269			sub convert {
270			my $self = shift;
271
272			my $text = shift \|\| return;
273			from_to($text, $self->{encoding}, 'UTF-8');
274			return $text;
275			}
276
277	dpavlin	1	=head1 AUTHOR
278
279			Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
280
281			=head1 COPYRIGHT & LICENSE
282
283			Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.
284
285			This program is free software; you can redistribute it and/or modify it
286			under the same terms as Perl itself.
287
288			=cut
289
290			1; # End of WebPAC::Output::Estraier