WebPAC/Output/KinoSearch.pm

package WebPAC::Output::KinoSearch;

use warnings;
use strict;

use base qw/WebPAC::Common WebPAC::Output Class::Accessor/;
__PACKAGE__->mk_accessors(qw(
        path
        database
        input
        encoding
        clean

        index
));

use KinoSearch::Simple;
use File::Path;
use Encode qw/decode/;
use Data::Dump qw/dump/;
use Storable;

=head1 NAME

WebPAC::Output::KinoSearch - Create KinoSearch full text index

=head1 VERSION

Version 0.05

=cut

our $VERSION = '0.05';

=head1 SYNOPSIS

Create full text index using KinoSearch index from data with
type C<search>.

=head1 FUNCTIONS

=head2 new

Open KinoSearch index

 my $out = new WebPAC::Output::KinoSearch({
        path => '/path/to/invindex',
        database => 'demo',
        encoding => 'iso-8859-2',
        clean => 1,
 });

Options are:

=over 4

=item path

path to KinoSearch index to use

=item database

name of database from which data comes

=item encoding

character encoding of C<data_structure> if it's differenet than C<ISO-8859-2>
(and it probably is). This encoding will be converted to C<UTF-8> for
index.

=back

=head2 init

  $out->init;

=cut

sub init {
        my $self = shift;

        my $log = $self->_get_logger;

        #$log->debug("self: ", sub { dump($self) });

        foreach my $p (qw/path database/) {
                $log->logdie("need $p") unless ($self->$p);
        }

#       $log->logdie("fields is not ARRAY") unless (ref($self->{fields}) eq 'ARRAY');

        $self->encoding( 'ISO-8859-2' ) unless $self->encoding;

        ## FIXME we shouldn't re-create whole KinoSearch index every time!
        $self->clean( 1 );

        if ( ! -e $self->path ) {
                mkpath $self->path || $log->logdie("can't create ", $self->path,": $!");
                $log->info("created ", $self->path);
        } elsif ( $self->clean ) {
                $log->info("removing existing ", $self->path);
                rmtree $self->path || $log->logdie("can't remove ", $self->path,": $!");
                mkpath $self->path || $log->logdie("can't create ", $self->path,": $!");
        }

        my $path = $self->path . '/' . $self->database;

        $log->info("using index $path with encoding ", $self->encoding);

        my $index = KinoSearch::Simple->new(
                path => $path,
                language => 'en',
        );

        $log->logdie("can't open $path: $!") unless $index;

        $self->index( $index );

}


=head2 add

Adds one entry

  $out->add( 42, $ds );

=cut

sub add {
        my $self = shift;

        my ( $id, $ds ) = @_;

        my $log = $self->_get_logger;
        $log->logdie("need id") unless defined $id;
        $log->logdie("need ds") unless $ds;

        my $hash = $self->ds_to_hash( $ds, 'search' ) || return;

        $hash->{id}       ||= $id;
        $hash->{database} ||= $self->database;
        $hash->{input}    ||= $self->input;

        foreach my $f ( keys %$hash ) {
                if ( ref($hash->{$f}) eq 'ARRAY' ) {
                        $hash->{$f} = join(' <*> ', @{ $hash->{$f} });
                }
#               $hash->{$f} = decode( $self->encoding, $hash->{$f} );
                $self->{field_count}->{$f}++;
        }

        $log->debug("add( $id, ", sub { dump($ds) }," ) => ", sub { dump( $hash ) });

        $self->index->add_doc( $hash );

        $self->{count}++;

        return 1;
}

=head2 finish

Close index

 $out->finish;

=cut

sub finish {
        my $self = shift;

        my $log = $self->_get_logger();

        $log->info("indexed ", $self->{count}, " records");

        $log->debug("field usage: ", dump( $self->{field_count} ));

}

=head1 AUTHOR

Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>

=head1 COPYRIGHT & LICENSE

Copyright 2005-2007 Dobrica Pavlinusic, All Rights Reserved.

This program is free software; you can redistribute it and/or modify it
under the same terms as Perl itself.

=cut

1; # End of WebPAC::Output::Estraier
1	dpavlin	431	package WebPAC::Output::KinoSearch;
2
3			use warnings;
4			use strict;
5
6	dpavlin	914	use base qw/WebPAC::Common WebPAC::Output Class::Accessor/;
7			__PACKAGE__->mk_accessors(qw(
8			path
9			database
10	dpavlin	939	input
11	dpavlin	914	encoding
12			clean
13	dpavlin	431
14	dpavlin	914	index
15			));
16
17			use KinoSearch::Simple;
18			use File::Path;
19	dpavlin	949	use Encode qw/decode/;
20	dpavlin	887	use Data::Dump qw/dump/;
21	dpavlin	536	use Storable;
22	dpavlin	431
23			=head1 NAME
24
25			WebPAC::Output::KinoSearch - Create KinoSearch full text index
26
27			=head1 VERSION
28
29	dpavlin	919	Version 0.05
30	dpavlin	431
31			=cut
32
33	dpavlin	919	our $VERSION = '0.05';
34	dpavlin	431
35			=head1 SYNOPSIS
36
37			Create full text index using KinoSearch index from data with
38			type C<search>.
39
40			=head1 FUNCTIONS
41
42			=head2 new
43
44			Open KinoSearch index
45
46	dpavlin	917	my $out = new WebPAC::Output::KinoSearch({
47	dpavlin	914	path => '/path/to/invindex',
48	dpavlin	431	database => 'demo',
49			encoding => 'iso-8859-2',
50			clean => 1,
51	dpavlin	914	});
52	dpavlin	431
53			Options are:
54
55			=over 4
56
57	dpavlin	914	=item path
58	dpavlin	431
59			path to KinoSearch index to use
60
61			=item database
62
63			name of database from which data comes
64
65			=item encoding
66
67			character encoding of C<data_structure> if it's differenet than C<ISO-8859-2>
68			(and it probably is). This encoding will be converted to C<UTF-8> for
69			index.
70
71			=back
72
73	dpavlin	917	=head2 init
74
75			$out->init;
76
77	dpavlin	431	=cut
78
79	dpavlin	914	sub init {
80			my $self = shift;
81	dpavlin	431
82			my $log = $self->_get_logger;
83
84	dpavlin	887	#$log->debug("self: ", sub { dump($self) });
85	dpavlin	431
86	dpavlin	914	foreach my $p (qw/path database/) {
87			$log->logdie("need $p") unless ($self->$p);
88	dpavlin	431	}
89
90	dpavlin	914	# $log->logdie("fields is not ARRAY") unless (ref($self->{fields}) eq 'ARRAY');
91	dpavlin	431
92	dpavlin	914	$self->encoding( 'ISO-8859-2' ) unless $self->encoding;
93	dpavlin	431
94	dpavlin	939	## FIXME we shouldn't re-create whole KinoSearch index every time!
95			$self->clean( 1 );
96
97	dpavlin	914	if ( ! -e $self->path ) {
98			mkpath $self->path \|\| $log->logdie("can't create ", $self->path,": $!");
99			$log->info("created ", $self->path);
100	dpavlin	919	} elsif ( $self->clean ) {
101			$log->info("removing existing ", $self->path);
102			rmtree $self->path \|\| $log->logdie("can't remove ", $self->path,": $!");
103			mkpath $self->path \|\| $log->logdie("can't create ", $self->path,": $!");
104	dpavlin	914	}
105	dpavlin	610
106	dpavlin	914	my $path = $self->path . '/' . $self->database;
107	dpavlin	431
108	dpavlin	914	$log->info("using index $path with encoding ", $self->encoding);
109	dpavlin	609
110	dpavlin	914	my $index = KinoSearch::Simple->new(
111			path => $path,
112			language => 'en',
113	dpavlin	431	);
114
115	dpavlin	914	$log->logdie("can't open $path: $!") unless $index;
116	dpavlin	536
117	dpavlin	914	$self->index( $index );
118	dpavlin	431
119			}
120
121
122			=head2 add
123
124	dpavlin	914	Adds one entry
125	dpavlin	431
126	dpavlin	917	$out->add( 42, $ds );
127	dpavlin	431
128			=cut
129
130			sub add {
131			my $self = shift;
132
133	dpavlin	914	my ( $id, $ds ) = @_;
134	dpavlin	431
135			my $log = $self->_get_logger;
136	dpavlin	914	$log->logdie("need id") unless defined $id;
137			$log->logdie("need ds") unless $ds;
138	dpavlin	431
139	dpavlin	914	my $hash = $self->ds_to_hash( $ds, 'search' ) \|\| return;
140	dpavlin	431
141	dpavlin	939	$hash->{id} \|\|= $id;
142	dpavlin	919	$hash->{database} \|\|= $self->database;
143	dpavlin	939	$hash->{input} \|\|= $self->input;
144	dpavlin	431
145	dpavlin	924	foreach my $f ( keys %$hash ) {
146			if ( ref($hash->{$f}) eq 'ARRAY' ) {
147			$hash->{$f} = join(' <*> ', @{ $hash->{$f} });
148			}
149	dpavlin	949	# $hash->{$f} = decode( $self->encoding, $hash->{$f} );
150	dpavlin	987	$self->{field_count}->{$f}++;
151	dpavlin	924	}
152
153	dpavlin	919	$log->debug("add( $id, ", sub { dump($ds) }," ) => ", sub { dump( $hash ) });
154
155	dpavlin	914	$self->index->add_doc( $hash );
156	dpavlin	431
157	dpavlin	922	$self->{count}++;
158
159	dpavlin	431	return 1;
160			}
161
162	dpavlin	434	=head2 finish
163	dpavlin	431
164	dpavlin	434	Close index
165
166	dpavlin	917	$out->finish;
167	dpavlin	434
168			=cut
169
170			sub finish {
171			my $self = shift;
172
173	dpavlin	536	my $log = $self->_get_logger();
174
175	dpavlin	922	$log->info("indexed ", $self->{count}, " records");
176	dpavlin	536
177	dpavlin	987	$log->debug("field usage: ", dump( $self->{field_count} ));
178
179	dpavlin	434	}
180
181	dpavlin	431	=head1 AUTHOR
182
183			Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
184
185			=head1 COPYRIGHT & LICENSE
186
187	dpavlin	914	Copyright 2005-2007 Dobrica Pavlinusic, All Rights Reserved.
188	dpavlin	431
189			This program is free software; you can redistribute it and/or modify it
190			under the same terms as Perl itself.
191
192			=cut
193
194			1; # End of WebPAC::Output::Estraier