Grep/Search/KinoSearch.pm

package Grep::Search;

use strict;
use warnings;
use base qw( Class::Accessor );
Grep::Search->mk_accessors( qw( invindexer create index_path ) );

use Data::Dump qw/dump/;
use KinoSearch::InvIndexer;
use KinoSearch::Searcher;
use Jifty::Util;

my $debug = 0;

=head1 NAME

Grep::Search - full text search using L<KinoSearch>

=head1 METHODS

=head2 new

  my $search = Grep::Search->new();

=cut

sub log { Jifty->web->log }

sub new {
        my $class = shift;        
        my $self = $class->SUPER::new(@_);

        my $index_path = Jifty::Util->app_root . '/var/invindex';

        $self->index_path( $index_path );

        if (! -e "$index_path") {
                $self->log->debug("Creating new index $index_path");
                $self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->clobber( $index_path ) ) );
        } else {
                $self->log->debug("Opening index: $index_path");
                $self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->open( $index_path ) ) );
        }

        return $self;
}

=head2 add

  $search->add( $record, $owner_id );

=cut

sub add {
        my $self = shift;

        my $i = shift or die "no record to add";
        my $uid = shift;

        die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));

        my $pk = { $i->primary_keys };

        my $doc;

        my @columns = map { $_->name } $i->columns;

        foreach my $c ( @columns ) {

                my $v = $i->$c;

                if ( ref($v) ne '' ) {

                        foreach my $f_c ( qw/id name title/ ) {
                                if ( $i->$c->can( $f_c ) ) {
                                        my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
                                        my $col = $c . '_' . $f_c;
                                        if ( $f_v ) {
                                                warn "  # $col = $f_v\n" if ($debug);
                                                $doc->{ $col } = $f_v;
                                        } else {
                                                warn "  . $col is NULL\n" if ($debug);
                                        }
                                }
                        }

                        if ($v->isa('Jifty::DateTime')) {
                                warn "  d $c = $v\n" if ($debug);
                                $doc->{$c} = $v;
                        } else {
                                warn "  s $c = $v [",ref($v),"]\n" if ($debug);
                        }
                        next;
                }

                next if (! defined($v) || $v eq '');

                $v =~ s/<[^>]+>/ /gs;

                if ( defined( $pk->{$c} ) ) {
                        $doc->{ $c } = $v;
                        warn "  * $c = $v\n" if ($debug);
                } else {
                        $doc->{ $c } = $v;
                        warn "  + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
                }
        }

        # add _owner_id to speed up filtering of search results
        $uid ||= Jifty->web->current_user->id;
        $doc->{ '_owner_id' } = $uid;

        $self->invindexer->add_doc( $doc );

        $self->log->debug("added ", $i->id, " for user $uid to index");
}

=head2 collection

  my $ItemCollection = $search->collection( 'search query' );

=cut

sub collection {
        my $self = shift;

        my $q = shift or die "no q?";

        my $searcher = KinoSearch::Searcher->new(
                invindex => Grep::Search::Schema->open( $self->index_path ), );
        $self->log->debug("$searcher created");

        my $full_q = "($q) AND _owner_id:" . Jifty->web->current_user->id;

        $self->log->debug("searching for '$q' using $full_q");
        my $hits = $searcher->search(
                query           => $full_q,
#               offset          => $offset,
#               num_wanted      => $hits_per_page,
        );

        my $num_hits = $hits->total_hits;

        $self->log->debug("found $num_hits results");

        my $collection = Grep::Model::ItemCollection->new();

        my @results;

        my $i = 0;
        while ( my $hit = $hits->fetch_hit_hashref ) {

                my $score = $hit->{score};
                my $title = $hit->{title};
                my $id = $hit->{id};

                $self->log->debug("result $i $score $title");

                my $item = Grep::Model::Item->new();
                my ($ok,$msg) = $item->load_by_cols( id => $id );

                if ( $ok ) {
                        $collection->add_record( $item );
                } else {
                        warn "can't load item $id\n";
                }

        }

        $self->log->debug("finished search");

        return $collection;
}

=head2 finish

  $search->finish

=cut

sub finish {
        my $self = shift;
        if ($self->invindexer) {
                $self->log->debug("closing index");
                $self->invindexer->finish;
        }

        $self->log->debug("finish");

        undef $self;

        return;
}

=cut

=head2 snippet

  my $short = $self->snippet( 50, $text );

=cut

sub snippet {
        my $self = shift;

        my $len = shift or die "no len?";
        my $m = join(" ", @_);

        $m =~ s/\s+/ /gs;

        if (length($m) > $len) {
                return substr($m,0,$len) . '...';
        } else {
                return $m;
        }
}

package Grep::Search::KeywordField;
use base qw( KinoSearch::Schema::FieldSpec );
sub analyzed { 0 }

package Grep::Search::Schema;

=head1 NAME

Grep::Search::Schema - data definition

=cut

use base 'KinoSearch::Schema';
use KinoSearch::Analysis::PolyAnalyzer;

our %FIELDS = (
        id                              => 'Grep::Search::KeywordField',

        in_feed_id              => 'Grep::Search::KeywordField',
        in_feed_url             => 'Grep::Search::KeywordField',
        in_feed_title   => 'KinoSearch::Schema::FieldSpec',
        in_feed_owner   => 'Grep::Search::KeywordField',
        in_feed_created_on      => 'Grep::Search::KeywordField',

        title                   => 'KinoSearch::Schema::FieldSpec',
        link                    => 'Grep::Search::KeywordField',
        content                 => 'KinoSearch::Schema::FieldSpec',
        summary                 => 'KinoSearch::Schema::FieldSpec',
        category                => 'KinoSearch::Schema::FieldSpec',
        author                  => 'KinoSearch::Schema::FieldSpec',
        issued                  => 'Grep::Search::KeywordField',
        modified                => 'Grep::Search::KeywordField',

        _owner_id               => 'Grep::Search::KeywordField',
);

sub analyzer {
        return KinoSearch::Analysis::PolyAnalyzer->new( language => 'en' );
}

1;
1	dpavlin	47	package Grep::Search;
2
3			use strict;
4			use warnings;
5	dpavlin	112	use base qw( Class::Accessor );
6	dpavlin	127	Grep::Search->mk_accessors( qw( invindexer create index_path ) );
7	dpavlin	47
8			use Data::Dump qw/dump/;
9	dpavlin	127	use KinoSearch::InvIndexer;
10			use KinoSearch::Searcher;
11	dpavlin	47	use Jifty::Util;
12
13	dpavlin	109	my $debug = 0;
14	dpavlin	47
15	dpavlin	109	=head1 NAME
16	dpavlin	47
17	dpavlin	127	Grep::Search - full text search using L<KinoSearch>
18	dpavlin	47
19	dpavlin	109	=head1 METHODS
20	dpavlin	47
21	dpavlin	109	=head2 new
22	dpavlin	58
23	dpavlin	109	my $search = Grep::Search->new();
24
25			=cut
26
27	dpavlin	112	sub log { Jifty->web->log }
28
29	dpavlin	109	sub new {
30			my $class = shift;
31			my $self = $class->SUPER::new(@_);
32
33	dpavlin	127	my $index_path = Jifty::Util->app_root . '/var/invindex';
34	dpavlin	109
35			$self->index_path( $index_path );
36
37	dpavlin	127	if (! -e "$index_path") {
38	dpavlin	109	$self->log->debug("Creating new index $index_path");
39	dpavlin	127	$self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->clobber( $index_path ) ) );
40	dpavlin	47	} else {
41	dpavlin	109	$self->log->debug("Opening index: $index_path");
42	dpavlin	127	$self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->open( $index_path ) ) );
43	dpavlin	47	}
44
45	dpavlin	109	return $self;
46	dpavlin	49	}
47
48	dpavlin	47	=head2 add
49
50	dpavlin	109	$search->add( $record, $owner_id );
51	dpavlin	47
52			=cut
53
54			sub add {
55			my $self = shift;
56
57			my $i = shift or die "no record to add";
58	dpavlin	64	my $uid = shift;
59	dpavlin	47
60			die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
61
62			my $pk = { $i->primary_keys };
63
64	dpavlin	127	my $doc;
65	dpavlin	47
66			my @columns = map { $_->name } $i->columns;
67
68			foreach my $c ( @columns ) {
69
70			my $v = $i->$c;
71
72			if ( ref($v) ne '' ) {
73	dpavlin	53
74			foreach my $f_c ( qw/id name title/ ) {
75			if ( $i->$c->can( $f_c ) ) {
76			my $f_v = $i->$c->$f_c \|\| $i->$c->{values}->{ $f_c };
77			my $col = $c . '_' . $f_c;
78			if ( $f_v ) {
79			warn " # $col = $f_v\n" if ($debug);
80	dpavlin	127	$doc->{ $col } = $f_v;
81	dpavlin	53	} else {
82			warn " . $col is NULL\n" if ($debug);
83			}
84			}
85			}
86
87			if ($v->isa('Jifty::DateTime')) {
88	dpavlin	47	warn " d $c = $v\n" if ($debug);
89	dpavlin	127	$doc->{$c} = $v;
90	dpavlin	47	} else {
91			warn " s $c = $v [",ref($v),"]\n" if ($debug);
92			}
93			next;
94			}
95
96			next if (! defined($v) \|\| $v eq '');
97
98			$v =~ s/<[^>]+>/ /gs;
99
100			if ( defined( $pk->{$c} ) ) {
101	dpavlin	127	$doc->{ $c } = $v;
102	dpavlin	47	warn " * $c = $v\n" if ($debug);
103			} else {
104	dpavlin	127	$doc->{ $c } = $v;
105	dpavlin	53	warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
106	dpavlin	47	}
107			}
108
109	dpavlin	64	# add _owner_id to speed up filtering of search results
110			$uid \|\|= Jifty->web->current_user->id;
111	dpavlin	127	$doc->{ '_owner_id' } = $uid;
112	dpavlin	64
113	dpavlin	127	$self->invindexer->add_doc( $doc );
114	dpavlin	109
115	dpavlin	98	$self->log->debug("added ", $i->id, " for user $uid to index");
116	dpavlin	47	}
117
118	dpavlin	109	=head2 collection
119	dpavlin	47
120	dpavlin	109	my $ItemCollection = $search->collection( 'search query' );
121	dpavlin	47
122			=cut
123
124			sub collection {
125			my $self = shift;
126
127			my $q = shift or die "no q?";
128
129	dpavlin	127	my $searcher = KinoSearch::Searcher->new(
130			invindex => Grep::Search::Schema->open( $self->index_path ), );
131	dpavlin	98	$self->log->debug("$searcher created");
132	dpavlin	47
133	dpavlin	64	my $full_q = "($q) AND _owner_id:" . Jifty->web->current_user->id;
134
135	dpavlin	127	$self->log->debug("searching for '$q' using $full_q");
136			my $hits = $searcher->search(
137			query => $full_q,
138			# offset => $offset,
139			# num_wanted => $hits_per_page,
140			);
141	dpavlin	64
142	dpavlin	127	my $num_hits = $hits->total_hits;
143	dpavlin	47
144	dpavlin	98	$self->log->debug("found $num_hits results");
145	dpavlin	47
146			my $collection = Grep::Model::ItemCollection->new();
147
148			my @results;
149
150	dpavlin	127	my $i = 0;
151			while ( my $hit = $hits->fetch_hit_hashref ) {
152	dpavlin	47
153	dpavlin	127	my $score = $hit->{score};
154			my $title = $hit->{title};
155			my $id = $hit->{id};
156	dpavlin	47
157	dpavlin	102	$self->log->debug("result $i $score $title");
158	dpavlin	47
159			my $item = Grep::Model::Item->new();
160			my ($ok,$msg) = $item->load_by_cols( id => $id );
161
162			if ( $ok ) {
163			$collection->add_record( $item );
164			} else {
165			warn "can't load item $id\n";
166			}
167
168			}
169
170	dpavlin	127	$self->log->debug("finished search");
171	dpavlin	47
172			return $collection;
173			}
174
175			=head2 finish
176
177	dpavlin	109	$search->finish
178	dpavlin	47
179			=cut
180
181			sub finish {
182			my $self = shift;
183	dpavlin	127	if ($self->invindexer) {
184	dpavlin	102	$self->log->debug("closing index");
185	dpavlin	127	$self->invindexer->finish;
186	dpavlin	47	}
187	dpavlin	58
188	dpavlin	110	$self->log->debug("finish");
189
190	dpavlin	112	undef $self;
191
192	dpavlin	58	return;
193	dpavlin	47	}
194
195	dpavlin	57	=cut
196	dpavlin	53
197			=head2 snippet
198
199			my $short = $self->snippet( 50, $text );
200
201			=cut
202
203			sub snippet {
204			my $self = shift;
205
206			my $len = shift or die "no len?";
207			my $m = join(" ", @_);
208
209			$m =~ s/\s+/ /gs;
210
211			if (length($m) > $len) {
212			return substr($m,0,$len) . '...';
213			} else {
214			return $m;
215			}
216			}
217
218	dpavlin	127	package Grep::Search::KeywordField;
219			use base qw( KinoSearch::Schema::FieldSpec );
220			sub analyzed { 0 }
221
222			package Grep::Search::Schema;
223
224			=head1 NAME
225
226			Grep::Search::Schema - data definition
227
228			=cut
229
230			use base 'KinoSearch::Schema';
231			use KinoSearch::Analysis::PolyAnalyzer;
232
233			our %FIELDS = (
234			id => 'Grep::Search::KeywordField',
235
236			in_feed_id => 'Grep::Search::KeywordField',
237			in_feed_url => 'Grep::Search::KeywordField',
238			in_feed_title => 'KinoSearch::Schema::FieldSpec',
239			in_feed_owner => 'Grep::Search::KeywordField',
240			in_feed_created_on => 'Grep::Search::KeywordField',
241
242			title => 'KinoSearch::Schema::FieldSpec',
243			link => 'Grep::Search::KeywordField',
244			content => 'KinoSearch::Schema::FieldSpec',
245			summary => 'KinoSearch::Schema::FieldSpec',
246			category => 'KinoSearch::Schema::FieldSpec',
247			author => 'KinoSearch::Schema::FieldSpec',
248			issued => 'Grep::Search::KeywordField',
249			modified => 'Grep::Search::KeywordField',
250
251			_owner_id => 'Grep::Search::KeywordField',
252			);
253
254			sub analyzer {
255			return KinoSearch::Analysis::PolyAnalyzer->new( language => 'en' );
256			}
257
258	dpavlin	47	1;