/[Grep]/lib/Grep/Search/KinoSearch.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /lib/Grep/Search/KinoSearch.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 185 - (hide annotations)
Wed Apr 9 17:39:40 2008 UTC (15 years, 10 months ago) by dpavlin
Original Path: lib/Grep/Search.pm
File size: 6351 byte(s)
don't open invindexer if we are only searching
1 dpavlin 47 package Grep::Search;
2    
3     use strict;
4     use warnings;
5 dpavlin 112 use base qw( Class::Accessor );
6 dpavlin 185 Grep::Search->mk_accessors( qw( create index_path hits ) );
7 dpavlin 47
8     use Data::Dump qw/dump/;
9 dpavlin 127 use KinoSearch::InvIndexer;
10     use KinoSearch::Searcher;
11 dpavlin 47 use Jifty::Util;
12    
13 dpavlin 185 my $debug = 1;
14 dpavlin 47
15 dpavlin 109 =head1 NAME
16 dpavlin 47
17 dpavlin 127 Grep::Search - full text search using L<KinoSearch>
18 dpavlin 47
19 dpavlin 109 =head1 METHODS
20 dpavlin 47
21 dpavlin 109 =head2 new
22 dpavlin 58
23 dpavlin 109 my $search = Grep::Search->new();
24    
25 dpavlin 145 my $search = Grep::Search->new( create => 1 );
26 dpavlin 144
27 dpavlin 109 =cut
28    
29 dpavlin 112 sub log { Jifty->web->log }
30    
31 dpavlin 109 sub new {
32     my $class = shift;
33     my $self = $class->SUPER::new(@_);
34    
35 dpavlin 127 my $index_path = Jifty::Util->app_root . '/var/invindex';
36 dpavlin 109
37     $self->index_path( $index_path );
38    
39 dpavlin 185 return $self;
40     }
41    
42     =head2 invindexer
43    
44     Accessor to call any method defined on L<KinoSearch::InvIndexer>
45    
46     $search->invindexer->delete_by_term( 'id', 42 );
47    
48     =cut
49    
50     my $indexes;
51    
52     sub invindexer {
53     my $self = shift;
54     my $invindexer;
55     my $index_path = $self->index_path or die "no index_path?";
56    
57     if ( $invindexer = $indexes->{$index_path} ) {
58     $self->log->debug("Using cached index $index_path");
59 dpavlin 47 } else {
60 dpavlin 185 if ( ! -e "$index_path" || $self->create ) {
61     $self->log->debug("Creating new index $index_path");
62     my $invindexer = KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->clobber( $index_path ) );
63     } else {
64     $self->log->debug("Opening index: $index_path");
65     $invindexer = KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->open( $index_path ) );
66     }
67     $indexes->{$index_path} = $invindexer;
68 dpavlin 47 }
69    
70 dpavlin 185 $self->invindexer( $invindexer );
71    
72 dpavlin 49 }
73    
74 dpavlin 47 =head2 add
75    
76 dpavlin 109 $search->add( $record, $owner_id );
77 dpavlin 47
78     =cut
79    
80     sub add {
81     my $self = shift;
82    
83     my $i = shift or die "no record to add";
84 dpavlin 64 my $uid = shift;
85 dpavlin 47
86     die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
87    
88     my $pk = { $i->primary_keys };
89    
90 dpavlin 127 my $doc;
91 dpavlin 47
92     my @columns = map { $_->name } $i->columns;
93    
94     foreach my $c ( @columns ) {
95    
96     my $v = $i->$c;
97    
98     if ( ref($v) ne '' ) {
99 dpavlin 53
100     foreach my $f_c ( qw/id name title/ ) {
101     if ( $i->$c->can( $f_c ) ) {
102     my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
103     my $col = $c . '_' . $f_c;
104     if ( $f_v ) {
105     warn " # $col = $f_v\n" if ($debug);
106 dpavlin 127 $doc->{ $col } = $f_v;
107 dpavlin 53 } else {
108     warn " . $col is NULL\n" if ($debug);
109     }
110     }
111     }
112    
113     if ($v->isa('Jifty::DateTime')) {
114 dpavlin 47 warn " d $c = $v\n" if ($debug);
115 dpavlin 127 $doc->{$c} = $v;
116 dpavlin 47 } else {
117     warn " s $c = $v [",ref($v),"]\n" if ($debug);
118     }
119     next;
120     }
121    
122     next if (! defined($v) || $v eq '');
123    
124 dpavlin 160 eval { $v =~ s/<[^>]+>/ /gs; };
125     if ($@) {
126     Jifty->log->error("can't strip html from $c in item ", $i->id);
127     next;
128     }
129 dpavlin 47
130     if ( defined( $pk->{$c} ) ) {
131 dpavlin 127 $doc->{ $c } = $v;
132 dpavlin 47 warn " * $c = $v\n" if ($debug);
133     } else {
134 dpavlin 127 $doc->{ $c } = $v;
135 dpavlin 53 warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
136 dpavlin 47 }
137     }
138    
139 dpavlin 64 # add _owner_id to speed up filtering of search results
140     $uid ||= Jifty->web->current_user->id;
141 dpavlin 127 $doc->{ '_owner_id' } = $uid;
142 dpavlin 64
143 dpavlin 127 $self->invindexer->add_doc( $doc );
144 dpavlin 109
145 dpavlin 98 $self->log->debug("added ", $i->id, " for user $uid to index");
146 dpavlin 151
147     return 1;
148 dpavlin 47 }
149    
150 dpavlin 109 =head2 collection
151 dpavlin 47
152 dpavlin 141 Return C<Grep::Model::ItemCollection> which is result of C<search query>
153    
154 dpavlin 109 my $ItemCollection = $search->collection( 'search query' );
155 dpavlin 47
156 dpavlin 141 =head2 hits
157    
158     Return number of results from last C<collection> call
159    
160     my $num_results = $search->hits;
161    
162 dpavlin 47 =cut
163    
164     sub collection {
165     my $self = shift;
166    
167     my $q = shift or die "no q?";
168    
169 dpavlin 127 my $searcher = KinoSearch::Searcher->new(
170     invindex => Grep::Search::Schema->open( $self->index_path ), );
171 dpavlin 98 $self->log->debug("$searcher created");
172 dpavlin 47
173 dpavlin 151 my $full_q = "($q)";
174 dpavlin 64
175 dpavlin 151 my $uid = Jifty->web->current_user->id;
176 dpavlin 153 $full_q .= ' AND _owner_id:' . $uid if (defined $uid);
177 dpavlin 151
178 dpavlin 127 $self->log->debug("searching for '$q' using $full_q");
179 dpavlin 153
180     my $query_parser = KinoSearch::QueryParser->new(
181     schema => Grep::Search::Schema->new,
182     fields => [ qw/ title link content summary category author / ],
183     );
184     $query_parser->set_heed_colons(1); # enable field:value AND/OR/NOT syntax
185     my $query = $query_parser->parse( $full_q );
186 dpavlin 127 my $hits = $searcher->search(
187 dpavlin 153 query => $query,
188 dpavlin 127 # offset => $offset,
189     # num_wanted => $hits_per_page,
190     );
191 dpavlin 64
192 dpavlin 141 $self->hits( $hits->total_hits );
193 dpavlin 47
194 dpavlin 141 $self->log->debug("found ", $self->hits, " results");
195 dpavlin 47
196     my $collection = Grep::Model::ItemCollection->new();
197    
198     my @results;
199    
200 dpavlin 127 my $i = 0;
201 dpavlin 180 while ( my $hit = $hits->fetch_hit ) {
202 dpavlin 47
203 dpavlin 127 my $score = $hit->{score};
204     my $title = $hit->{title};
205     my $id = $hit->{id};
206 dpavlin 47
207 dpavlin 144 $self->log->debug("result $i [$id] $title $score");
208 dpavlin 47
209     my $item = Grep::Model::Item->new();
210     my ($ok,$msg) = $item->load_by_cols( id => $id );
211    
212     if ( $ok ) {
213     $collection->add_record( $item );
214     } else {
215     warn "can't load item $id\n";
216     }
217    
218     }
219    
220 dpavlin 127 $self->log->debug("finished search");
221 dpavlin 47
222     return $collection;
223     }
224    
225     =head2 finish
226    
227 dpavlin 109 $search->finish
228 dpavlin 47
229     =cut
230    
231     sub finish {
232     my $self = shift;
233 dpavlin 127 if ($self->invindexer) {
234 dpavlin 102 $self->log->debug("closing index");
235 dpavlin 127 $self->invindexer->finish;
236 dpavlin 47 }
237 dpavlin 58
238 dpavlin 110 $self->log->debug("finish");
239    
240 dpavlin 112 undef $self;
241    
242 dpavlin 151 return 1;
243 dpavlin 47 }
244    
245 dpavlin 53 =head2 snippet
246    
247     my $short = $self->snippet( 50, $text );
248    
249     =cut
250    
251     sub snippet {
252     my $self = shift;
253    
254     my $len = shift or die "no len?";
255     my $m = join(" ", @_);
256    
257     $m =~ s/\s+/ /gs;
258    
259     if (length($m) > $len) {
260     return substr($m,0,$len) . '...';
261     } else {
262     return $m;
263     }
264     }
265    
266 dpavlin 127 package Grep::Search::KeywordField;
267     use base qw( KinoSearch::Schema::FieldSpec );
268     sub analyzed { 0 }
269 dpavlin 153 #sub indexed { 1 }
270     #sub stored { 1 }
271     sub vectorized { 0 }
272 dpavlin 127
273     package Grep::Search::Schema;
274    
275     =head1 NAME
276    
277 dpavlin 136 Grep::Search::Schema - schema definition for full-text search
278 dpavlin 127
279     =cut
280    
281     use base 'KinoSearch::Schema';
282     use KinoSearch::Analysis::PolyAnalyzer;
283    
284 dpavlin 131 our %fields = (
285 dpavlin 127 id => 'Grep::Search::KeywordField',
286    
287     in_feed_id => 'Grep::Search::KeywordField',
288     in_feed_url => 'Grep::Search::KeywordField',
289     in_feed_title => 'KinoSearch::Schema::FieldSpec',
290     in_feed_owner => 'Grep::Search::KeywordField',
291     in_feed_created_on => 'Grep::Search::KeywordField',
292    
293     title => 'KinoSearch::Schema::FieldSpec',
294     link => 'Grep::Search::KeywordField',
295     content => 'KinoSearch::Schema::FieldSpec',
296     summary => 'KinoSearch::Schema::FieldSpec',
297     category => 'KinoSearch::Schema::FieldSpec',
298     author => 'KinoSearch::Schema::FieldSpec',
299 dpavlin 169 created_on => 'Grep::Search::KeywordField',
300     last_update => 'Grep::Search::KeywordField',
301 dpavlin 127
302     _owner_id => 'Grep::Search::KeywordField',
303     );
304    
305     sub analyzer {
306     return KinoSearch::Analysis::PolyAnalyzer->new( language => 'en' );
307     }
308    
309 dpavlin 47 1;

  ViewVC Help
Powered by ViewVC 1.1.26