/[Grep]/lib/Grep/Search/KinoSearch.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /lib/Grep/Search/KinoSearch.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 189 - (show annotations)
Fri May 23 18:28:19 2008 UTC (15 years, 9 months ago) by dpavlin
File size: 6699 byte(s)
move KinoSearch supprot in own package
1 package Grep::Search::KinoSearch;
2
3 use strict;
4 use warnings;
5 use base qw( Class::Accessor );
6 Grep::Search::KinoSearch->mk_accessors( qw( create index_path hits ) );
7
8 use Data::Dump qw/dump/;
9 use KinoSearch::InvIndexer;
10 use KinoSearch::Searcher;
11 use Jifty::Util;
12
13 my $debug = 0;
14
15 =head1 NAME
16
17 Grep::Search::KinoSearch - full text search using L<KinoSearch>
18
19 =head1 METHODS
20
21 =head2 new
22
23 my $search = Grep::Search::KinoSearch->new();
24
25 my $search = Grep::Search::KinoSearch->new( create => 1 );
26
27 =cut
28
29 sub log { Jifty->web->log }
30
31 sub new {
32 my $class = shift;
33 my $self = $class->SUPER::new(@_);
34
35 my $index_path = Jifty::Util->app_root . '/var/invindex';
36
37 $self->index_path( $index_path );
38
39 return $self;
40 }
41
42 =head2 invindexer
43
44 Accessor to call any method defined on L<KinoSearch::InvIndexer>
45
46 $search->invindexer->delete_by_term( 'id', 42 );
47
48 =cut
49
50 our $indexes;
51
52 sub invindexer {
53 my $self = shift;
54 my $invindexer;
55 my $index_path = $self->index_path or die "no index_path?";
56
57 if ( $invindexer = $indexes->{$index_path} ) {
58 $self->log->debug("Using cached index $index_path");
59 } else {
60 if ( $self->create || -e $index_path ) {
61 $self->log->debug("Creating new index $index_path");
62 $invindexer = KinoSearch::InvIndexer->new( invindex => Grep::Search::KinoSearch::Schema->clobber( $index_path ) )
63 or die "can't create index $index_path: $!";
64 $self->create( 0 );
65 } else {
66 $self->log->debug("Opening index: $index_path");
67 $invindexer = KinoSearch::InvIndexer->new( invindex => Grep::Search::KinoSearch::Schema->open( $index_path ) )
68 or die "can't open index $index_path: $!";
69 }
70 $indexes->{$index_path} = $invindexer;
71 }
72
73 return $invindexer;
74 }
75
76 =head2 add
77
78 $search->add( $record, $owner_id );
79
80 =cut
81
82 sub add {
83 my $self = shift;
84
85 my $i = shift or die "no record to add";
86 my $uid = shift;
87
88 die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
89
90 my $pk = { $i->primary_keys };
91
92 my $doc;
93
94 my @columns = map { $_->name } $i->columns;
95
96 foreach my $c ( @columns ) {
97
98 my $v = $i->$c;
99
100 if ( ref($v) ne '' ) {
101
102 foreach my $f_c ( qw/id name title/ ) {
103 if ( $i->$c->can( $f_c ) ) {
104 my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
105 my $col = $c . '_' . $f_c;
106 if ( $f_v ) {
107 warn " # $col = $f_v\n" if ($debug);
108 $doc->{ $col } = $f_v;
109 } else {
110 warn " . $col is NULL\n" if ($debug);
111 }
112 }
113 }
114
115 if ($v->isa('Jifty::DateTime')) {
116 warn " d $c = $v\n" if ($debug);
117 $doc->{$c} = $v;
118 } else {
119 warn " s $c = $v [",ref($v),"]\n" if ($debug);
120 }
121 next;
122 }
123
124 next if (! defined($v) || $v eq '');
125
126 eval { $v =~ s/<[^>]+>/ /gs; };
127 if ($@) {
128 Jifty->log->error("can't strip html from $c in item ", $i->id);
129 next;
130 }
131
132 if ( defined( $pk->{$c} ) ) {
133 $doc->{ $c } = $v;
134 warn " * $c = $v\n" if ($debug);
135 } else {
136 $doc->{ $c } = $v;
137 warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
138 }
139 }
140
141 # add _owner_id to speed up filtering of search results
142 $uid ||= Jifty->web->current_user->id;
143 $doc->{ '_owner_id' } = $uid;
144
145 $self->invindexer->add_doc( $doc );
146
147 $self->log->debug("added ", $i->id, " for user $uid to index");
148
149 return 1;
150 }
151
152 =head2 collection
153
154 Return C<Grep::Model::ItemCollection> which is result of C<search query>
155
156 my $ItemCollection = $search->collection( 'search query' );
157
158 =head2 hits
159
160 Return number of results from last C<collection> call
161
162 my $num_results = $search->hits;
163
164 =cut
165
166 sub collection {
167 my $self = shift;
168
169 my $q = shift or die "no q?";
170
171 my $searcher = KinoSearch::Searcher->new(
172 invindex => Grep::Search::KinoSearch::Schema->open( $self->index_path ), );
173 $self->log->debug("$searcher created");
174
175 my $full_q = "($q)";
176
177 my $uid = Jifty->web->current_user->id;
178 $full_q .= ' AND _owner_id:' . $uid if (defined $uid);
179
180 $self->log->debug("searching for '$q' using $full_q");
181
182 my $query_parser = KinoSearch::QueryParser->new(
183 schema => Grep::Search::KinoSearch::Schema->new,
184 fields => [ qw/ title link content summary category author / ],
185 );
186 $query_parser->set_heed_colons(1); # enable field:value AND/OR/NOT syntax
187 my $query = $query_parser->parse( $full_q );
188 my $hits = $searcher->search(
189 query => $query,
190 # offset => $offset,
191 # num_wanted => $hits_per_page,
192 );
193
194 $self->hits( $hits->total_hits );
195
196 $self->log->debug("found ", $self->hits, " results");
197
198 my $collection = Grep::Model::ItemCollection->new();
199
200 my @results;
201
202 my $i = 0;
203 while ( my $hit = $hits->fetch_hit ) {
204
205 my $score = $hit->{score};
206 my $title = $hit->{title};
207 my $id = $hit->{id};
208
209 $self->log->debug("result $i [$id] $title $score");
210
211 my $item = Grep::Model::Item->new();
212 my ($ok,$msg) = $item->load_by_cols( id => $id );
213
214 if ( $ok ) {
215 $collection->add_record( $item );
216 } else {
217 warn "can't load item $id\n";
218 }
219
220 }
221
222 $self->log->debug("finished search");
223
224 return $collection;
225 }
226
227 =head2 finish
228
229 $search->finish
230
231 =cut
232
233 sub finish {
234 my $self = shift;
235 if ($self->invindexer) {
236 $self->log->debug("closing index");
237 $self->invindexer->finish;
238 }
239
240 $self->log->debug("finish");
241
242 undef $self;
243
244 return 1;
245 }
246
247 =head2 snippet
248
249 my $short = $self->snippet( 50, $text );
250
251 =cut
252
253 sub snippet {
254 my $self = shift;
255
256 my $len = shift or die "no len?";
257 my $m = join(" ", @_);
258
259 $m =~ s/\s+/ /gs;
260
261 if (length($m) > $len) {
262 return substr($m,0,$len) . '...';
263 } else {
264 return $m;
265 }
266 }
267
268 package Grep::Search::KinoSearch::KeywordField;
269 use base qw( KinoSearch::Schema::FieldSpec );
270 sub analyzed { 0 }
271 #sub indexed { 1 }
272 #sub stored { 1 }
273 sub vectorized { 0 }
274
275 package Grep::Search::KinoSearch::Schema;
276
277 =head1 NAME
278
279 Grep::Search::KinoSearch::Schema - schema definition for full-text search
280
281 =cut
282
283 use base 'KinoSearch::Schema';
284 use KinoSearch::Analysis::PolyAnalyzer;
285
286 our %fields = (
287 id => 'Grep::Search::KinoSearch::KeywordField',
288
289 in_feed_id => 'Grep::Search::KinoSearch::KeywordField',
290 in_feed_url => 'Grep::Search::KinoSearch::KeywordField',
291 in_feed_title => 'KinoSearch::Schema::FieldSpec',
292 in_feed_owner => 'Grep::Search::KinoSearch::KeywordField',
293 in_feed_created_on => 'Grep::Search::KinoSearch::KeywordField',
294
295 title => 'KinoSearch::Schema::FieldSpec',
296 link => 'Grep::Search::KinoSearch::KeywordField',
297 content => 'KinoSearch::Schema::FieldSpec',
298 summary => 'KinoSearch::Schema::FieldSpec',
299 category => 'KinoSearch::Schema::FieldSpec',
300 author => 'KinoSearch::Schema::FieldSpec',
301 created_on => 'Grep::Search::KinoSearch::KeywordField',
302 last_update => 'Grep::Search::KinoSearch::KeywordField',
303
304 _owner_id => 'Grep::Search::KinoSearch::KeywordField',
305 );
306
307 sub analyzer {
308 return KinoSearch::Analysis::PolyAnalyzer->new( language => 'en' );
309 }
310
311 1;

  ViewVC Help
Powered by ViewVC 1.1.26