/[Grep]/lib/Grep/Search/KinoSearch.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /lib/Grep/Search/KinoSearch.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 127 - (show annotations)
Sun Apr 29 00:16:05 2007 UTC (16 years, 10 months ago) by dpavlin
Original Path: lib/Grep/Search.pm
File size: 5110 byte(s)
Move from Lucene (mostly because locking problems prevented fastcgi
deployment, and later haunted development server too) to KinoSearch.
For good measure added (slow) de-duplication and increased version to 0.02
1 package Grep::Search;
2
3 use strict;
4 use warnings;
5 use base qw( Class::Accessor );
6 Grep::Search->mk_accessors( qw( invindexer create index_path ) );
7
8 use Data::Dump qw/dump/;
9 use KinoSearch::InvIndexer;
10 use KinoSearch::Searcher;
11 use Jifty::Util;
12
13 my $debug = 0;
14
15 =head1 NAME
16
17 Grep::Search - full text search using L<KinoSearch>
18
19 =head1 METHODS
20
21 =head2 new
22
23 my $search = Grep::Search->new();
24
25 =cut
26
27 sub log { Jifty->web->log }
28
29 sub new {
30 my $class = shift;
31 my $self = $class->SUPER::new(@_);
32
33 my $index_path = Jifty::Util->app_root . '/var/invindex';
34
35 $self->index_path( $index_path );
36
37 if (! -e "$index_path") {
38 $self->log->debug("Creating new index $index_path");
39 $self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->clobber( $index_path ) ) );
40 } else {
41 $self->log->debug("Opening index: $index_path");
42 $self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->open( $index_path ) ) );
43 }
44
45 return $self;
46 }
47
48 =head2 add
49
50 $search->add( $record, $owner_id );
51
52 =cut
53
54 sub add {
55 my $self = shift;
56
57 my $i = shift or die "no record to add";
58 my $uid = shift;
59
60 die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
61
62 my $pk = { $i->primary_keys };
63
64 my $doc;
65
66 my @columns = map { $_->name } $i->columns;
67
68 foreach my $c ( @columns ) {
69
70 my $v = $i->$c;
71
72 if ( ref($v) ne '' ) {
73
74 foreach my $f_c ( qw/id name title/ ) {
75 if ( $i->$c->can( $f_c ) ) {
76 my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
77 my $col = $c . '_' . $f_c;
78 if ( $f_v ) {
79 warn " # $col = $f_v\n" if ($debug);
80 $doc->{ $col } = $f_v;
81 } else {
82 warn " . $col is NULL\n" if ($debug);
83 }
84 }
85 }
86
87 if ($v->isa('Jifty::DateTime')) {
88 warn " d $c = $v\n" if ($debug);
89 $doc->{$c} = $v;
90 } else {
91 warn " s $c = $v [",ref($v),"]\n" if ($debug);
92 }
93 next;
94 }
95
96 next if (! defined($v) || $v eq '');
97
98 $v =~ s/<[^>]+>/ /gs;
99
100 if ( defined( $pk->{$c} ) ) {
101 $doc->{ $c } = $v;
102 warn " * $c = $v\n" if ($debug);
103 } else {
104 $doc->{ $c } = $v;
105 warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
106 }
107 }
108
109 # add _owner_id to speed up filtering of search results
110 $uid ||= Jifty->web->current_user->id;
111 $doc->{ '_owner_id' } = $uid;
112
113 $self->invindexer->add_doc( $doc );
114
115 $self->log->debug("added ", $i->id, " for user $uid to index");
116 }
117
118 =head2 collection
119
120 my $ItemCollection = $search->collection( 'search query' );
121
122 =cut
123
124 sub collection {
125 my $self = shift;
126
127 my $q = shift or die "no q?";
128
129 my $searcher = KinoSearch::Searcher->new(
130 invindex => Grep::Search::Schema->open( $self->index_path ), );
131 $self->log->debug("$searcher created");
132
133 my $full_q = "($q) AND _owner_id:" . Jifty->web->current_user->id;
134
135 $self->log->debug("searching for '$q' using $full_q");
136 my $hits = $searcher->search(
137 query => $full_q,
138 # offset => $offset,
139 # num_wanted => $hits_per_page,
140 );
141
142 my $num_hits = $hits->total_hits;
143
144 $self->log->debug("found $num_hits results");
145
146 my $collection = Grep::Model::ItemCollection->new();
147
148 my @results;
149
150 my $i = 0;
151 while ( my $hit = $hits->fetch_hit_hashref ) {
152
153 my $score = $hit->{score};
154 my $title = $hit->{title};
155 my $id = $hit->{id};
156
157 $self->log->debug("result $i $score $title");
158
159 my $item = Grep::Model::Item->new();
160 my ($ok,$msg) = $item->load_by_cols( id => $id );
161
162 if ( $ok ) {
163 $collection->add_record( $item );
164 } else {
165 warn "can't load item $id\n";
166 }
167
168 }
169
170 $self->log->debug("finished search");
171
172 return $collection;
173 }
174
175 =head2 finish
176
177 $search->finish
178
179 =cut
180
181 sub finish {
182 my $self = shift;
183 if ($self->invindexer) {
184 $self->log->debug("closing index");
185 $self->invindexer->finish;
186 }
187
188 $self->log->debug("finish");
189
190 undef $self;
191
192 return;
193 }
194
195 =cut
196
197 =head2 snippet
198
199 my $short = $self->snippet( 50, $text );
200
201 =cut
202
203 sub snippet {
204 my $self = shift;
205
206 my $len = shift or die "no len?";
207 my $m = join(" ", @_);
208
209 $m =~ s/\s+/ /gs;
210
211 if (length($m) > $len) {
212 return substr($m,0,$len) . '...';
213 } else {
214 return $m;
215 }
216 }
217
218 package Grep::Search::KeywordField;
219 use base qw( KinoSearch::Schema::FieldSpec );
220 sub analyzed { 0 }
221
222 package Grep::Search::Schema;
223
224 =head1 NAME
225
226 Grep::Search::Schema - data definition
227
228 =cut
229
230 use base 'KinoSearch::Schema';
231 use KinoSearch::Analysis::PolyAnalyzer;
232
233 our %FIELDS = (
234 id => 'Grep::Search::KeywordField',
235
236 in_feed_id => 'Grep::Search::KeywordField',
237 in_feed_url => 'Grep::Search::KeywordField',
238 in_feed_title => 'KinoSearch::Schema::FieldSpec',
239 in_feed_owner => 'Grep::Search::KeywordField',
240 in_feed_created_on => 'Grep::Search::KeywordField',
241
242 title => 'KinoSearch::Schema::FieldSpec',
243 link => 'Grep::Search::KeywordField',
244 content => 'KinoSearch::Schema::FieldSpec',
245 summary => 'KinoSearch::Schema::FieldSpec',
246 category => 'KinoSearch::Schema::FieldSpec',
247 author => 'KinoSearch::Schema::FieldSpec',
248 issued => 'Grep::Search::KeywordField',
249 modified => 'Grep::Search::KeywordField',
250
251 _owner_id => 'Grep::Search::KeywordField',
252 );
253
254 sub analyzer {
255 return KinoSearch::Analysis::PolyAnalyzer->new( language => 'en' );
256 }
257
258 1;

  ViewVC Help
Powered by ViewVC 1.1.26