/[Grep]/lib/Grep/Search.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /lib/Grep/Search.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 160 - (hide annotations)
Sun Jun 10 22:23:34 2007 UTC (16 years, 10 months ago) by dpavlin
File size: 6037 byte(s)
be more robust at html stripping
1 dpavlin 47 package Grep::Search;
2    
3     use strict;
4     use warnings;
5 dpavlin 112 use base qw( Class::Accessor );
6 dpavlin 141 Grep::Search->mk_accessors( qw( invindexer create index_path hits ) );
7 dpavlin 47
8     use Data::Dump qw/dump/;
9 dpavlin 127 use KinoSearch::InvIndexer;
10     use KinoSearch::Searcher;
11 dpavlin 47 use Jifty::Util;
12    
13 dpavlin 109 my $debug = 0;
14 dpavlin 47
15 dpavlin 109 =head1 NAME
16 dpavlin 47
17 dpavlin 127 Grep::Search - full text search using L<KinoSearch>
18 dpavlin 47
19 dpavlin 109 =head1 METHODS
20 dpavlin 47
21 dpavlin 109 =head2 new
22 dpavlin 58
23 dpavlin 109 my $search = Grep::Search->new();
24    
25 dpavlin 145 my $search = Grep::Search->new( create => 1 );
26 dpavlin 144
27 dpavlin 145 =head2 invindexer
28    
29     Accessor to call any method defined on L<KinoSearch::InvIndexer>
30    
31     $search->invindexer->delete_by_term( 'id', 42 );
32    
33 dpavlin 109 =cut
34    
35 dpavlin 112 sub log { Jifty->web->log }
36    
37 dpavlin 109 sub new {
38     my $class = shift;
39     my $self = $class->SUPER::new(@_);
40    
41 dpavlin 127 my $index_path = Jifty::Util->app_root . '/var/invindex';
42 dpavlin 109
43     $self->index_path( $index_path );
44    
45 dpavlin 144 if ( ! -e "$index_path" || $self->create ) {
46 dpavlin 109 $self->log->debug("Creating new index $index_path");
47 dpavlin 127 $self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->clobber( $index_path ) ) );
48 dpavlin 47 } else {
49 dpavlin 109 $self->log->debug("Opening index: $index_path");
50 dpavlin 127 $self->invindexer( KinoSearch::InvIndexer->new( invindex => Grep::Search::Schema->open( $index_path ) ) );
51 dpavlin 47 }
52    
53 dpavlin 109 return $self;
54 dpavlin 49 }
55    
56 dpavlin 47 =head2 add
57    
58 dpavlin 109 $search->add( $record, $owner_id );
59 dpavlin 47
60     =cut
61    
62     sub add {
63     my $self = shift;
64    
65     my $i = shift or die "no record to add";
66 dpavlin 64 my $uid = shift;
67 dpavlin 47
68     die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
69    
70     my $pk = { $i->primary_keys };
71    
72 dpavlin 127 my $doc;
73 dpavlin 47
74     my @columns = map { $_->name } $i->columns;
75    
76     foreach my $c ( @columns ) {
77    
78     my $v = $i->$c;
79    
80     if ( ref($v) ne '' ) {
81 dpavlin 53
82     foreach my $f_c ( qw/id name title/ ) {
83     if ( $i->$c->can( $f_c ) ) {
84     my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
85     my $col = $c . '_' . $f_c;
86     if ( $f_v ) {
87     warn " # $col = $f_v\n" if ($debug);
88 dpavlin 127 $doc->{ $col } = $f_v;
89 dpavlin 53 } else {
90     warn " . $col is NULL\n" if ($debug);
91     }
92     }
93     }
94    
95     if ($v->isa('Jifty::DateTime')) {
96 dpavlin 47 warn " d $c = $v\n" if ($debug);
97 dpavlin 127 $doc->{$c} = $v;
98 dpavlin 47 } else {
99     warn " s $c = $v [",ref($v),"]\n" if ($debug);
100     }
101     next;
102     }
103    
104     next if (! defined($v) || $v eq '');
105    
106 dpavlin 160 eval { $v =~ s/<[^>]+>/ /gs; };
107     if ($@) {
108     Jifty->log->error("can't strip html from $c in item ", $i->id);
109     next;
110     }
111 dpavlin 47
112     if ( defined( $pk->{$c} ) ) {
113 dpavlin 127 $doc->{ $c } = $v;
114 dpavlin 47 warn " * $c = $v\n" if ($debug);
115     } else {
116 dpavlin 127 $doc->{ $c } = $v;
117 dpavlin 53 warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
118 dpavlin 47 }
119     }
120    
121 dpavlin 64 # add _owner_id to speed up filtering of search results
122     $uid ||= Jifty->web->current_user->id;
123 dpavlin 127 $doc->{ '_owner_id' } = $uid;
124 dpavlin 64
125 dpavlin 127 $self->invindexer->add_doc( $doc );
126 dpavlin 109
127 dpavlin 98 $self->log->debug("added ", $i->id, " for user $uid to index");
128 dpavlin 151
129     return 1;
130 dpavlin 47 }
131    
132 dpavlin 109 =head2 collection
133 dpavlin 47
134 dpavlin 141 Return C<Grep::Model::ItemCollection> which is result of C<search query>
135    
136 dpavlin 109 my $ItemCollection = $search->collection( 'search query' );
137 dpavlin 47
138 dpavlin 141 =head2 hits
139    
140     Return number of results from last C<collection> call
141    
142     my $num_results = $search->hits;
143    
144 dpavlin 47 =cut
145    
146     sub collection {
147     my $self = shift;
148    
149     my $q = shift or die "no q?";
150    
151 dpavlin 127 my $searcher = KinoSearch::Searcher->new(
152     invindex => Grep::Search::Schema->open( $self->index_path ), );
153 dpavlin 98 $self->log->debug("$searcher created");
154 dpavlin 47
155 dpavlin 151 my $full_q = "($q)";
156 dpavlin 64
157 dpavlin 151 my $uid = Jifty->web->current_user->id;
158 dpavlin 153 $full_q .= ' AND _owner_id:' . $uid if (defined $uid);
159 dpavlin 151
160 dpavlin 127 $self->log->debug("searching for '$q' using $full_q");
161 dpavlin 153
162     my $query_parser = KinoSearch::QueryParser->new(
163     schema => Grep::Search::Schema->new,
164     fields => [ qw/ title link content summary category author / ],
165     );
166     $query_parser->set_heed_colons(1); # enable field:value AND/OR/NOT syntax
167     my $query = $query_parser->parse( $full_q );
168 dpavlin 127 my $hits = $searcher->search(
169 dpavlin 153 query => $query,
170 dpavlin 127 # offset => $offset,
171     # num_wanted => $hits_per_page,
172     );
173 dpavlin 64
174 dpavlin 141 $self->hits( $hits->total_hits );
175 dpavlin 47
176 dpavlin 141 $self->log->debug("found ", $self->hits, " results");
177 dpavlin 47
178     my $collection = Grep::Model::ItemCollection->new();
179    
180     my @results;
181    
182 dpavlin 127 my $i = 0;
183     while ( my $hit = $hits->fetch_hit_hashref ) {
184 dpavlin 47
185 dpavlin 127 my $score = $hit->{score};
186     my $title = $hit->{title};
187     my $id = $hit->{id};
188 dpavlin 47
189 dpavlin 144 $self->log->debug("result $i [$id] $title $score");
190 dpavlin 47
191     my $item = Grep::Model::Item->new();
192     my ($ok,$msg) = $item->load_by_cols( id => $id );
193    
194     if ( $ok ) {
195     $collection->add_record( $item );
196     } else {
197     warn "can't load item $id\n";
198     }
199    
200     }
201    
202 dpavlin 127 $self->log->debug("finished search");
203 dpavlin 47
204     return $collection;
205     }
206    
207     =head2 finish
208    
209 dpavlin 109 $search->finish
210 dpavlin 47
211     =cut
212    
213     sub finish {
214     my $self = shift;
215 dpavlin 127 if ($self->invindexer) {
216 dpavlin 102 $self->log->debug("closing index");
217 dpavlin 127 $self->invindexer->finish;
218 dpavlin 47 }
219 dpavlin 58
220 dpavlin 110 $self->log->debug("finish");
221    
222 dpavlin 112 undef $self;
223    
224 dpavlin 151 return 1;
225 dpavlin 47 }
226    
227 dpavlin 53 =head2 snippet
228    
229     my $short = $self->snippet( 50, $text );
230    
231     =cut
232    
233     sub snippet {
234     my $self = shift;
235    
236     my $len = shift or die "no len?";
237     my $m = join(" ", @_);
238    
239     $m =~ s/\s+/ /gs;
240    
241     if (length($m) > $len) {
242     return substr($m,0,$len) . '...';
243     } else {
244     return $m;
245     }
246     }
247    
248 dpavlin 127 package Grep::Search::KeywordField;
249     use base qw( KinoSearch::Schema::FieldSpec );
250     sub analyzed { 0 }
251 dpavlin 153 #sub indexed { 1 }
252     #sub stored { 1 }
253     sub vectorized { 0 }
254 dpavlin 127
255     package Grep::Search::Schema;
256    
257     =head1 NAME
258    
259 dpavlin 136 Grep::Search::Schema - schema definition for full-text search
260 dpavlin 127
261     =cut
262    
263     use base 'KinoSearch::Schema';
264     use KinoSearch::Analysis::PolyAnalyzer;
265    
266 dpavlin 131 our %fields = (
267 dpavlin 127 id => 'Grep::Search::KeywordField',
268    
269     in_feed_id => 'Grep::Search::KeywordField',
270     in_feed_url => 'Grep::Search::KeywordField',
271     in_feed_title => 'KinoSearch::Schema::FieldSpec',
272     in_feed_owner => 'Grep::Search::KeywordField',
273     in_feed_created_on => 'Grep::Search::KeywordField',
274    
275     title => 'KinoSearch::Schema::FieldSpec',
276     link => 'Grep::Search::KeywordField',
277     content => 'KinoSearch::Schema::FieldSpec',
278     summary => 'KinoSearch::Schema::FieldSpec',
279     category => 'KinoSearch::Schema::FieldSpec',
280     author => 'KinoSearch::Schema::FieldSpec',
281     issued => 'Grep::Search::KeywordField',
282     modified => 'Grep::Search::KeywordField',
283    
284     _owner_id => 'Grep::Search::KeywordField',
285     );
286    
287     sub analyzer {
288     return KinoSearch::Analysis::PolyAnalyzer->new( language => 'en' );
289     }
290    
291 dpavlin 47 1;

  ViewVC Help
Powered by ViewVC 1.1.26