lib/Grep/Search.pm

package Grep::Search;

use strict;
use warnings;

use Data::Dump qw/dump/;
use Lucene;
use Jifty::Util;

my $index_path = Jifty::Util->app_root . '/var/lucene';

my ( $analyzer, $store, $writer );

my $debug = 1;
my $create;

sub create {

        if (defined( $create )) {
                Jifty->log->debug("using previous create $create");
                return $create;
        }

        if (! -e "$index_path/segments") {
                $create = 1;
                Jifty->log->debug("create index $index_path");
        } else {
                $create = 0;
                Jifty->log->debug("open index: $index_path");
        }
        return $create;
}

sub analyzer {
        my $self = shift;
        $analyzer ||= new Lucene::Analysis::Standard::StandardAnalyzer();
        return $analyzer;
}

sub store {
        my $self = shift;

        $store ||= Lucene::Store::FSDirectory->getDirectory( $index_path, $self->create );
        return $store;
}

sub writer {
        my $self = shift;
        $writer ||= new Lucene::Index::IndexWriter( $self->store, $self->analyzer, $self->create );
        return $writer;
}

=head2 add

  Grep::Search->add( $record, $owner_id );

=cut

sub add {
        my $self = shift;

        my $i = shift or die "no record to add";
        my $uid = shift;

        die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));

        my $pk = { $i->primary_keys };

        my $doc = new Lucene::Document;

        my @columns = map { $_->name } $i->columns;

        foreach my $c ( @columns ) {

                my $v = $i->$c;

                if ( ref($v) ne '' ) {

                        foreach my $f_c ( qw/id name title/ ) {
                                if ( $i->$c->can( $f_c ) ) {
                                        my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
                                        my $col = $c . '_' . $f_c;
                                        if ( $f_v ) {
                                                warn "  # $col = $f_v\n" if ($debug);
                                                $doc->add(Lucene::Document::Field->Text( $col, $f_v ));
                                        } else {
                                                warn "  . $col is NULL\n" if ($debug);
                                        }
                                }
                        }

                        if ($v->isa('Jifty::DateTime')) {
                                warn "  d $c = $v\n" if ($debug);
                                $doc->add(Lucene::Document::Field->Keyword( $c, "$v" ));
                        } else {
                                warn "  s $c = $v [",ref($v),"]\n" if ($debug);
                        }
                        next;
                }

                next if (! defined($v) || $v eq '');

                $v =~ s/<[^>]+>/ /gs;

                if ( defined( $pk->{$c} ) ) {
                        $doc->add(Lucene::Document::Field->Keyword( $c, $v ));
                        warn "  * $c = $v\n" if ($debug);
                } else {
                        $doc->add(Lucene::Document::Field->Text( $c, $v ));
                        warn "  + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
                }
        }

        # add _owner_id to speed up filtering of search results
        $uid ||= Jifty->web->current_user->id;
        $doc->add(Lucene::Document::Field->Keyword( '_owner_id', $uid ));

        $self->writer->addDocument($doc);

        Jifty->log->debug("added ", $i->id, " for user $uid to index");
}

=head2

  my $ItemCollection = Grep::Search->collection( 'search query' );

=cut

sub collection {
        my $self = shift;

        my $q = shift or die "no q?";

        return if ( $self->create );

        my $searcher = new Lucene::Search::IndexSearcher($self->store);
        my $parser = new Lucene::QueryParser("content", $self->analyzer);

        my $full_q = "($q) AND _owner_id:" . Jifty->web->current_user->id;

        my $query = $parser->parse( $full_q );

        Jifty->log->debug("searching for '$q' using ", $query->toString);

        my $hits = $searcher->search($query);
        my $num_hits = $hits->length();

        Jifty->log->debug("found $num_hits results");

        my $collection = Grep::Model::ItemCollection->new();

        my @results;

        for ( my $i = 0; $i < $num_hits; $i++ ) {

                my $doc = $hits->doc( $i );

                my $score = $hits->score($i);
                my $title = $doc->get("title");
                my $id = $doc->get("id");

                warn "## $i $score $title\n";

                my $item = Grep::Model::Item->new();
                my ($ok,$msg) = $item->load_by_cols( id => $id );

                if ( $ok ) {
                        $collection->add_record( $item );
                } else {
                        warn "can't load item $id\n";
                }

        }

        undef $hits;
        undef $query;
        undef $parser;
        undef $searcher;

        return $collection;
}

=head2 finish

  Grep::Search->finish

=cut

sub finish {
        my $self = shift;
        if ($writer) {
                warn "closing index\n";
                $writer->close;
        }
        undef $writer;
        undef $store;
        undef $create;

        return;
}

=for TODO

sub _signal {
        my $s = shift;
        warn "catched SIG $s\n";
        finish();
        exit(0);
}

$SIG{'__DIE__'} = \&_signal;
$SIG{'INT'} = \&_signal;
$SIG{'QUIT'} = \&_signal;

=cut

=head2 snippet

  my $short = $self->snippet( 50, $text );


=cut

sub snippet {
        my $self = shift;

        my $len = shift or die "no len?";
        my $m = join(" ", @_);

        $m =~ s/\s+/ /gs;

        if (length($m) > $len) {
                return substr($m,0,$len) . '...';
        } else {
                return $m;
        }
}

1;
1	package Grep::Search;
2
3	use strict;
4	use warnings;
5
6	use Data::Dump qw/dump/;
7	use Lucene;
8	use Jifty::Util;
9
10	my $index_path = Jifty::Util->app_root . '/var/lucene';
11
12	my ( $analyzer, $store, $writer );
13
14	my $debug = 1;
15	my $create;
16
17	sub create {
18
19	if (defined( $create )) {
20	Jifty->log->debug("using previous create $create");
21	return $create;
22	}
23
24	if (! -e "$index_path/segments") {
25	$create = 1;
26	Jifty->log->debug("create index $index_path");
27	} else {
28	$create = 0;
29	Jifty->log->debug("open index: $index_path");
30	}
31	return $create;
32	}
33
34	sub analyzer {
35	my $self = shift;
36	$analyzer \|\|= new Lucene::Analysis::Standard::StandardAnalyzer();
37	return $analyzer;
38	}
39
40	sub store {
41	my $self = shift;
42
43	$store \|\|= Lucene::Store::FSDirectory->getDirectory( $index_path, $self->create );
44	return $store;
45	}
46
47	sub writer {
48	my $self = shift;
49	$writer \|\|= new Lucene::Index::IndexWriter( $self->store, $self->analyzer, $self->create );
50	return $writer;
51	}
52
53	=head2 add
54
55	Grep::Search->add( $record, $owner_id );
56
57	=cut
58
59	sub add {
60	my $self = shift;
61
62	my $i = shift or die "no record to add";
63	my $uid = shift;
64
65	die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
66
67	my $pk = { $i->primary_keys };
68
69	my $doc = new Lucene::Document;
70
71	my @columns = map { $_->name } $i->columns;
72
73	foreach my $c ( @columns ) {
74
75	my $v = $i->$c;
76
77	if ( ref($v) ne '' ) {
78
79	foreach my $f_c ( qw/id name title/ ) {
80	if ( $i->$c->can( $f_c ) ) {
81	my $f_v = $i->$c->$f_c \|\| $i->$c->{values}->{ $f_c };
82	my $col = $c . '_' . $f_c;
83	if ( $f_v ) {
84	warn " # $col = $f_v\n" if ($debug);
85	$doc->add(Lucene::Document::Field->Text( $col, $f_v ));
86	} else {
87	warn " . $col is NULL\n" if ($debug);
88	}
89	}
90	}
91
92	if ($v->isa('Jifty::DateTime')) {
93	warn " d $c = $v\n" if ($debug);
94	$doc->add(Lucene::Document::Field->Keyword( $c, "$v" ));
95	} else {
96	warn " s $c = $v [",ref($v),"]\n" if ($debug);
97	}
98	next;
99	}
100
101	next if (! defined($v) \|\| $v eq '');
102
103	$v =~ s/<[^>]+>/ /gs;
104
105	if ( defined( $pk->{$c} ) ) {
106	$doc->add(Lucene::Document::Field->Keyword( $c, $v ));
107	warn " * $c = $v\n" if ($debug);
108	} else {
109	$doc->add(Lucene::Document::Field->Text( $c, $v ));
110	warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
111	}
112	}
113
114	# add _owner_id to speed up filtering of search results
115	$uid \|\|= Jifty->web->current_user->id;
116	$doc->add(Lucene::Document::Field->Keyword( '_owner_id', $uid ));
117
118	$self->writer->addDocument($doc);
119
120	Jifty->log->debug("added ", $i->id, " for user $uid to index");
121	}
122
123	=head2
124
125	my $ItemCollection = Grep::Search->collection( 'search query' );
126
127	=cut
128
129	sub collection {
130	my $self = shift;
131
132	my $q = shift or die "no q?";
133
134	return if ( $self->create );
135
136	my $searcher = new Lucene::Search::IndexSearcher($self->store);
137	my $parser = new Lucene::QueryParser("content", $self->analyzer);
138
139	my $full_q = "($q) AND _owner_id:" . Jifty->web->current_user->id;
140
141	my $query = $parser->parse( $full_q );
142
143	Jifty->log->debug("searching for '$q' using ", $query->toString);
144
145	my $hits = $searcher->search($query);
146	my $num_hits = $hits->length();
147
148	Jifty->log->debug("found $num_hits results");
149
150	my $collection = Grep::Model::ItemCollection->new();
151
152	my @results;
153
154	for ( my $i = 0; $i < $num_hits; $i++ ) {
155
156	my $doc = $hits->doc( $i );
157
158	my $score = $hits->score($i);
159	my $title = $doc->get("title");
160	my $id = $doc->get("id");
161
162	warn "## $i $score $title\n";
163
164	my $item = Grep::Model::Item->new();
165	my ($ok,$msg) = $item->load_by_cols( id => $id );
166
167	if ( $ok ) {
168	$collection->add_record( $item );
169	} else {
170	warn "can't load item $id\n";
171	}
172
173	}
174
175	undef $hits;
176	undef $query;
177	undef $parser;
178	undef $searcher;
179
180	return $collection;
181	}
182
183	=head2 finish
184
185	Grep::Search->finish
186
187	=cut
188
189	sub finish {
190	my $self = shift;
191	if ($writer) {
192	warn "closing index\n";
193	$writer->close;
194	}
195	undef $writer;
196	undef $store;
197	undef $create;
198
199	return;
200	}
201
202	=for TODO
203
204	sub _signal {
205	my $s = shift;
206	warn "catched SIG $s\n";
207	finish();
208	exit(0);
209	}
210
211	$SIG{'__DIE__'} = \&_signal;
212	$SIG{'INT'} = \&_signal;
213	$SIG{'QUIT'} = \&_signal;
214
215	=cut
216
217	=head2 snippet
218
219	my $short = $self->snippet( 50, $text );
220
221
222	=cut
223
224	sub snippet {
225	my $self = shift;
226
227	my $len = shift or die "no len?";
228	my $m = join(" ", @_);
229
230	$m =~ s/\s+/ /gs;
231
232	if (length($m) > $len) {
233	return substr($m,0,$len) . '...';
234	} else {
235	return $m;
236	}
237	}
238
239	1;