lib/Grep/Search.pm

package Grep::Search;

use strict;
use warnings;
use base qw( Class::Accessor Jifty::Object );
Grep::Search->mk_accessors( qw( analyzer store writer create index_path ) );

use Data::Dump qw/dump/;
use Lucene;
use Jifty::Util;

my $debug = 0;

=head1 NAME

Grep::Search - full text search

=head1 METHODS

=head2 new

  my $search = Grep::Search->new();

=cut

sub new {
        my $class = shift;        
        my $self = $class->SUPER::new(@_);

        my $index_path = Jifty::Util->app_root . '/var/lucene';

        $self->index_path( $index_path );

        if (! -e "$index_path/segments") {
                $self->create( 1 );
                $self->log->debug("Creating new index $index_path");
        } else {
                $self->create( 0 );
                $self->log->debug("Opening index: $index_path");
        }

        $self->analyzer( new Lucene::Analysis::Standard::StandardAnalyzer() );
        $self->log->debug($self->analyzer . " created");

        $self->store( Lucene::Store::FSDirectory->getDirectory( $index_path, $self->create ) );
        $self->log->debug($self->store, " created");

        return $self;
}


=head2 add

  $search->add( $record, $owner_id );

=cut

sub add {
        my $self = shift;

        my $i = shift or die "no record to add";
        my $uid = shift;

        die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));

        my $pk = { $i->primary_keys };

        my $doc = new Lucene::Document;

        my @columns = map { $_->name } $i->columns;

        foreach my $c ( @columns ) {

                my $v = $i->$c;

                if ( ref($v) ne '' ) {

                        foreach my $f_c ( qw/id name title/ ) {
                                if ( $i->$c->can( $f_c ) ) {
                                        my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
                                        my $col = $c . '_' . $f_c;
                                        if ( $f_v ) {
                                                warn "  # $col = $f_v\n" if ($debug);
                                                $doc->add(Lucene::Document::Field->Text( $col, $f_v ));
                                        } else {
                                                warn "  . $col is NULL\n" if ($debug);
                                        }
                                }
                        }

                        if ($v->isa('Jifty::DateTime')) {
                                warn "  d $c = $v\n" if ($debug);
                                $doc->add(Lucene::Document::Field->Keyword( $c, "$v" ));
                        } else {
                                warn "  s $c = $v [",ref($v),"]\n" if ($debug);
                        }
                        next;
                }

                next if (! defined($v) || $v eq '');

                $v =~ s/<[^>]+>/ /gs;

                if ( defined( $pk->{$c} ) ) {
                        $doc->add(Lucene::Document::Field->Keyword( $c, $v ));
                        warn "  * $c = $v\n" if ($debug);
                } else {
                        $doc->add(Lucene::Document::Field->Text( $c, $v ));
                        warn "  + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
                }
        }

        # add _owner_id to speed up filtering of search results
        $uid ||= Jifty->web->current_user->id;
        $doc->add(Lucene::Document::Field->Keyword( '_owner_id', $uid ));

        if (! defined( $self->writer )) {
                $self->writer( new Lucene::Index::IndexWriter( $self->store, $self->analyzer, $self->create ) );
                $self->log->debug($self->writer, " created");
        }

        $self->writer->addDocument($doc);

        $self->log->debug("added ", $i->id, " for user $uid to index");
}

=head2 collection

  my $ItemCollection = $search->collection( 'search query' );

=cut

sub collection {
        my $self = shift;

        my $q = shift or die "no q?";

        return if ( $self->create );

        my $searcher = new Lucene::Search::IndexSearcher($self->store);
        $self->log->debug("$searcher created");
        my $parser = new Lucene::QueryParser("content", $self->analyzer);
        $self->log->debug("$parser created");

        my $full_q = "($q) AND _owner_id:" . Jifty->web->current_user->id;

        my $query = $parser->parse( $full_q );

        $self->log->debug("searching for '$q' using ", $query->toString);

        my $hits = $searcher->search($query);
        my $num_hits = $hits->length();

        $self->log->debug("found $num_hits results");

        my $collection = Grep::Model::ItemCollection->new();

        my @results;

        for ( my $i = 0; $i < $num_hits; $i++ ) {

                my $doc = $hits->doc( $i );

                my $score = $hits->score($i);
                my $title = $doc->get("title");
                my $id = $doc->get("id");

                $self->log->debug("result $i $score $title");

                my $item = Grep::Model::Item->new();
                my ($ok,$msg) = $item->load_by_cols( id => $id );

                if ( $ok ) {
                        $collection->add_record( $item );
                } else {
                        warn "can't load item $id\n";
                }

        }

        undef $hits;
        undef $query;
        undef $parser;
        $searcher->close;
        undef $searcher;

        return $collection;
}

=head2 finish

  $search->finish

=cut

sub finish {
        my $self = shift;
        if ($self->writer) {
                $self->log->debug("closing index");
                $self->writer->close;
        }

        $self->writer( undef );
        $self->store( undef );
        $self->create( undef );
        $self->analyzer( undef );

        $self->log->debug("finish");

        return;
}

=for TODO

sub _signal {
        my $s = shift;
        warn "catched SIG $s\n";
        finish();
        exit(0);
}

$SIG{'__DIE__'} = \&_signal;
$SIG{'INT'} = \&_signal;
$SIG{'QUIT'} = \&_signal;

=cut

=head2 snippet

  my $short = $self->snippet( 50, $text );

=cut

sub snippet {
        my $self = shift;

        my $len = shift or die "no len?";
        my $m = join(" ", @_);

        $m =~ s/\s+/ /gs;

        if (length($m) > $len) {
                return substr($m,0,$len) . '...';
        } else {
                return $m;
        }
}

1;
1	dpavlin	47	package Grep::Search;
2
3			use strict;
4			use warnings;
5	dpavlin	109	use base qw( Class::Accessor Jifty::Object );
6			Grep::Search->mk_accessors( qw( analyzer store writer create index_path ) );
7	dpavlin	47
8			use Data::Dump qw/dump/;
9			use Lucene;
10			use Jifty::Util;
11
12	dpavlin	109	my $debug = 0;
13	dpavlin	47
14	dpavlin	109	=head1 NAME
15	dpavlin	47
16	dpavlin	109	Grep::Search - full text search
17	dpavlin	47
18	dpavlin	109	=head1 METHODS
19	dpavlin	47
20	dpavlin	109	=head2 new
21	dpavlin	58
22	dpavlin	109	my $search = Grep::Search->new();
23
24			=cut
25
26			sub new {
27			my $class = shift;
28			my $self = $class->SUPER::new(@_);
29
30			my $index_path = Jifty::Util->app_root . '/var/lucene';
31
32			$self->index_path( $index_path );
33
34	dpavlin	47	if (! -e "$index_path/segments") {
35	dpavlin	109	$self->create( 1 );
36			$self->log->debug("Creating new index $index_path");
37	dpavlin	47	} else {
38	dpavlin	109	$self->create( 0 );
39			$self->log->debug("Opening index: $index_path");
40	dpavlin	47	}
41
42	dpavlin	109	$self->analyzer( new Lucene::Analysis::Standard::StandardAnalyzer() );
43			$self->log->debug($self->analyzer . " created");
44	dpavlin	49
45	dpavlin	109	$self->store( Lucene::Store::FSDirectory->getDirectory( $index_path, $self->create ) );
46			$self->log->debug($self->store, " created");
47	dpavlin	49
48	dpavlin	109	return $self;
49	dpavlin	49	}
50
51	dpavlin	109
52	dpavlin	47	=head2 add
53
54	dpavlin	109	$search->add( $record, $owner_id );
55	dpavlin	47
56			=cut
57
58			sub add {
59			my $self = shift;
60
61			my $i = shift or die "no record to add";
62	dpavlin	64	my $uid = shift;
63	dpavlin	47
64			die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
65
66			my $pk = { $i->primary_keys };
67
68			my $doc = new Lucene::Document;
69
70			my @columns = map { $_->name } $i->columns;
71
72			foreach my $c ( @columns ) {
73
74			my $v = $i->$c;
75
76			if ( ref($v) ne '' ) {
77	dpavlin	53
78			foreach my $f_c ( qw/id name title/ ) {
79			if ( $i->$c->can( $f_c ) ) {
80			my $f_v = $i->$c->$f_c \|\| $i->$c->{values}->{ $f_c };
81			my $col = $c . '_' . $f_c;
82			if ( $f_v ) {
83			warn " # $col = $f_v\n" if ($debug);
84			$doc->add(Lucene::Document::Field->Text( $col, $f_v ));
85			} else {
86			warn " . $col is NULL\n" if ($debug);
87			}
88			}
89			}
90
91			if ($v->isa('Jifty::DateTime')) {
92	dpavlin	47	warn " d $c = $v\n" if ($debug);
93			$doc->add(Lucene::Document::Field->Keyword( $c, "$v" ));
94			} else {
95			warn " s $c = $v [",ref($v),"]\n" if ($debug);
96			}
97			next;
98			}
99
100			next if (! defined($v) \|\| $v eq '');
101
102			$v =~ s/<[^>]+>/ /gs;
103
104			if ( defined( $pk->{$c} ) ) {
105			$doc->add(Lucene::Document::Field->Keyword( $c, $v ));
106			warn " * $c = $v\n" if ($debug);
107			} else {
108			$doc->add(Lucene::Document::Field->Text( $c, $v ));
109	dpavlin	53	warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
110	dpavlin	47	}
111			}
112
113	dpavlin	64	# add _owner_id to speed up filtering of search results
114			$uid \|\|= Jifty->web->current_user->id;
115			$doc->add(Lucene::Document::Field->Keyword( '_owner_id', $uid ));
116
117	dpavlin	109	if (! defined( $self->writer )) {
118			$self->writer( new Lucene::Index::IndexWriter( $self->store, $self->analyzer, $self->create ) );
119			$self->log->debug($self->writer, " created");
120			}
121
122	dpavlin	49	$self->writer->addDocument($doc);
123	dpavlin	47
124	dpavlin	98	$self->log->debug("added ", $i->id, " for user $uid to index");
125	dpavlin	47	}
126
127	dpavlin	109	=head2 collection
128	dpavlin	47
129	dpavlin	109	my $ItemCollection = $search->collection( 'search query' );
130	dpavlin	47
131			=cut
132
133			sub collection {
134			my $self = shift;
135
136			my $q = shift or die "no q?";
137
138	dpavlin	58	return if ( $self->create );
139
140	dpavlin	49	my $searcher = new Lucene::Search::IndexSearcher($self->store);
141	dpavlin	98	$self->log->debug("$searcher created");
142	dpavlin	49	my $parser = new Lucene::QueryParser("content", $self->analyzer);
143	dpavlin	98	$self->log->debug("$parser created");
144	dpavlin	47
145	dpavlin	64	my $full_q = "($q) AND _owner_id:" . Jifty->web->current_user->id;
146
147			my $query = $parser->parse( $full_q );
148
149	dpavlin	98	$self->log->debug("searching for '$q' using ", $query->toString);
150	dpavlin	47
151			my $hits = $searcher->search($query);
152			my $num_hits = $hits->length();
153
154	dpavlin	98	$self->log->debug("found $num_hits results");
155	dpavlin	47
156			my $collection = Grep::Model::ItemCollection->new();
157
158			my @results;
159
160			for ( my $i = 0; $i < $num_hits; $i++ ) {
161
162			my $doc = $hits->doc( $i );
163
164			my $score = $hits->score($i);
165			my $title = $doc->get("title");
166			my $id = $doc->get("id");
167
168	dpavlin	102	$self->log->debug("result $i $score $title");
169	dpavlin	47
170			my $item = Grep::Model::Item->new();
171			my ($ok,$msg) = $item->load_by_cols( id => $id );
172
173			if ( $ok ) {
174			$collection->add_record( $item );
175			} else {
176			warn "can't load item $id\n";
177			}
178
179			}
180
181			undef $hits;
182			undef $query;
183			undef $parser;
184	dpavlin	76	$searcher->close;
185	dpavlin	47	undef $searcher;
186
187			return $collection;
188			}
189
190			=head2 finish
191
192	dpavlin	109	$search->finish
193	dpavlin	47
194			=cut
195
196			sub finish {
197			my $self = shift;
198	dpavlin	109	if ($self->writer) {
199	dpavlin	102	$self->log->debug("closing index");
200	dpavlin	109	$self->writer->close;
201	dpavlin	47	}
202	dpavlin	58
203	dpavlin	110	$self->writer( undef );
204			$self->store( undef );
205			$self->create( undef );
206			$self->analyzer( undef );
207	dpavlin	109
208	dpavlin	110	$self->log->debug("finish");
209
210	dpavlin	58	return;
211	dpavlin	47	}
212
213	dpavlin	57	=for TODO
214
215	dpavlin	47	sub _signal {
216			my $s = shift;
217			warn "catched SIG $s\n";
218			finish();
219			exit(0);
220			}
221
222			$SIG{'__DIE__'} = \&_signal;
223			$SIG{'INT'} = \&_signal;
224			$SIG{'QUIT'} = \&_signal;
225
226	dpavlin	57	=cut
227	dpavlin	53
228			=head2 snippet
229
230			my $short = $self->snippet( 50, $text );
231
232			=cut
233
234			sub snippet {
235			my $self = shift;
236
237			my $len = shift or die "no len?";
238			my $m = join(" ", @_);
239
240			$m =~ s/\s+/ /gs;
241
242			if (length($m) > $len) {
243			return substr($m,0,$len) . '...';
244			} else {
245			return $m;
246			}
247			}
248
249	dpavlin	47	1;