/[Grep]/bin/reindex.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /bin/reindex.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 47 by dpavlin, Wed Feb 21 03:04:48 2007 UTC revision 193 by dpavlin, Fri May 23 22:31:37 2008 UTC
# Line 7  use strict; Line 7  use strict;
7  use lib 'lib';  use lib 'lib';
8    
9  use Jifty;  use Jifty;
 use Lucene;  
10  use Grep::Search;  use Grep::Search;
11  use Data::Dump qw/dump/;  use Data::Dump qw/dump/;
12    use Text::DeDuper;
13    use Encode;
14    use Getopt::Long;
15    
16    my $keep_duplicates = 1;
17    
18    GetOptions(
19            'duplicates!' => $keep_duplicates,
20    );
21    
22    $|=1;
23    
24  BEGIN { Jifty->new; };  BEGIN { Jifty->new; };
25    
26  my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1 );  my $system_user = Grep::CurrentUser->superuser;
27    my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1, current_user => $system_user );
28  $coll->unlimit;  $coll->unlimit;
29    
30  print "indexing ", $coll->count, " items ";  Jifty->log->info( "indexing ", $coll->count, " items ", $keep_duplicates ? "" : "removing duplicates " );
31    
32    my $search = Grep::Search->new({ create => 1 });
33    my $deduper = Text::DeDuper->new();
34    
35    my ( $total, $duplicates ) = ( 0, 0 );
36    
37  while ( my $i = $coll->next ) {  while ( my $i = $coll->next ) {
38    
39          Grep::Search->add( $i );          print $i->id;
40    
41            if ( ! $keep_duplicates ) {
42    
43                    my $c = encode('utf-8', $i->content);
44    
45          print $i->id, ' ';                  my @s = sort $deduper->find_similar( $c );
46                    if ( @s ) {
47                            #warn " similar = ",dump( @s );
48    
49                            foreach my $id ( @s ) {
50                                    next if $id == $i->id;  # keep current
51                                    my $si = Grep::Model::Item->new();
52                                    $si->load( $id ) or die "can't find similar item $id";
53                                    print " -$id-";
54                                    $si->delete;
55                                    $duplicates++;
56                                    $search->invindexer->delete_by_term( 'id', $id );
57                            }
58                    }
59    
60                    $deduper->add_doc( $i->id, $c );
61            }
62    
63            $search->add( $i, $i->in_feed->owner->id );
64            print ' ';
65            $total++;
66  }  }
67    
68  print "\n";  Jifty->log->info( "$total records indexed", $duplicates ? " ($duplicates duplicates)" : "" );
69    
70  Grep::Search->finish;  $search->finish;
Legend:
Removed from v.47  
changed lines
  Added in v.193
  ViewVC Help
Powered by ViewVC 1.1.26