/[Search-Estraier]/trunk/scripts/dbi-indexer.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/scripts/dbi-indexer.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 147 by dpavlin, Wed May 10 21:41:35 2006 UTC revision 195 by dpavlin, Tue Nov 14 16:39:08 2006 UTC
# Line 6  use DBI; Line 6  use DBI;
6  use Data::Dumper;  use Data::Dumper;
7  use Encode qw/from_to/;  use Encode qw/from_to/;
8  use Time::HiRes qw/time/;  use Time::HiRes qw/time/;
9    use Getopt::Long;
10    
11  =head1 NAME  =head1 NAME
12    
# Line 14  dbi-indexer.pl - example indexer of DBI Line 15  dbi-indexer.pl - example indexer of DBI
15  =cut  =cut
16    
17  my $c = {  my $c = {
18          node_url => 'http://localhost:1978/node/dbi',          node_url => 'http://localhost:1978/node/dbi-template1',
19          dbi => 'Pg:dbname=azop',          dbi => 'Pg:dbname=template1',
20            dbuser => 'postgres',
21          sql => qq{          sql => qq{
22                  select * from history_collection_view_cache                  select * from pg_database
23          },          },
24          pk_col => '_id',          pk_col => 'datname',
25          db_encoding => 'iso-8859-2',          db_encoding => 'iso-8859-2',
26          debug => 0,          debug => 0,
27            estuser => 'admin',
28            estpasswd => 'admin',
29            quiet => 0,
30  };  };
31    
32    GetOptions($c, qw/node_url=s dbi=s sql=s pk_col=s eb_encoding=s debug+ quiet+ estuser=s estpasswd=s dbuser=s dbpasswd=s/);
33    
34    warn "# c: ", Dumper($c) if ($c->{debug});
35    
36  # create and configure node  # create and configure node
37  my $node = new Search::Estraier::Node(  my $node = new Search::Estraier::Node(
38          url => $c->{node_url},          url => $c->{node_url},
39          user => 'admin',          user => $c->{estuser},
40          passwd => 'admin',          passwd => $c->{estpasswd},
41          croak_on_error => 1,          croak_on_error => 1,
42          create => 1,          create => 1,
43          debug => $c->{debug},          debug => $c->{debug} >= 4 ? 1 : 0,
44  );  );
45    
46  # create DBI connection  # create DBI connection
47  my $dbh = DBI->connect("DBI:$c->{dbi}","","") || die $DBI::errstr;  my $dbh = DBI->connect("DBI:$c->{dbi}", $c->{dbuser}, $c->{dbpasswd}) || die $DBI::errstr;
48    
49  my $sth = $dbh->prepare($c->{sql}) || die $dbh->errstr();  my $sth = $dbh->prepare($c->{sql}) || die $dbh->errstr();
50  $sth->execute() || die $sth->errstr();  $sth->execute() || die $sth->errstr();
# Line 50  my $pk_col = $c->{pk_col} || 'id'; Line 59  my $pk_col = $c->{pk_col} || 'id';
59    
60  while (my $row = $sth->fetchrow_hashref() ) {  while (my $row = $sth->fetchrow_hashref() ) {
61    
62          warn "# row: ",Dumper($row) if ($c->{debug});          warn "# row: ",Dumper($row) if ($c->{debug} >= 3);
63    
64          # create document          # create document
65          my $doc = new Search::Estraier::Document;          my $doc = new Search::Estraier::Document;
# Line 61  while (my $row = $sth->fetchrow_hashref( Line 70  while (my $row = $sth->fetchrow_hashref(
70                  die "can't find pk_col column '$pk_col' in results\n";                  die "can't find pk_col column '$pk_col' in results\n";
71          }          }
72    
73          printf "%4d ",$i;          my $out = '';
74            $out .= sprintf("%4d ",$i);
75    
76          while (my ($col,$val) = each %{$row}) {          while (my ($col,$val) = each %{$row}) {
77    
# Line 75  while (my $row = $sth->fetchrow_hashref( Line 85  while (my $row = $sth->fetchrow_hashref(
85                          # add body text to document (make it searchable using full-text index)                          # add body text to document (make it searchable using full-text index)
86                          $doc->add_text($val);                          $doc->add_text($val);
87    
88                          print "R";                          $out .= "R";
89                  } else {                  } else {
90                          print ".";                          $out .= ".";
91                  }                  }
92    
93          }          }
94    
95          warn "# doc draft: ",$doc->dump_draft, "\n" if ($c->{debug});          warn "# doc draft: ",$doc->dump_draft, "\n" if ($c->{debug} >= 2);
96    
97          die "error: ", $node->status,"\n" unless (eval { $node->put_doc($doc) });          die "error: ", $node->status,"\n" unless (eval { $node->put_doc($doc) });
98    
99          printf (" %d%% %.1f/s\n", int(( $i++ / $total) * 100), ( $i / (time() - $t) ) );          printf ("%s %d%% %.1f/s\n", $out, int(( $i++ / $total) * 100), ( $i / (time() - $t) ) ) unless ($c->{quiet});
100    
101  }  }
Legend:
Removed from v.147  
changed lines
  Added in v.195
  ViewVC Help
Powered by ViewVC 1.1.26