/[Search-Estraier]/trunk/scripts/dbi-indexer.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/scripts/dbi-indexer.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 144 by dpavlin, Wed May 10 20:31:02 2006 UTC revision 183 by dpavlin, Thu Aug 31 14:43:06 2006 UTC
# Line 5  use Search::Estraier 0.06; Line 5  use Search::Estraier 0.06;
5  use DBI;  use DBI;
6  use Data::Dumper;  use Data::Dumper;
7  use Encode qw/from_to/;  use Encode qw/from_to/;
8    use Time::HiRes qw/time/;
9    use Getopt::Long;
10    
11  =head1 NAME  =head1 NAME
12    
# Line 18  my $c = { Line 20  my $c = {
20          sql => qq{          sql => qq{
21                  select * from history_collection_view_cache                  select * from history_collection_view_cache
22          },          },
23            pk_col => '_id',
24            db_encoding => 'iso-8859-2',
25          debug => 0,          debug => 0,
26            user => 'admin',
27            passwd => 'admin',
28  };  };
29    
30    GetOptions($c, qw/node_url=s sql=s pk_col=s eb_encoding=s debug+ estuser=s estpasswd=s dbuser=s dbpasswd=s/);
31    
32    warn "# c: ", Dumper($c) if ($c->{debug});
33    
34  # create and configure node  # create and configure node
35  my $node = new Search::Estraier::Node(  my $node = new Search::Estraier::Node(
36          url => $c->{node_url},          url => $c->{node_url},
37          user => 'admin',          user => $c->{estuser},
38          passwd => 'admin',          passwd => $c->{estpasswd},
39          croak_on_error => 1,          croak_on_error => 1,
40          create => 1,          create => 1,
41          debug => $c->{debug},          debug => $c->{debug} >= 4 ? 1 : 0,
42  );  );
43    
44  # create DBI connection  # create DBI connection
45  my $dbh = DBI->connect("DBI:$c->{dbi}","","") || die $DBI::errstr;  my $dbh = DBI->connect("DBI:$c->{dbi}", $c->{dbuser}, $c->{dbpasswd}) || die $DBI::errstr;
46    
47  my $sth = $dbh->prepare($c->{sql}) || die $dbh->errstr();  my $sth = $dbh->prepare($c->{sql}) || die $dbh->errstr();
48  $sth->execute() || die $sth->errstr();  $sth->execute() || die $sth->errstr();
# Line 42  warn "# columns: ",join(",",@{ $sth->{NA Line 52  warn "# columns: ",join(",",@{ $sth->{NA
52  my $total = $sth->rows;  my $total = $sth->rows;
53  my $i = 1;  my $i = 1;
54    
55    my $t = time();
56    my $pk_col = $c->{pk_col} || 'id';
57    
58  while (my $row = $sth->fetchrow_hashref() ) {  while (my $row = $sth->fetchrow_hashref() ) {
59    
60          warn "# row: ",Dumper($row) if ($c->{debug});          warn "# row: ",Dumper($row) if ($c->{debug} >= 3);
61    
62          # create document          # create document
63          my $doc = new Search::Estraier::Document;          my $doc = new Search::Estraier::Document;
64    
65          $doc->add_attr('@uri', $row->{_id});          if (my $id = $row->{$pk_col}) {
66                    $doc->add_attr('@uri', $id);
67            } else {
68                    die "can't find pk_col column '$pk_col' in results\n";
69            }
70    
71          printf "%4d ",$i;          printf "%4d ",$i;
72    
# Line 57  while (my $row = $sth->fetchrow_hashref( Line 74  while (my $row = $sth->fetchrow_hashref(
74    
75                  if ($val) {                  if ($val) {
76                          # change encoding?                          # change encoding?
77                          from_to($val, 'ISO-8859-2', 'UTF-8');                          from_to($val, ($c->{db_encoding} || 'ISO-8859-1'), 'UTF-8');
78    
79                          # add attributes (make column usable from attribute search)                          # add attributes (make column usable from attribute search)
80                          $doc->add_attr($col, $val);                          $doc->add_attr($col, $val);
# Line 72  while (my $row = $sth->fetchrow_hashref( Line 89  while (my $row = $sth->fetchrow_hashref(
89    
90          }          }
91    
92          print " ", int(( $i++ / $total) * 100), "%\n";          warn "# doc draft: ",$doc->dump_draft, "\n" if ($c->{debug} >= 2);
   
         warn "# doc draft: ",$doc->dump_draft, "\n" if ($c->{debug});  
93    
94          die "error: ", $node->status,"\n" unless (eval { $node->put_doc($doc) });          die "error: ", $node->status,"\n" unless (eval { $node->put_doc($doc) });
95    
96            printf (" %d%% %.1f/s\n", int(( $i++ / $total) * 100), ( $i / (time() - $t) ) );
97    
98  }  }
Legend:
Removed from v.144  
changed lines
  Added in v.183
  ViewVC Help
Powered by ViewVC 1.1.26