/[Search-Estraier]/trunk/scripts/dbi-indexer.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/scripts/dbi-indexer.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 131 by dpavlin, Mon May 8 21:05:32 2006 UTC revision 147 by dpavlin, Wed May 10 21:41:35 2006 UTC
# Line 1  Line 1 
1  #!/usr/bin/perl -w  #!/usr/bin/perl -w
2    
3  use strict;  use strict;
4  use Search::Estraier;  use Search::Estraier 0.06;
5  use DBI;  use DBI;
6  use Data::Dumper;  use Data::Dumper;
7  use Encode qw/from_to/;  use Encode qw/from_to/;
8    use Time::HiRes qw/time/;
9    
10  =head1 NAME  =head1 NAME
11    
# Line 12  dbi-indexer.pl - example indexer of DBI Line 13  dbi-indexer.pl - example indexer of DBI
13    
14  =cut  =cut
15    
16  my $node_url = 'http://localhost:1978/node/dbi';  my $c = {
17  my $dbi = 'Pg:dbname=azop';          node_url => 'http://localhost:1978/node/dbi',
18  my $sql = qq{          dbi => 'Pg:dbname=azop',
19          select * from history_collection_view_cache          sql => qq{
20                    select * from history_collection_view_cache
21            },
22            pk_col => '_id',
23            db_encoding => 'iso-8859-2',
24            debug => 0,
25  };  };
 my $debug = 0;  
26    
27  # create and configure node  # create and configure node
28  my $node = new Search::Estraier::Node(  my $node = new Search::Estraier::Node(
29          url => $node_url,          url => $c->{node_url},
30          user => 'admin',          user => 'admin',
31          passwd => 'admin',          passwd => 'admin',
32          croak_on_error => 1,          croak_on_error => 1,
33          debug => $debug,          create => 1,
34            debug => $c->{debug},
35  );  );
36    
37  # create DBI connection  # create DBI connection
38  my $dbh = DBI->connect("DBI:$dbi","","") || die $DBI::errstr;  my $dbh = DBI->connect("DBI:$c->{dbi}","","") || die $DBI::errstr;
39    
40  my $sth = $dbh->prepare($sql) || die $dbh->errstr();  my $sth = $dbh->prepare($c->{sql}) || die $dbh->errstr();
41  $sth->execute() || die $sth->errstr();  $sth->execute() || die $sth->errstr();
42    
43  warn "# columns: ",join(",",@{ $sth->{NAME} }),"\n" if ($debug);  warn "# columns: ",join(",",@{ $sth->{NAME} }),"\n" if ($c->{debug});
44    
45  my $total = $sth->rows;  my $total = $sth->rows;
46  my $i = 1;  my $i = 1;
47    
48    my $t = time();
49    my $pk_col = $c->{pk_col} || 'id';
50    
51  while (my $row = $sth->fetchrow_hashref() ) {  while (my $row = $sth->fetchrow_hashref() ) {
52    
53          warn "# row: ",Dumper($row) if ($debug);          warn "# row: ",Dumper($row) if ($c->{debug});
54    
55          # create document          # create document
56          my $doc = new Search::Estraier::Document;          my $doc = new Search::Estraier::Document;
57    
58          $doc->add_attr('@uri', $row->{_id});          if (my $id = $row->{$pk_col}) {
59                    $doc->add_attr('@uri', $id);
60            } else {
61                    die "can't find pk_col column '$pk_col' in results\n";
62            }
63    
64          printf "%4d ",$i;          printf "%4d ",$i;
65    
# Line 54  while (my $row = $sth->fetchrow_hashref( Line 67  while (my $row = $sth->fetchrow_hashref(
67    
68                  if ($val) {                  if ($val) {
69                          # change encoding?                          # change encoding?
70                          from_to($val, 'ISO-8859-2', 'UTF-8');                          from_to($val, ($c->{db_encoding} || 'ISO-8859-1'), 'UTF-8');
71    
72                          # add attributes (make column usable from attribute search)                          # add attributes (make column usable from attribute search)
73                          $doc->add_attr($col, $val);                          $doc->add_attr($col, $val);
# Line 69  while (my $row = $sth->fetchrow_hashref( Line 82  while (my $row = $sth->fetchrow_hashref(
82    
83          }          }
84    
85          print " ", int(( $i++ / $total) * 100), "%\n";          warn "# doc draft: ",$doc->dump_draft, "\n" if ($c->{debug});
   
         warn "# doc draft: ",$doc->dump_draft, "\n" if ($debug);  
86    
87          die "error: ", $node->status,"\n" unless (eval { $node->put_doc($doc) });          die "error: ", $node->status,"\n" unless (eval { $node->put_doc($doc) });
88    
89            printf (" %d%% %.1f/s\n", int(( $i++ / $total) * 100), ( $i / (time() - $t) ) );
90    
91  }  }
Legend:
Removed from v.131  
changed lines
  Added in v.147
  ViewVC Help
Powered by ViewVC 1.1.26