/[couchdb]/scripts/reblog2couchdb.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /scripts/reblog2couchdb.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 5 by dpavlin, Tue Aug 5 15:03:24 2008 UTC revision 10 by dpavlin, Fri Aug 8 19:19:12 2008 UTC
# Line 9  $|++; Line 9  $|++;
9    
10  my $database = 'reblog';  my $database = 'reblog';
11    
12  my $connect = "DBI:mysql:database=$database";  my $dbi = "DBI:mysql:database=$database";
13  $connect .= ";host=127.0.0.1;port=13306";       # XXX over ssh  $dbi .= ";host=127.0.0.1;port=13306";   # XXX over ssh
14    
 my $dbh = DBI->connect($connect,"","") || die $DBI::errstr;  
   
 # select all posts which have been read or unread  
15  my $sql = qq{  my $sql = qq{
16          select          select
17                  items.id as _id,                  items.id as _id,
# Line 23  my $sql = qq{ Line 20  my $sql = qq{
20                  feeds.title as feed_title,                  feeds.title as feed_title,
21                  feeds.link as feed_link,                  feeds.link as feed_link,
22                  feeds.description as feed_description                  feeds.description as feed_description
23    --              t.value_long as tags
24          from items          from items
25          join items_userdata on items.id = item_id          join items_userdata on items.id = item_id
 --      where label = 'read' and value_numeric = ?  
26          join feeds on items.feed_id = feeds.id          join feeds on items.feed_id = feeds.id
27    --      left outer join items_userdata as t on items.id = t.item_id and t.label='tags'
28            where items.id > ? and items_userdata.label = 'published' and items_userdata.value_numeric = 1
29          order by items.id asc          order by items.id asc
30          limit 5000          limit 1000
31  };  };
32    
 my $sth = $dbh->prepare($sql) || die $dbh->errstr();  
 $sth->execute( 1 ) || die $sth->errstr();  
   
 warn dump( $sth->{NAME} );  
   
 print "found ",$sth->rows," items to process...";  
   
33  my $c = CouchDB::Client->new(uri => 'http://localhost:5984/');  my $c = CouchDB::Client->new(uri => 'http://localhost:5984/');
34    
35  $c->testConnection or die "The server cannot be reached";  $c->testConnection or die "The server cannot be reached";
36  print "Running version " . $c->serverInfo->{version} . "\n";  print "CouchDB version " . $c->serverInfo->{version} . "\n";
37  my $db = $c->newDB( $database );  my $db = $c->newDB( $database );
38  $db->create unless $c->dbExists( $database );  $db->create unless $c->dbExists( $database );
39    
40  my @docs = $db->listDocs;  my $status = $db->newDoc( '_sync' );
41  my $row_id = shift @docs || 0;  eval { $status->retrieve };
42    $status->create if $@;
43    
44    print "status ",dump( $status->{data} ), "\n";
45    
46    my $last_row = $status->{data}->{last_row_id} || 0;
47    
48    sub commit_last_row {
49            warn "commit_last_row $last_row\n";
50            $status->{data}->{last_row_id} = $last_row;
51            $status->update;
52    }
53    
54    my $dbh = DBI->connect($dbi,"","") || die $DBI::errstr;
55    
56    print "Fetching items from $dbi id > $last_row\n";
57    
58    my $sth = $dbh->prepare($sql) || die $dbh->errstr();
59    $sth->execute( $last_row ) || die $sth->errstr();
60    
61    warn dump( $sth->{NAME} );
62    
63    print "found ",$sth->rows," items to process...\n";
64    
65  my $pk = 'id';  my $pk = 'id';
66    
67    my $count = 0;
68    
69  while (my $row = $sth->fetchrow_hashref() ) {  while (my $row = $sth->fetchrow_hashref() ) {
70          my $_id = $row->{_id} || die "row needs _id";          my $_id = $row->{_id} || die "row needs _id";
71          my $doc = $db->newDoc( $_id );          my $doc = $db->newDoc( $_id );
# Line 71  while (my $row = $sth->fetchrow_hashref( Line 87  while (my $row = $sth->fetchrow_hashref(
87                  $doc->retrieve;                  $doc->retrieve;
88                  row2doc( $row, $doc )->update;                  row2doc( $row, $doc )->update;
89  #               eval { $doc->update };  #               eval { $doc->update };
90                  warn ( $@ ? "ERROR $_id $@" : "updated $_id" ), $/;                  warn $@ ? "$count ERROR $_id $@\n" : "$count updated $_id\n";
91          } else {          } else {
92                  warn "created ",dump( $row ),$/;                  warn "$count created $_id\n";
93          }          }
94    
95            $last_row = $row->{id};
96            $count++;
97    
98            commit_last_row if $count % 100 == 0    # checkpoint every 100 records
99  }  }
100    
101    commit_last_row;
102    
103  __END__  __END__
104    
105  $sql = qq{  $sql = qq{
Legend:
Removed from v.5  
changed lines
  Added in v.10
  ViewVC Help
Powered by ViewVC 1.1.26