/[Sack]/trunk/bin/couchdb2shards.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/bin/couchdb2shards.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 206 by dpavlin, Sat Nov 21 13:21:34 2009 UTC revision 207 by dpavlin, Sat Nov 21 13:34:53 2009 UTC
# Line 8  use Storable qw(); Line 8  use Storable qw();
8  use JSON;  use JSON;
9  use Data::Dump qw(dump);  use Data::Dump qw(dump);
10  use Time::HiRes qw(time);  use Time::HiRes qw(time);
11    use File::Path qw(make_path remove_tree);
12    
13  my $name = 'pxelator';  my $name = 'pxelator';
14  my $shard_size = 5000;  my $shard_size = 1000;
15    
16    
17  my $path = '/tmp/sack';  my $path = "/tmp/sack/$name";
18  mkdir $path;  remove_tree $path;
19  $path .= '/' . $name;  make_path $path;
 mkdir $path;  
20    
21    
22  sub couchdb_socket {  sub couchdb_socket {
# Line 52  warn "# skip total: $total\n"; Line 52  warn "# skip total: $total\n";
52  our $shard;  our $shard;
53  our $shard_nr = 0;  our $shard_nr = 0;
54  my $t_start = time();  my $t_start = time();
55    my $total_bytes;
56    
57  sub save_shard {  sub save_shard {
58          my $shard_path = sprintf("%s/%06d.%d", $path, $shard_nr++ * $shard_size, $shard_size);          my $shard_path = sprintf("%s/%06d.%d", $path, $shard_nr++ * $shard_size, $shard_size);
# Line 60  sub save_shard { Line 61  sub save_shard {
61          my $dt = time() - $t_start;          my $dt = time() - $t_start;
62          my $pos = $shard_nr * $shard_size;          my $pos = $shard_nr * $shard_size;
63          my $rec_s = $pos / $dt;          my $rec_s = $pos / $dt;
64          my $total_t = $total / $rec_s;          my $end_t = ( $total / $rec_s ) - $dt;
65            my $shard_size = -s $shard_path;
66    
67          warn sprintf "shard %s %d bytes\t%3.2f%% %.2f/s ETA %d:%02ds\n"          warn sprintf "shard %s %d bytes\t%8.2f%% %8.2f/s  ETA %d:%02ds\n"
68                  , $shard_path, -s $shard_path                  , $shard_path, $shard_size
69                  , $pos * 100 / $total                  , $pos * 100 / $total
70                  , $rec_s,                  , $rec_s,
71                  , $total_t / 60, $total_t % 60                  , $end_t / 60, $end_t % 60
72                  ;                  ;
73    
74            $total_bytes += $shard_size;
75    
76          $shard = [];          $shard = [];
77  }  }
78    
# Line 85  while(<$sock>) { Line 89  while(<$sock>) {
89          }          }
90  }  }
91  save_shard;  save_shard;
92    
93    warn "sharded $path $total_bytes bytes\n";
Legend:
Removed from v.206  
changed lines
  Added in v.207
  ViewVC Help
Powered by ViewVC 1.1.26