/[Search-Estraier]/trunk/scripts/estcp.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/scripts/estcp.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 80 by dpavlin, Mon Jan 16 23:08:07 2006 UTC revision 83 by dpavlin, Tue Jan 17 00:41:18 2006 UTC
# Line 3  Line 3 
3  use strict;  use strict;
4  use Search::Estraier;  use Search::Estraier;
5  use URI::Escape qw/uri_escape/;  use URI::Escape qw/uri_escape/;
6    use Time::HiRes;
7    use POSIX qw/strftime/;
8    
9  =head1 NAME  =head1 NAME
10    
# Line 15  my ($from,$to) = @ARGV; Line 17  my ($from,$to) = @ARGV;
17  die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);  die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);
18    
19  my $debug = 0;  my $debug = 0;
20    my $max = 256;
21    
22    $max = 1024;
23    
24  # create and configure node  # create and configure node
25  my $from_n = new Search::Estraier::Node(  my $from_n = new Search::Estraier::Node(
# Line 30  my $to_n = new Search::Estraier::Node( Line 35  my $to_n = new Search::Estraier::Node(
35    
36  print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";  print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";
37    
38  my $tick = int($from_n->doc_num / 78);  my $doc_num = $from_n->doc_num || 1;
39    
 print "[";  
 my $res;  
40  my $prev;  my $prev;
41  my $i = 0;  my $i = 0;
42    my $more = 1;
43    
44    my $t = time();
45    
46  do {  while($more) {
47            my $res;
48          $from_n->shuttle_url( $from_n->{url} . '/list',          $from_n->shuttle_url( $from_n->{url} . '/list',
49                  'application/x-www-form-urlencoded',                  'application/x-www-form-urlencoded',
50                  'max=256' . ( $prev ? '&prev=' . uri_escape( $prev ) : '' ),                  'max=' . $max . ( $prev ? '&prev=' . uri_escape( $prev ) : '' ),
51                  \$res,                  \$res,
52          );          );
53          last unless ($res);          if (! $res || $res eq '') {
54                    $more = 0;
55                    last;
56            }
57          foreach my $l (split(/\n/,$res)) {          foreach my $l (split(/\n/,$res)) {
58                  (my $id, $prev) = split(/\t/,$l, 2);                  (my $id, $prev) = split(/\t/,$l, 2);
59                  $to_n->put_doc( $from_n->get_doc( $id ));                  $to_n->put_doc( $from_n->get_doc( $id ));
60                    $i++;
61          }          }
62          print "$prev\n";          warn "$prev\n";
63          print "." if ($i++ % $tick == 0);  
64  } while ($res);          my $rate = ( $i / (time() - $t) );
65            printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",
66                    $i,
67                    ($i * 100 / $doc_num),
68                    $rate,
69                    strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),
70            );
71    
72    }
73    
74    print "Copy completed.\n";
75    
 print "]\n";  
Legend:
Removed from v.80  
changed lines
  Added in v.83
  ViewVC Help
Powered by ViewVC 1.1.26