/[Search-Estraier]/trunk/scripts/estcp-mt.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/scripts/estcp-mt.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

trunk/scripts/estcp.pl revision 85 by dpavlin, Tue Jan 17 15:00:50 2006 UTC trunk/scripts/estcp-mt.pl revision 86 by dpavlin, Thu Jan 19 14:33:33 2006 UTC
# Line 5  use Search::Estraier; Line 5  use Search::Estraier;
5  use URI::Escape qw/uri_escape/;  use URI::Escape qw/uri_escape/;
6  use Time::HiRes;  use Time::HiRes;
7  use POSIX qw/strftime/;  use POSIX qw/strftime/;
8    use Config;
9    use threads;
10    use Thread::Queue;
11    
12  =head1 NAME  =head1 NAME
13    
# Line 12  estcp.pl - copy Hyper Estraier index fro Line 15  estcp.pl - copy Hyper Estraier index fro
15    
16  =cut  =cut
17    
18    die "Your perl isn't compiled with support for ithreads\n" unless ($Config{useithreads});
19    
20    
21  my ($from,$to) = @ARGV;  my ($from,$to) = @ARGV;
22    
23  die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);  die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);
# Line 45  unless(eval{ $to_n->name }) { Line 51  unless(eval{ $to_n->name }) {
51          }          }
52  }  }
53    
54  print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";  # total processed elements
55    my $i : shared = 1;
56    
57    my $q_id = Thread::Queue->new;
58    my $q_drafts = Thread::Queue->new;
59    
60    my $get_thr = threads->new( sub {
61            while (my $id = $q_id->dequeue) {
62                    print STDERR "get_thr, id: $id\n" if ($debug);
63                    my $doc_draft = $from_n->_fetch_doc( id => $id, chomp_resbody => 1 );
64                    $q_drafts->enqueue( $doc_draft );
65            }
66    } );
67    
68    my $t = time();
69    my $t_refresh = time();
70  my $doc_num = $from_n->doc_num || 1;  my $doc_num = $from_n->doc_num || 1;
71    
72    my $put_thr = threads->new( sub {
73            while (my $doc_draft = $q_drafts->dequeue) {
74                    print STDERR "put_thr, $doc_draft\n" if ($debug);
75                    $to_n->shuttle_url( $to_n->{url} . '/put_doc', 'text/x-estraier-draft', $doc_draft, undef) == 200 or die "can't insert $doc_draft\n";
76    
77                    $i++;
78                    if (time() - $t_refresh > 3) {
79                            my $rate = ( $i / ((time() - $t) || 1) );
80                            printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",
81                                    $i,
82                                    ($i * 100 / $doc_num),
83                                    $rate,
84                                    strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),
85                            );
86                            $t_refresh = time();
87                    }
88    
89            }
90    } );
91    
92    print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";
93    
94  my $prev;  my $prev;
 my $i = 0;  
95  my $more = 1;  my $more = 1;
96    
 my $t = time();  
   
97  while($more) {  while($more) {
98          my $res;          my $res;
99          $from_n->shuttle_url( $from_n->{url} . '/list',          $from_n->shuttle_url( $from_n->{url} . '/list',
# Line 71  while($more) { Line 110  while($more) {
110    
111                  #$to_n->put_doc( $from_n->get_doc( $id ));                  #$to_n->put_doc( $from_n->get_doc( $id ));
112    
113                  my $doc_draft = $from_n->_fetch_doc( id => $id, chomp_resbody => 1 );                  #my $doc_draft = $from_n->_fetch_doc( id => $id, chomp_resbody => 1 );
114                  $to_n->shuttle_url( $to_n->{url} . '/put_doc', 'text/x-estraier-draft', $doc_draft, undef) == 200 or die "can't insert $doc_draft\n";                  #$to_n->shuttle_url( $to_n->{url} . '/put_doc', 'text/x-estraier-draft', $doc_draft, undef) == 200 or die "can't insert $doc_draft\n";
115    
116                  $i++;                  $q_id->enqueue( $id );
117          }          }
118          warn "$prev\n";          warn "$prev\n";
119    
         my $rate = ( $i / (time() - $t) );  
         printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",  
                 $i,  
                 ($i * 100 / $doc_num),  
                 $rate,  
                 strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),  
         );  
   
120  }  }
121    
122    $get_thr->join;
123    $put_thr->join;
124    
125  print "Copy completed.\n";  print "Copy completed.\n";
126    
Legend:
Removed from v.85  
changed lines
  Added in v.86
  ViewVC Help
Powered by ViewVC 1.1.26