/[Search-Estraier]/trunk/scripts/estcp.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /trunk/scripts/estcp.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 88 - (hide annotations)
Sat Jan 21 18:25:09 2006 UTC (18 years, 2 months ago) by dpavlin
File MIME type: text/plain
File size: 2139 byte(s)
fix URL extraction, make it less chatty (without -d flag)
1 dpavlin 80 #!/usr/bin/perl -w
2    
3     use strict;
4     use Search::Estraier;
5     use URI::Escape qw/uri_escape/;
6 dpavlin 82 use Time::HiRes;
7     use POSIX qw/strftime/;
8 dpavlin 80
9     =head1 NAME
10    
11     estcp.pl - copy Hyper Estraier index from one node to another
12    
13     =cut
14    
15     my ($from,$to) = @ARGV;
16    
17     die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);
18    
19     my $debug = 0;
20 dpavlin 83 my $max = 256;
21 dpavlin 80
22     # create and configure node
23     my $from_n = new Search::Estraier::Node(
24     url => $from,
25     croak_on_error => 1,
26     debug => $debug,
27     );
28     my $to_n = new Search::Estraier::Node(
29     url => $to,
30     croak_on_error => 1,
31     debug => $debug,
32     );
33    
34 dpavlin 85 unless(eval{ $to_n->name }) {
35 dpavlin 88 if ($to =~ m#^(http://.+)/node/([^/]+)$#) {
36 dpavlin 85 my ($url,$name) = ($1,$2);
37     print "Creating '$name' on $url\n";
38     $to_n->shuttle_url( $url . '/master?action=nodeadd',
39     'application/x-www-form-urlencoded',
40     'name=' . uri_escape($name) . '&label=' . uri_escape( $name ),
41     undef,
42     );
43     } else {
44     die "can't extract node name from $to\n";
45     }
46     }
47    
48 dpavlin 80 print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";
49    
50 dpavlin 82 my $doc_num = $from_n->doc_num || 1;
51 dpavlin 80
52     my $prev;
53 dpavlin 83 my $i = 0;
54     my $more = 1;
55 dpavlin 80
56 dpavlin 82 my $t = time();
57    
58 dpavlin 83 while($more) {
59     my $res;
60 dpavlin 80 $from_n->shuttle_url( $from_n->{url} . '/list',
61     'application/x-www-form-urlencoded',
62 dpavlin 83 'max=' . $max . ( $prev ? '&prev=' . uri_escape( $prev ) : '' ),
63 dpavlin 80 \$res,
64     );
65 dpavlin 83 if (! $res || $res eq '') {
66     $more = 0;
67     last;
68     }
69 dpavlin 80 foreach my $l (split(/\n/,$res)) {
70     (my $id, $prev) = split(/\t/,$l, 2);
71 dpavlin 84
72     #$to_n->put_doc( $from_n->get_doc( $id ));
73    
74     my $doc_draft = $from_n->_fetch_doc( id => $id, chomp_resbody => 1 );
75     $to_n->shuttle_url( $to_n->{url} . '/put_doc', 'text/x-estraier-draft', $doc_draft, undef) == 200 or die "can't insert $doc_draft\n";
76    
77 dpavlin 82 $i++;
78 dpavlin 80 }
79 dpavlin 88 warn "$prev\n" if ($debug);
80 dpavlin 82
81     my $rate = ( $i / (time() - $t) );
82     printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",
83     $i,
84     ($i * 100 / $doc_num),
85     $rate,
86     strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),
87     );
88    
89 dpavlin 83 }
90 dpavlin 80
91 dpavlin 83 print "Copy completed.\n";
92    

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26