trunk/scripts/estcp.pl

#!/usr/bin/perl -w

use strict;
use Search::Estraier;
use URI::Escape qw/uri_escape/;
use Time::HiRes;
use POSIX qw/strftime/;

=head1 NAME

estcp.pl - copy Hyper Estraier index from one node to another

=cut

my ($from,$to) = @ARGV;

die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);

my $debug = 0;

# create and configure node
my $from_n = new Search::Estraier::Node(
        url => $from,
        croak_on_error => 1,
        debug => $debug,
);
my $to_n = new Search::Estraier::Node(
        url => $to,
        croak_on_error => 1,
        debug => $debug,
);

print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";

my $doc_num = $from_n->doc_num || 1;

my $res;
my $prev;
my $i = 1;

my $t = time();

do {
        $from_n->shuttle_url( $from_n->{url} . '/list',
                'application/x-www-form-urlencoded',
                'max=256' . ( $prev ? '&prev=' . uri_escape( $prev ) : '' ),
                \$res,
        );
        last unless ($res);
        foreach my $l (split(/\n/,$res)) {
                (my $id, $prev) = split(/\t/,$l, 2);
                $to_n->put_doc( $from_n->get_doc( $id ));
                $i++;
        }
        warn "$prev\n";

        my $rate = ( $i / (time() - $t) );
        printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",
                $i,
                ($i * 100 / $doc_num),
                $rate, 
                strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),
        );

} while ($res);

1	dpavlin	80	#!/usr/bin/perl -w
2
3			use strict;
4			use Search::Estraier;
5			use URI::Escape qw/uri_escape/;
6	dpavlin	82	use Time::HiRes;
7			use POSIX qw/strftime/;
8	dpavlin	80
9			=head1 NAME
10
11			estcp.pl - copy Hyper Estraier index from one node to another
12
13			=cut
14
15			my ($from,$to) = @ARGV;
16
17			die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);
18
19			my $debug = 0;
20
21			# create and configure node
22			my $from_n = new Search::Estraier::Node(
23			url => $from,
24			croak_on_error => 1,
25			debug => $debug,
26			);
27			my $to_n = new Search::Estraier::Node(
28			url => $to,
29			croak_on_error => 1,
30			debug => $debug,
31			);
32
33			print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";
34
35	dpavlin	82	my $doc_num = $from_n->doc_num \|\| 1;
36	dpavlin	80
37			my $res;
38			my $prev;
39	dpavlin	82	my $i = 1;
40	dpavlin	80
41	dpavlin	82	my $t = time();
42
43	dpavlin	80	do {
44			$from_n->shuttle_url( $from_n->{url} . '/list',
45			'application/x-www-form-urlencoded',
46			'max=256' . ( $prev ? '&prev=' . uri_escape( $prev ) : '' ),
47			\$res,
48			);
49			last unless ($res);
50			foreach my $l (split(/\n/,$res)) {
51			(my $id, $prev) = split(/\t/,$l, 2);
52			$to_n->put_doc( $from_n->get_doc( $id ));
53	dpavlin	82	$i++;
54	dpavlin	80	}
55	dpavlin	82	warn "$prev\n";
56
57			my $rate = ( $i / (time() - $t) );
58			printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",
59			$i,
60			($i * 100 / $doc_num),
61			$rate,
62			strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),
63			);
64
65	dpavlin	80	} while ($res);
66