trunk/scripts/estcp.pl

#!/usr/bin/perl -w

use strict;
use Search::Estraier;
use URI::Escape qw/uri_escape/;
use Time::HiRes;
use POSIX qw/strftime/;

=head1 NAME

estcp.pl - copy Hyper Estraier index from one node to another

=cut

my ($from,$to) = @ARGV;

die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);

my $debug = 0;
my $max = 256;

# create and configure node
my $from_n = new Search::Estraier::Node(
        url => $from,
        croak_on_error => 1,
        debug => $debug,
        user => 'admin',
        passwd => 'admin',
);
my $to_n = new Search::Estraier::Node(
        url => $to,
        croak_on_error => 1,
        debug => $debug,
        user => 'admin',
        passwd => 'admin',
);

unless(eval{ $to_n->name }) {
        if ($to =~ m#^(http://.+)/node/([^/]+)$#) {
                my ($url,$name) = ($1,$2);
                print "Creating '$name' on $url\n";
                $to_n->shuttle_url( $url . '/master?action=nodeadd',
                        'application/x-www-form-urlencoded',
                        'name=' . uri_escape($name) . '&label=' . uri_escape( $name ),
                        undef,
                );
        } else {
                die "can't extract node name from $to\n";
        }
}

print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";

my $doc_num = $from_n->doc_num || 1;

my $prev;
my $i = 0;
my $more = 1;

my $t = time();

while($more) {
        my $res;
        $from_n->shuttle_url( $from_n->{url} . '/list',
                'application/x-www-form-urlencoded',
                'max=' . $max . ( $prev ? '&prev=' . uri_escape( $prev ) : '' ),
                \$res,
        );
        if (! $res || $res eq '') {
                $more = 0;
                last;
        }
        foreach my $l (split(/\n/,$res)) {
                (my $id, $prev) = split(/\t/,$l, 2);

                #$to_n->put_doc( $from_n->get_doc( $id ));

                my $doc_draft = $from_n->_fetch_doc( id => $id, chomp_resbody => 1 );
                $to_n->shuttle_url( $to_n->{url} . '/put_doc', 'text/x-estraier-draft', $doc_draft, undef) == 200 or die "can't insert $doc_draft\n";

                $i++;
        }
        warn "$prev\n" if ($debug);

        my $rate = ( $i / (time() - $t) );
        printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",
                $i,
                ($i * 100 / $doc_num),
                $rate, 
                strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),
        );

}

print "Copy completed.\n";

1	dpavlin	80	#!/usr/bin/perl -w
2
3			use strict;
4			use Search::Estraier;
5			use URI::Escape qw/uri_escape/;
6	dpavlin	82	use Time::HiRes;
7			use POSIX qw/strftime/;
8	dpavlin	80
9			=head1 NAME
10
11			estcp.pl - copy Hyper Estraier index from one node to another
12
13			=cut
14
15			my ($from,$to) = @ARGV;
16
17			die "usage: $0 http://localhost:1978/node/from http://remote.example.com:1978/node/to\n" unless ($from && $to);
18
19			my $debug = 0;
20	dpavlin	83	my $max = 256;
21	dpavlin	80
22			# create and configure node
23			my $from_n = new Search::Estraier::Node(
24			url => $from,
25			croak_on_error => 1,
26			debug => $debug,
27	dpavlin	133	user => 'admin',
28			passwd => 'admin',
29	dpavlin	80	);
30			my $to_n = new Search::Estraier::Node(
31			url => $to,
32			croak_on_error => 1,
33			debug => $debug,
34	dpavlin	133	user => 'admin',
35			passwd => 'admin',
36	dpavlin	80	);
37
38	dpavlin	85	unless(eval{ $to_n->name }) {
39	dpavlin	88	if ($to =~ m#^(http://.+)/node/([^/]+)$#) {
40	dpavlin	85	my ($url,$name) = ($1,$2);
41			print "Creating '$name' on $url\n";
42			$to_n->shuttle_url( $url . '/master?action=nodeadd',
43			'application/x-www-form-urlencoded',
44			'name=' . uri_escape($name) . '&label=' . uri_escape( $name ),
45			undef,
46			);
47			} else {
48			die "can't extract node name from $to\n";
49			}
50			}
51
52	dpavlin	80	print "Copy from ",$from_n->name," (",$from_n->label,") to ",$to_n->name," (",$to_n->label,") - ",$from_n->doc_num," documents (",$from_n->word_num," words, ",$from_n->size," bytes)\n";
53
54	dpavlin	82	my $doc_num = $from_n->doc_num \|\| 1;
55	dpavlin	80
56			my $prev;
57	dpavlin	83	my $i = 0;
58			my $more = 1;
59	dpavlin	80
60	dpavlin	82	my $t = time();
61
62	dpavlin	83	while($more) {
63			my $res;
64	dpavlin	80	$from_n->shuttle_url( $from_n->{url} . '/list',
65			'application/x-www-form-urlencoded',
66	dpavlin	83	'max=' . $max . ( $prev ? '&prev=' . uri_escape( $prev ) : '' ),
67	dpavlin	80	\$res,
68			);
69	dpavlin	83	if (! $res \|\| $res eq '') {
70			$more = 0;
71			last;
72			}
73	dpavlin	80	foreach my $l (split(/\n/,$res)) {
74			(my $id, $prev) = split(/\t/,$l, 2);
75	dpavlin	84
76			#$to_n->put_doc( $from_n->get_doc( $id ));
77
78			my $doc_draft = $from_n->_fetch_doc( id => $id, chomp_resbody => 1 );
79			$to_n->shuttle_url( $to_n->{url} . '/put_doc', 'text/x-estraier-draft', $doc_draft, undef) == 200 or die "can't insert $doc_draft\n";
80
81	dpavlin	82	$i++;
82	dpavlin	80	}
83	dpavlin	88	warn "$prev\n" if ($debug);
84	dpavlin	82
85			my $rate = ( $i / (time() - $t) );
86			printf("%d records, %1.2f%% [%1.2f rec/s] estimated finish: %s\n",
87			$i,
88			($i * 100 / $doc_num),
89			$rate,
90			strftime("%Y-%m-%d %H:%M:%S", localtime( time() + int(($doc_num-$i) / $rate))),
91			);
92
93	dpavlin	83	}
94	dpavlin	80
95	dpavlin	83	print "Copy completed.\n";
96