trunk/bin/couchdb2shards.pl

#!/usr/bin/perl

# http://wiki.apache.org/couchdb/HTTP_Bulk_Document_API

use lib 'lib';
use Sack::Shard;

use IO::Socket::INET;
use JSON;
use Data::Dump qw(dump);
use autodie;

my $name = 'pxelator';
my $shard_size = 5000;


sub couchdb_socket {
        IO::Socket::INET->new(
                PeerAddr => '10.60.0.91',
                PeerPort => 5984,
                Proto => 'tcp',
        ) || die $!;
}

sub get_chunk {
        my $sock = shift;
        my $chunk;
        while(<$sock>) {
                $chunk .= $_;
                last if /^[\n\r]+$/;
        }
#       warn "# $sock\n$chunk\n";
        return $chunk;
}

my $sock = couchdb_socket;

print $sock "GET /$name/_all_docs?include_docs=true HTTP/1.0\r\n\r\n";

get_chunk($sock);

my $total = <$sock>;
$total =~ s{^.*total_rows\D+(\d+).+$}{$1};
warn "# $name total: $total\n";

Sack::Shard::create( $name, $total, $shard_size );

while(<$sock>) {
        if ( /"id":"([^"]+)"/ ) {

                s/,[\r\n]+$//; # cleanup JSON
                my $json = from_json( $_ );
                Sack::Shard::add( $json->{doc} );
        } else {
                warn "UNKNOWN: $_";
        }
}
Sack::Shard::finish;

1	#!/usr/bin/perl
2
3	# http://wiki.apache.org/couchdb/HTTP_Bulk_Document_API
4
5	use lib 'lib';
6	use Sack::Shard;
7
8	use IO::Socket::INET;
9	use JSON;
10	use Data::Dump qw(dump);
11	use autodie;
12
13	my $name = 'pxelator';
14	my $shard_size = 5000;
15
16
17	sub couchdb_socket {
18	IO::Socket::INET->new(
19	PeerAddr => '10.60.0.91',
20	PeerPort => 5984,
21	Proto => 'tcp',
22	) \|\| die $!;
23	}
24
25	sub get_chunk {
26	my $sock = shift;
27	my $chunk;
28	while(<$sock>) {
29	$chunk .= $_;
30	last if /^[\n\r]+$/;
31	}
32	# warn "# $sock\n$chunk\n";
33	return $chunk;
34	}
35
36	my $sock = couchdb_socket;
37
38	print $sock "GET /$name/_all_docs?include_docs=true HTTP/1.0\r\n\r\n";
39
40	get_chunk($sock);
41
42	my $total = <$sock>;
43	$total =~ s{^.*total_rows\D+(\d+).+$}{$1};
44	warn "# $name total: $total\n";
45
46	Sack::Shard::create( $name, $total, $shard_size );
47
48	while(<$sock>) {
49	if ( /"id":"([^"]+)"/ ) {
50
51	s/,[\r\n]+$//; # cleanup JSON
52	my $json = from_json( $_ );
53	Sack::Shard::add( $json->{doc} );
54	} else {
55	warn "UNKNOWN: $_";
56	}
57	}
58	Sack::Shard::finish;
59