trunk/bin/lorry.pl

#!/usr/bin/perl

use warnings;
use strict;

use Data::Dump qw(dump);
use File::Slurp;

use lib 'lib';
use Sack::From;
use Sack::Lorry;
use Sack::Color;
use Getopt::Long;
use Time::HiRes qw(time);


my $view   = 'views/00.demo.pl';
my $offset = 0;
my $limit  = 5000;
my $cloud_file;


GetOptions(
        'view=s'  => \$view,
        'limit=i' => \$limit,
        'cloud=s' => \$cloud_file,
) or die $!;

my $t = time;
sub duration {
        my $now = time;
        my $d = $now - $t;
        warn sprintf "%.4fs %s\n", $d, join(' ',@_);
        $t = $now;
}


my $lorry = Sack::Lorry->new;

my @cloud = -e $cloud_file ? read_file $cloud_file : ( 'localhost' );
warn "bring up cloud ",dump @cloud;

my $from  = Sack::From->new(
        path => '/data/isi/full.txt',
        limit => $limit * scalar @cloud,
);

duration 'load finished';

my $info;

foreach my $host ( @cloud ) {

        if ( my $port = $lorry->start_node( $host ) ) {

                warn "started [$port] on $host\n";

                my $data = $from->shard( $limit );
                $lorry->send_to( $port, { data => $data } ) || die "can't send to $port: $!";

                $info->{$port} = {
                        host => $host,
                        offset => $offset,
                        limit => $limit,
                };

                $offset += $limit;

        }
}

duration 'cloud up';

warn "info ",dump $info;

warn "load status ", dump( $lorry->get_from_all );

our $out;

while ( $view ) {
        print "sack> ";
        my $cmd = <STDIN>;
        last unless defined $cmd; # CTRL+D
        duration 'repl wait';
        chomp($cmd);
        if ( $cmd =~ m{^v} ) {
                $out = $lorry->view( $view );
                duration 'view';
        } elsif ( $cmd =~ m{^d} ) {
                warn dump $out;
                duration 'dump';
        } else {
                warn "unknown $cmd\n";
        }
}

warn "exit all nodes\n";
$lorry->send_to_all( { exit => 1 } );

duration 'total usage';

1	dpavlin	92	#!/usr/bin/perl
2
3			use warnings;
4			use strict;
5
6			use Data::Dump qw(dump);
7	dpavlin	93	use File::Slurp;
8	dpavlin	92
9			use lib 'lib';
10			use Sack::From;
11			use Sack::Lorry;
12	dpavlin	102	use Sack::Color;
13	dpavlin	107	use Getopt::Long;
14			use Time::HiRes qw(time);
15	dpavlin	92
16	dpavlin	107
17	dpavlin	93	my $view = 'views/00.demo.pl';
18	dpavlin	92	my $offset = 0;
19	dpavlin	100	my $limit = 5000;
20	dpavlin	108	my $cloud_file;
21	dpavlin	92
22
23	dpavlin	107	GetOptions(
24			'view=s' => \$view,
25			'limit=i' => \$limit,
26			'cloud=s' => \$cloud_file,
27			) or die $!;
28
29			my $t = time;
30			sub duration {
31			my $now = time;
32			my $d = $now - $t;
33			warn sprintf "%.4fs %s\n", $d, join(' ',@_);
34			$t = $now;
35			}
36
37
38	dpavlin	92	my $lorry = Sack::Lorry->new;
39
40	dpavlin	108	my @cloud = -e $cloud_file ? read_file $cloud_file : ( 'localhost' );
41	dpavlin	107	warn "bring up cloud ",dump @cloud;
42
43	dpavlin	95	my $from = Sack::From->new(
44			path => '/data/isi/full.txt',
45			limit => $limit * scalar @cloud,
46			);
47
48	dpavlin	107	duration 'load finished';
49
50	dpavlin	92	my $info;
51
52			foreach my $host ( @cloud ) {
53	dpavlin	101
54	dpavlin	92	if ( my $port = $lorry->start_node( $host ) ) {
55
56			warn "started [$port] on $host\n";
57
58	dpavlin	95	my $data = $from->shard( $limit );
59			$lorry->send_to( $port, { data => $data } ) \|\| die "can't send to $port: $!";
60	dpavlin	92
61			$info->{$port} = {
62			host => $host,
63			offset => $offset,
64			limit => $limit,
65			};
66
67			$offset += $limit;
68
69			}
70			}
71
72	dpavlin	107	duration 'cloud up';
73
74	dpavlin	92	warn "info ",dump $info;
75
76	dpavlin	93	warn "load status ", dump( $lorry->get_from_all );
77	dpavlin	92
78	dpavlin	110	our $out;
79
80	dpavlin	100	while ( $view ) {
81			print "sack> ";
82			my $cmd = <STDIN>;
83			last unless defined $cmd; # CTRL+D
84	dpavlin	110	duration 'repl wait';
85	dpavlin	100	chomp($cmd);
86	dpavlin	110	if ( $cmd =~ m{^v} ) {
87			$out = $lorry->view( $view );
88			duration 'view';
89			} elsif ( $cmd =~ m{^d} ) {
90			warn dump $out;
91			duration 'dump';
92			} else {
93			warn "unknown $cmd\n";
94			}
95	dpavlin	100	}
96	dpavlin	93
97	dpavlin	92	warn "exit all nodes\n";
98	dpavlin	93	$lorry->send_to_all( { exit => 1 } );
99
100	dpavlin	107	duration 'total usage';
101