/[Sack]/trunk/bin/sack.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /trunk/bin/sack.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 59 - (hide annotations)
Fri Sep 25 12:24:42 2009 UTC (14 years, 8 months ago) by dpavlin
File MIME type: text/plain
File size: 7986 byte(s)
print merge ticks
1 dpavlin 1 #!/usr/bin/perl
2    
3     use warnings;
4     use strict;
5    
6 dpavlin 50 our $VERSION = '0.04';
7 dpavlin 21
8 dpavlin 1 use Time::HiRes qw(time);
9     use Data::Dump qw(dump);
10     use File::Slurp;
11 dpavlin 4 use Getopt::Long;
12 dpavlin 11 use IO::Socket::INET;
13 dpavlin 30 use Storable qw/freeze thaw store/;
14 dpavlin 1
15 dpavlin 4
16 dpavlin 19 my $debug = 0;
17 dpavlin 4 my $path = '/data/isi/full.txt';
18 dpavlin 11 my $limit = 5000;
19 dpavlin 4 my $offset = 0;
20 dpavlin 6 my @views;
21 dpavlin 30 my $port = 0; # interactive
22 dpavlin 11 my @nodes;
23 dpavlin 4
24    
25     GetOptions(
26     'path=s' => \$path,
27     'offset=i' => \$offset,
28     'limit=i' => \$limit,
29 dpavlin 6 'view=s' => \@views,
30 dpavlin 30 'listen|port=i' => \$port,
31 dpavlin 11 'connect=s' => \@nodes,
32 dpavlin 19 'debug!' => \$debug,
33 dpavlin 5 ) or die $!;
34 dpavlin 4
35 dpavlin 1 my $t = time;
36    
37 dpavlin 12
38 dpavlin 21 sub send_nodes;
39    
40 dpavlin 12 our $prefix;
41 dpavlin 21 sub BEGIN {
42 dpavlin 12 $prefix = $0;
43 dpavlin 30 if ( $prefix !~ m{^/} ) {
44 dpavlin 12 chomp( my $pwd = `pwd` );
45     $prefix = "$pwd/$prefix";
46     }
47 dpavlin 51 $prefix =~ s{^(.*)/srv/Sack/.+$}{$1};
48 dpavlin 12 warn "# prefix $prefix";
49 dpavlin 21
50     $SIG{INT} = sub {
51     my $signame = shift;
52     send_nodes 'exit';
53 dpavlin 33 #clean if $clean; # FIXME
54 dpavlin 21 die "SIG$signame";
55     };
56 dpavlin 12 }
57    
58 dpavlin 33 use lib "$prefix/srv/Sack/lib/";
59     use Sack::Digest;
60 dpavlin 45 our $digest = Sack::Digest->new( port => $port, clean => 1 );
61     sub digest { $digest->to_int($_[0]) }
62 dpavlin 12
63     use lib "$prefix/srv/webpac2/lib/";
64 dpavlin 1 use WebPAC::Input::ISI;
65 dpavlin 24
66     $WebPAC::Input::ISI::subfields = undef; # disable parsing of subfields
67    
68 dpavlin 1 my $input = WebPAC::Input::ISI->new(
69 dpavlin 12 path => "$prefix/$path",
70 dpavlin 4 offset => $offset,
71     limit => $limit,
72 dpavlin 1 );
73    
74 dpavlin 25 our $num_records = $input->size;
75 dpavlin 1
76     sub report {
77 dpavlin 30 my $description = join(' ',@_);
78 dpavlin 1 my $dt = time - $t;
79     printf "%s in %1.4fs %.2f/s\n", $description, $dt, $input->size / $dt;
80 dpavlin 10 $t = time;
81 dpavlin 1 }
82    
83    
84     report $input->size . ' records loaded';
85    
86     mkdir 'out' unless -e 'out';
87    
88 dpavlin 5 our $out;
89    
90 dpavlin 8 our $cache;
91    
92 dpavlin 13 our $connected;
93    
94 dpavlin 11 sub send_nodes {
95 dpavlin 17 my $content = $#_ > 0 ? pop @_ : ''; # no content with just one argument!
96 dpavlin 21 my $header = defined $content ? length($content) : 0;
97 dpavlin 11 $header .= ' ' . join(' ', @_) if @_;
98    
99     foreach my $node ( @nodes ) {
100    
101     my $sock = IO::Socket::INET->new(
102 dpavlin 56 PeerAddr => '127.0.0.1',
103     PeerPort => $node,
104 dpavlin 11 Proto => 'tcp',
105 dpavlin 16 );
106 dpavlin 11
107 dpavlin 16 if ( ! $sock ) {
108     warn "can't connect to $node - $!"; # FIXME die?
109     next;
110     }
111    
112 dpavlin 30 warn "[$port] >>>> $node $header\n";
113 dpavlin 11 print $sock "$header\n$content" || warn "can't send $header to $node: $!";
114    
115 dpavlin 13 $connected->{$node} = $sock;
116 dpavlin 11 }
117     }
118    
119 dpavlin 16 sub get_node {
120     my $node = shift;
121    
122     my $sock = $connected->{$node};
123     if ( ! $sock ) {
124 dpavlin 30 warn "[$port] ERROR lost connection to $node";
125 dpavlin 16 delete $connected->{$node};
126     return;
127     }
128     chomp( my $size = <$sock> );
129 dpavlin 58 warn "[$port] <<<< $node $size bytes\n" if $debug || $size > 1024;
130 dpavlin 16 my $data;
131     read $sock, $data, $size;
132     return $data;
133     }
134    
135     sub send_sock {
136     my ( $sock, $data ) = @_;
137     my $size = length $data;
138 dpavlin 58 warn "[$port] >>>> $size bytes\n" if $debug || $size > 1024;
139 dpavlin 16 print $sock "$size\n$data" || warn "can't send $size bytes to ", $sock->peerhost;
140     }
141    
142 dpavlin 13 sub merge_out {
143 dpavlin 50 my ( $from_node, $new ) = @_;
144 dpavlin 13
145 dpavlin 59 my $remote_digest = Sack::Digest->new( port => $from_node );
146     my ( $local, $remote ) = ( 0, 0 );
147 dpavlin 50
148 dpavlin 59 my $tick = 0;
149     print STDERR "[$port] merge $from_node";
150 dpavlin 50
151 dpavlin 13 foreach my $k1 ( keys %$new ) {
152    
153     foreach my $k2 ( keys %{ $new->{$k1} } ) {
154    
155 dpavlin 30 my $n = delete $new->{$k1}->{$k2};
156 dpavlin 50
157     if ( $k1 =~ m{#} ) {
158     die "ASSERT $k1 $k2" unless $k2 =~ m{^\d+$};
159     #warn "XXX $k1 $k2";
160 dpavlin 59 my $md5 = $remote_digest->{nr_md5}->[$k2] || warn "[$port] no2md5 $n not found in $from_node\n";
161 dpavlin 50 if ( my $local_k2 = $digest->{md5_nr}->{$md5} ) {
162     $k2 = $local_k2;
163     $local++;
164     } else {
165     $k2 = $digest->to_int( $remote_digest->{md5}->{$md5} );
166     $remote++;
167     }
168     }
169    
170 dpavlin 30 my $ref = ref $out->{$k1}->{$k2};
171 dpavlin 50 #warn "XXXX $k1 $k2 $ref";
172 dpavlin 13 if ( ! defined $out->{$k1}->{$k2} ) {
173     $out->{$k1}->{$k2} = $n;
174     } elsif ( $k1 =~ m{\+} ) {
175 dpavlin 19 # warn "## agregate $k1 $k2";
176 dpavlin 13 $out->{$k1}->{$k2} += $n;
177 dpavlin 21 } elsif ( $ref eq 'ARRAY' ) {
178     if ( ref $n eq 'ARRAY' ) {
179     push @{ $out->{$k1}->{$k2} }, $_ foreach @$n;
180     } else {
181     push @{ $out->{$k1}->{$k2} }, $n;
182     }
183 dpavlin 13 } elsif ( $ref eq '' ) {
184     $out->{$k1}->{$k2} = [ $out->{$k1}->{$k2}, $n ];
185     } else {
186     die "can't merge $k2 [$ref] from ",dump($n), " into ", dump($out->{$k1}->{$k2});
187     }
188 dpavlin 59
189     if ( $tick++ % 1000 == 0 ) {
190     print STDERR ".";
191     } elsif ( $tick % 10000 == 0 ) {
192     print STDERR $tick;
193     }
194 dpavlin 13 }
195     }
196    
197 dpavlin 59 print STDERR "$tick\n";
198    
199     warn "[$port] merge local $local remote $remote from $from_node\n";
200 dpavlin 19 warn "## merge out ", dump $out if $debug;
201 dpavlin 13 }
202    
203 dpavlin 11 sub run_code {
204     my ( $view, $code ) = @_;
205    
206 dpavlin 21 warn "\n#### CODE $view START ####\n$code\n#### CODE $view END ####\n" if $debug;
207 dpavlin 11
208     send_nodes view => $view => $code;
209    
210     undef $out;
211    
212     my $affected = 0;
213     $t = time;
214    
215     foreach my $pos ( $offset + 1 .. $offset + $input->size ) {
216     my $rec = $cache->{$pos} ||= $input->fetch_rec( $pos );
217     if ( ! $rec ) {
218 dpavlin 41 print STDERR "END @ $pos";
219 dpavlin 11 last;
220     }
221    
222     eval "$code";
223     if ( $@ ) {
224 dpavlin 41 warn "ABORT $pos $@\n";
225 dpavlin 20 last;
226 dpavlin 11 } else {
227     $affected++;
228     }
229 dpavlin 30
230 dpavlin 40 $pos % 10000 == 0 ? print STDERR $pos :
231     $pos % 1000 == 0 ? print STDERR "." : 0 ;
232 dpavlin 11 };
233    
234 dpavlin 30 report "\n[$port] RECS $affected $view";
235 dpavlin 11
236     warn "WARN no \$out defined!" unless defined $out;
237 dpavlin 13
238 dpavlin 46 $digest->sync;
239    
240 dpavlin 13 if ( $connected ) {
241 dpavlin 24 foreach my $node ( keys %$connected ) {
242 dpavlin 30 warn "[$port] get_node $node\n";
243 dpavlin 24 my $o = get_node $node;
244     my $s = length $o;
245     $o = thaw $o;
246 dpavlin 30 warn "[$port] merge $node $s bytes\n";
247 dpavlin 50 merge_out $node => $o;
248 dpavlin 24 }
249 dpavlin 13 }
250 dpavlin 11 }
251    
252 dpavlin 1 sub run_views {
253 dpavlin 6 @views = sort glob 'views/*.pl' unless @views;
254 dpavlin 1 warn "# views ", dump @views;
255    
256     foreach my $view ( @views ) {
257    
258     next if system("perl -c $view") != 0;
259    
260     my $code = read_file $view;
261    
262 dpavlin 11 run_code $view => $code;
263 dpavlin 1
264 dpavlin 11 if ( defined $out ) {
265 dpavlin 1
266 dpavlin 11 my $path = $view;
267     $path =~ s{views?/}{out/} || die "no view in $view";
268 dpavlin 30 $path =~ s{\.pl}{.storable};
269 dpavlin 1
270 dpavlin 9 unlink "$path.last" if -e "$path.last";
271     rename $path, "$path.last";
272 dpavlin 30
273     store $out => $path;
274 dpavlin 31 report "[$port] SAVE $path $offset-$limit", -s $path, "bytes";
275 dpavlin 35
276     if ( -s $path < 4096 ) {
277 dpavlin 45 print '$out = ', dump $digest->undigest_out($out);
278 dpavlin 35 }
279 dpavlin 11 }
280    
281     }
282    
283     }
284    
285 dpavlin 35
286     sub info_tabs {
287     "$port\t$offset\t$limit\t$num_records\t$path\t"
288 dpavlin 36 . join("\t", map {
289 dpavlin 35 my $b = $_;
290 dpavlin 46 $b =~ s{^.+\.$port\.([^/]+)$}{$1};
291 dpavlin 35 "$b " . -s $_
292 dpavlin 46 } glob "/dev/shm/sack.$port.*" );
293 dpavlin 35 }
294    
295    
296 dpavlin 30 if ( $port ) {
297 dpavlin 11 my $sock = IO::Socket::INET->new(
298     Listen => SOMAXCONN,
299 dpavlin 21 LocalAddr => '127.0.0.1',
300 dpavlin 30 LocalPort => $port,
301 dpavlin 11 Proto => 'tcp',
302     Reuse => 1,
303     ) or die $!;
304    
305     while (1) {
306    
307 dpavlin 30 warn "[$port] READY path: $path offset: $offset limit: $limit #recs: $num_records\n";
308 dpavlin 11
309     my $client = $sock->accept();
310    
311 dpavlin 30 warn "[$port] <<<< connect from ", $client->peerhost, $/;
312 dpavlin 11
313     my @header = split(/\s/, <$client>);
314 dpavlin 30 warn "[$port] <<<< header ",dump(@header),$/;
315 dpavlin 11
316     my $size = shift @header;
317    
318     my $content;
319     read $client, $content, $size;
320    
321     if ( $header[0] eq 'view' ) {
322     run_code $header[1] => $content;
323 dpavlin 16 send_sock $client => freeze $out;
324     } elsif ( $header[0] eq 'info' ) {
325 dpavlin 35 my $info = info_tabs;
326 dpavlin 30 warn "[$port] info $info\n";
327 dpavlin 16 send_sock $client => $info;
328     } elsif ( $header[0] eq 'exit' ) {
329 dpavlin 30 warn "[$port] exit";
330 dpavlin 16 exit;
331 dpavlin 5 } else {
332 dpavlin 30 warn "[$port] UNKNOWN $header[0]";
333 dpavlin 1 }
334    
335     }
336     }
337    
338 dpavlin 30 sub info {
339     send_nodes 'info' => $2;
340    
341     my @info = (
342 dpavlin 36 "port\toffset\tlimit\t#recs\tpath",
343     "----\t------\t-----\t-----\t----",
344 dpavlin 35 info_tabs,
345 dpavlin 30 );
346    
347     push @info, get_node $_ foreach @nodes;
348    
349     print "[$port] INFO\n"
350     , join("\n", @info)
351     , "\n\n" ;
352    
353     return @info;
354     }
355    
356     info;
357 dpavlin 3 run_views;
358    
359 dpavlin 1 while ( 1 ) {
360    
361     print "sack> ";
362 dpavlin 21 chomp( my $cmd = <STDIN> );
363 dpavlin 1
364 dpavlin 21 if ( $cmd =~ m{^(h|\?)} ) {
365     print << "__HELP__"
366     Sacks Lorry v$VERSION - path: $path offset: $offset limit: $limit
367    
368     View Run run views
369     VI \\e Output show output of last run
370     Info [\$VERSION] instrospect
371     Quit EXit shutdown
372    
373     __HELP__
374     } elsif ( $cmd =~ m{^(vi|\\e|o)}i ) {
375 dpavlin 33 #system "vi out/*";
376 dpavlin 53 $digest->sync;
377 dpavlin 33 system "bin/storableedit.pl", (glob('out/*.storable'))[0];
378 dpavlin 21 } elsif ( $cmd =~ m{^i(?:nfo)?\s?(.+)?$}i ) {
379 dpavlin 30 info;
380 dpavlin 21 } elsif ( $cmd =~ m{^(q|e|x)}i ) {
381 dpavlin 16 warn "# exit";
382     send_nodes 'exit';
383     exit;
384 dpavlin 21 } elsif ( $cmd =~ m{^(v|r)}i ) {
385 dpavlin 3 run_views;
386 dpavlin 55 } elsif ( $cmd =~ m{^n(ode)?\s*(\d+)}i ) {
387     push @nodes, $1;
388     info;
389 dpavlin 21 } elsif ( $cmd ) {
390     warn "UNKNOWN ", dump $cmd;
391 dpavlin 3 }
392    
393 dpavlin 1 }
394    

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26