trunk/bin/sack.pl

#!/usr/bin/perl

use warnings;
use strict;

use Time::HiRes qw(time);
use Data::Dump qw(dump);
use File::Slurp;
use Getopt::Long;


my $path   = '/data/isi/full.txt';
my $limit  = 10000;
my $offset = 0; 


GetOptions(
        'path=s'   => \$path,
        'offset=i' => \$offset,
        'limit=i'  => \$limit,
) or die $!;

my $t = time;

use lib '/srv/webpac2/lib/';
use WebPAC::Input::ISI;
my $input = WebPAC::Input::ISI->new(
        path   => $path,
        offset => $offset,
        limit  => $limit,
);


sub report {
        my $description = shift;
        my $dt = time - $t;
        printf "%s in %1.4fs %.2f/s\n", $description, $dt, $input->size / $dt;
}


report $input->size . ' records loaded';

mkdir 'out' unless -e 'out';

our $out;

sub run_views {
        my @views = glob 'views/*.pl';
        warn "# views ", dump @views;

        foreach my $view ( @views ) {

                my ( $nr, $package ) = ( $1, $2 )
                        if $view =~ m{/(\d+)\.([^/]+(\.pl)?$)};

                undef $out;

                next if system("perl -c $view") != 0;

                my $code = read_file $view;
                warn "## CODE\n$code\n## CODE\n";

                my $affected = 0;
                $t = time;

                foreach my $pos ( $offset + 1 .. $input->size ) {
                        my $rec = $input->fetch_rec($pos);
                        if ( ! $rec ) {
                                warn "END at $pos";
                                last;
                        }

                        eval "$code";
                        if ( $@ ) {
                                warn "ERROR [$pos] $@\n";
                        } else {
                                $affected++;
                        }
                };

                report "$affected affected records $view";

                if ( defined $out ) {
                        my $dump = dump $out;
                        my $len  = length $dump;
                        my $path = "out/$nr.$package";
                        print "out $view $offset/$limit $len bytes $path"
                                , ( $len < 10000 ?  " \$out = $dump" : ' SAVED ONLY' )
                                , "\n"
                                ;

                        write_file $path, $dump;
                        report "save $path";
                } else {
                        warn "W: no \$out defined!";
                }

        }
}

run_views;

while ( 1 ) {

        print "sack> ";
        my $cmd = <STDIN>;

        if ( $cmd =~ m{(vi|\\e|out)}i ) {
                system "vi out/*";
        } else {
                run_views;
        }

}

1	#!/usr/bin/perl
2
3	use warnings;
4	use strict;
5
6	use Time::HiRes qw(time);
7	use Data::Dump qw(dump);
8	use File::Slurp;
9	use Getopt::Long;
10
11
12	my $path = '/data/isi/full.txt';
13	my $limit = 10000;
14	my $offset = 0;
15
16
17	GetOptions(
18	'path=s' => \$path,
19	'offset=i' => \$offset,
20	'limit=i' => \$limit,
21	) or die $!;
22
23	my $t = time;
24
25	use lib '/srv/webpac2/lib/';
26	use WebPAC::Input::ISI;
27	my $input = WebPAC::Input::ISI->new(
28	path => $path,
29	offset => $offset,
30	limit => $limit,
31	);
32
33
34	sub report {
35	my $description = shift;
36	my $dt = time - $t;
37	printf "%s in %1.4fs %.2f/s\n", $description, $dt, $input->size / $dt;
38	}
39
40
41	report $input->size . ' records loaded';
42
43	mkdir 'out' unless -e 'out';
44
45	our $out;
46
47	sub run_views {
48	my @views = glob 'views/*.pl';
49	warn "# views ", dump @views;
50
51	foreach my $view ( @views ) {
52
53	my ( $nr, $package ) = ( $1, $2 )
54	if $view =~ m{/(\d+)\.([^/]+(\.pl)?$)};
55
56	undef $out;
57
58	next if system("perl -c $view") != 0;
59
60	my $code = read_file $view;
61	warn "## CODE\n$code\n## CODE\n";
62
63	my $affected = 0;
64	$t = time;
65
66	foreach my $pos ( $offset + 1 .. $input->size ) {
67	my $rec = $input->fetch_rec($pos);
68	if ( ! $rec ) {
69	warn "END at $pos";
70	last;
71	}
72
73	eval "$code";
74	if ( $@ ) {
75	warn "ERROR [$pos] $@\n";
76	} else {
77	$affected++;
78	}
79	};
80
81	report "$affected affected records $view";
82
83	if ( defined $out ) {
84	my $dump = dump $out;
85	my $len = length $dump;
86	my $path = "out/$nr.$package";
87	print "out $view $offset/$limit $len bytes $path"
88	, ( $len < 10000 ? " \$out = $dump" : ' SAVED ONLY' )
89	, "\n"
90	;
91
92	write_file $path, $dump;
93	report "save $path";
94	} else {
95	warn "W: no \$out defined!";
96	}
97
98	}
99	}
100
101	run_views;
102
103	while ( 1 ) {
104
105	print "sack> ";
106	my $cmd = <STDIN>;
107
108	if ( $cmd =~ m{(vi\|\\e\|out)}i ) {
109	system "vi out/*";
110	} else {
111	run_views;
112	}
113
114	}
115