/[Sack]/trunk/lib/Sack/Shards.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/lib/Sack/Shards.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 185 by dpavlin, Sun Nov 8 13:11:38 2009 UTC revision 186 by dpavlin, Sun Nov 8 13:21:01 2009 UTC
# Line 1  Line 1 
1  package Sack::From;  package Sack::Shards;
2    
3  use warnings;  use warnings;
4  use strict;  use strict;
5    
6    use Storable;
7  use Data::Dump qw(dump);  use Data::Dump qw(dump);
8    
9    use lib '/srv/Sack/lib';
10    use Sack;
11    
12  use lib "/srv/webpac2/lib/";  use lib "/srv/webpac2/lib/";
13  use WebPAC::Input::ISI;  use WebPAC::Input::ISI;
14    
15  $WebPAC::Input::ISI::subfields = undef; # disable parsing of subfields  $WebPAC::Input::ISI::subfields = undef; # disable parsing of subfields
16    
 use Storable;  
   
17  sub new {  sub new {
18      my $class = shift;      my $class = shift;
19      my $self  = bless {@_}, $class;          my $args = {@_};
20        my $self  = bless $args, $class;
21            my $limit = delete $args->{limit} || die "no shard size limit?";
22    
23          warn "# from ",dump( @_ );          duration 'sharding',dump $args;
24    
25          $self->{input} = WebPAC::Input::ISI->new( @_ );          $self->{input} = WebPAC::Input::ISI->new( %$args );
26          warn "got ", $self->size, " records for ", dump(@_), $/;  
27            duration "got ", $self->size, " records";
28    
29          $self->{pos} = 1;          $self->{pos} = 1;
30            my $offset = 0;
31    
32            while ( $offset <= $self->size ) {
33                    $self->shard( $offset, $limit );
34                    $offset += $limit;
35            }
36    
37          return $self;          return $self;
38  }  }
# Line 54  sub shard { Line 65  sub shard {
65          store $data, $path;          store $data, $path;
66          warn "store $path ", -s $path, " bytes\n";          warn "store $path ", -s $path, " bytes\n";
67    
68            duration "shard", $path;
69    
70          return $data;          return $data;
71  }  }
72    
Legend:
Removed from v.185  
changed lines
  Added in v.186
  ViewVC Help
Powered by ViewVC 1.1.26