/[webpac2]/trunk/lib/WebPAC/Input.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/lib/WebPAC/Input.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 760 by dpavlin, Wed Oct 25 15:56:44 2006 UTC revision 761 by dpavlin, Wed Oct 25 17:10:08 2006 UTC
# Line 16  WebPAC::Input - read different file form Line 16  WebPAC::Input - read different file form
16    
17  =head1 VERSION  =head1 VERSION
18    
19  Version 0.13  Version 0.14
20    
21  =cut  =cut
22    
23  our $VERSION = '0.13';  our $VERSION = '0.14';
24    
25  =head1 SYNOPSIS  =head1 SYNOPSIS
26    
# Line 43  Perhaps a little code snippet. Line 43  Perhaps a little code snippet.
43    
44          my $db = WebPAC::Input->new(          my $db = WebPAC::Input->new(
45                  module => 'WebPAC::Input::ISIS',                  module => 'WebPAC::Input::ISIS',
                 low_mem => 1,  
46          );          );
47    
48          $db->open( path => '/path/to/database' );          $db->open( path => '/path/to/database' );
# Line 63  Create new input database object. Line 62  Create new input database object.
62    my $db = new WebPAC::Input(    my $db = new WebPAC::Input(
63          module => 'WebPAC::Input::MARC',          module => 'WebPAC::Input::MARC',
64          encoding => 'ISO-8859-2',          encoding => 'ISO-8859-2',
         low_mem => 1,  
65          recode => 'char pairs',          recode => 'char pairs',
66          no_progress_bar => 1,          no_progress_bar => 1,
67    );    );
# Line 75  Optional parametar C<encoding> specify a Line 73  Optional parametar C<encoding> specify a
73  used internally). This should probably be your terminal encoding, and by  used internally). This should probably be your terminal encoding, and by
74  default, it C<ISO-8859-2>.  default, it C<ISO-8859-2>.
75    
 Default is not to use C<low_mem> options (see L<MEMORY USAGE> below).  
   
76  C<recode> is optional string constisting of character or words pairs that  C<recode> is optional string constisting of character or words pairs that
77  should be replaced in input stream.  should be replaced in input stream.
78    
# Line 96  sub new { Line 92  sub new {
92    
93          $log->logconfess("code_page argument is not suppored any more. change it to encoding") if ($self->{lookup});          $log->logconfess("code_page argument is not suppored any more. change it to encoding") if ($self->{lookup});
94          $log->logconfess("lookup argument is not suppored any more. rewrite call to lookup_ref") if ($self->{lookup});          $log->logconfess("lookup argument is not suppored any more. rewrite call to lookup_ref") if ($self->{lookup});
95            $log->logconfess("low_mem argument is not suppored any more. rewrite it to load_row and save_row") if ($self->{low_mem});
96    
97          $log->logconfess("specify low-level file format module") unless ($self->{module});          $log->logconfess("specify low-level file format module") unless ($self->{module});
98          my $module_path = $self->{module};          my $module_path = $self->{module};
# Line 119  sub new { Line 116  sub new {
116    
117  This function will read whole database in memory and produce lookups.  This function will read whole database in memory and produce lookups.
118    
119     my $store;     # simple in-memory hash
120    
121   $input->open(   $input->open(
122          path => '/path/to/database/file',          path => '/path/to/database/file',
123          code_page => 'cp852',          code_page => 'cp852',
# Line 134  This function will read whole database i Line 133  This function will read whole database i
133                  901 => { '*' => { '^b' => ' ; ' } },                  901 => { '*' => { '^b' => ' ; ' } },
134          },          },
135          modify_file => 'conf/modify/mapping.map',          modify_file => 'conf/modify/mapping.map',
136            save_row => sub {
137                    my $a = shift;
138                    $store->{ $a->{id} } = $a->{row};
139            },
140            load_row => sub {
141                    my $a = shift;
142                    return defined($store->{ $a->{id} }) &&
143                            $store->{ $a->{id} };
144            },
145    
146   );   );
147    
148  By default, C<code_page> is assumed to be C<cp852>.  By default, C<code_page> is assumed to be C<cp852>.
# Line 154  C<modify_file> is alternative for C<modi Line 163  C<modify_file> is alternative for C<modi
163  (hopefully) simplier sintax than YAML or perl (see L</modify_file_regex>). This option  (hopefully) simplier sintax than YAML or perl (see L</modify_file_regex>). This option
164  overrides C<modify_records> if both exists for same input.  overrides C<modify_records> if both exists for same input.
165    
166    C<save_row> and C<load_row> are low-level implementation of store engine. Calling convention
167    is documented in example above.
168    
169  Returns size of database, regardless of C<offset> and C<limit>  Returns size of database, regardless of C<offset> and C<limit>
170  parametars, see also C<size>.  parametars, see also C<size>.
171    
# Line 180  sub open { Line 192  sub open {
192                  $self->{$v} = $arg->{$v} if ($arg->{$v});                  $self->{$v} = $arg->{$v} if ($arg->{$v});
193          }          }
194    
195            if ($arg->{load_row} || $arg->{save_row}) {
196                    $log->logconfess("save_row and load_row must be defined in pair and be CODE") unless (
197                            ref($arg->{load_row}) eq 'CODE' &&
198                            ref($arg->{save_row}) eq 'CODE'
199                    );
200                    $self->{load_row} = $arg->{load_row};
201                    $self->{save_row} = $arg->{save_row};
202                    $log->debug("using load_row and save_row instead of in-memory hash");
203            }
204    
205          my $filter_ref;          my $filter_ref;
206          my $recode_regex;          my $recode_regex;
207          my $recode_map;          my $recode_map;
# Line 259  sub open { Line 281  sub open {
281    
282          $log->info("processing $self->{size}/$size records [$from_rec-$to_rec] convert $code_page -> $self->{encoding}", $self->{stats} ? ' [stats]' : '');          $log->info("processing $self->{size}/$size records [$from_rec-$to_rec] convert $code_page -> $self->{encoding}", $self->{stats} ? ' [stats]' : '');
283    
         # turn on low_mem for databases with more than 100000 records!  
         if (! $self->{low_mem} && $size > 100000) {  
                 $log->warn("Using on-disk storage instead of memory for input data. This will affect performance.");  
                 $self->{low_mem}++;  
         }  
   
         # running with low_mem flag? well, use DBM::Deep then.  
         if ($self->{'low_mem'}) {  
                 $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");  
   
                 my $db_file = "data.db";  
   
                 if (-e $db_file) {  
                         unlink $db_file or $log->logdie("can't remove '$db_file' from last run");  
                         $log->debug("removed '$db_file' from last run");  
                 }  
   
                 require DBM::Deep;  
   
                 my $db = new DBM::Deep $db_file;  
   
                 $log->logdie("DBM::Deep error: $!") unless ($db);  
   
                 if ($db->error()) {  
                         $log->logdie("can't open '$db_file' under low_mem: ",$db->error());  
                 } else {  
                         $log->debug("using file '$db_file' for DBM::Deep");  
                 }  
   
                 $self->{'db'} = $db;  
         }  
   
284          # read database          # read database
285          for (my $pos = $from_rec; $pos <= $to_rec; $pos++) {          for (my $pos = $from_rec; $pos <= $to_rec; $pos++) {
286    
# Line 333  sub open { Line 323  sub open {
323                  }                  }
324    
325                  # store                  # store
326                  if ($self->{low_mem}) {                  if ($self->{save_row}) {
327                          $self->{db}->put($pos, $rec);                          $self->{save_row}->({
328                                    id => $pos,
329                                    row => $rec,
330                            });
331                  } else {                  } else {
332                          $self->{data}->{$pos} = $rec;                          $self->{data}->{$pos} = $rec;
333                  }                  }
# Line 425  sub fetch { Line 418  sub fetch {
418    
419          my $rec;          my $rec;
420    
421          if ($self->{low_mem}) {          if ($self->{load_row}) {
422                  $rec = $self->{db}->get($mfn);                  $rec = $self->{load_row}->({ id => $mfn });
423          } else {          } else {
424                  $rec = $self->{data}->{$mfn};                  $rec = $self->{data}->{$mfn};
425          }          }
# Line 671  sub modify_file_regexps { Line 664  sub modify_file_regexps {
664          return $regexpes;          return $regexpes;
665  }  }
666    
 =head1 MEMORY USAGE  
   
 C<low_mem> options is double-edged sword. If enabled, WebPAC  
 will run on memory constraint machines (which doesn't have enough  
 physical RAM to create memory structure for whole source database).  
   
 If your machine has 512Mb or more of RAM and database is around 10000 records,  
 memory shouldn't be an issue. If you don't have enough physical RAM, you  
 might consider using virtual memory (if your operating system is handling it  
 well, like on FreeBSD or Linux) instead of dropping to L<DBM::Deep> to handle  
 parsed structure of ISIS database (this is what C<low_mem> option does).  
   
 Hitting swap at end of reading source database is probably o.k. However,  
 hitting swap before 90% will dramatically decrease performance and you will  
 be better off with C<low_mem> and using rest of availble memory for  
 operating system disk cache (Linux is particuallary good about this).  
 However, every access to database record will require disk access, so  
 generation phase will be slower 10-100 times.  
   
 Parsed structures are essential - you just have option to trade RAM memory  
 (which is fast) for disk space (which is slow). Be sure to have planty of  
 disk space if you are using C<low_mem> and thus L<DBM::Deep>.  
   
 However, when WebPAC is running on desktop machines (or laptops :-), it's  
 highly undesireable for system to start swapping. Using C<low_mem> option can  
 reduce WecPAC memory usage to around 64Mb for same database with lookup  
 fields and sorted indexes which stay in RAM. Performance will suffer, but  
 memory usage will really be minimal. It might be also more confortable to  
 run WebPAC reniced on those machines.  
   
   
667  =head1 AUTHOR  =head1 AUTHOR
668    
669  Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>  Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
Legend:
Removed from v.760  
changed lines
  Added in v.761
  ViewVC Help
Powered by ViewVC 1.1.26