/[webpac2]/trunk/lib/WebPAC/Input.pm

This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!

Diff of /trunk/lib/WebPAC/Input.pm

Parent Directory | Revision Log | View Patch Patch

-revision 11 by dpavlin,
Sat Jul 16 20:54:28 2005 UTC
+revision 285 by dpavlin,
Sun Dec 18 21:06:39 2005 UTC
 Line 3 
 package WebPAC::Input;
  use warnings;
  use strict;
+ use blib;
+ use WebPAC::Common;
+ use base qw/WebPAC::Common/;
+ use Text::Iconv;
  =head1 NAME
  WebPAC::Input - core module for input file format
  =head1 VERSION
- Version 0.01
+ Version 0.02
  =cut
- our $VERSION = '0.01';
+ our $VERSION = '0.02';
  =head1 SYNOPSIS
- This module will load particular loader module and execute it's functions.
+ This module is used as base class for all database specific modules
+ (basically, files which have one handle, fixed size while indexing and some
+ kind of numeric idefinirier which goes from 1 to filesize).
  Perhaps a little code snippet.
-Line 54 
 default, it C<ISO-8859-2>.
+Line 62 
 default, it C<ISO-8859-2>.
  Default is not to use C<low_mem> options (see L<MEMORY USAGE> below).
+ This function will also call low-level C<init> if it exists with same
+ parametars.
  =cut
  sub new {
          my $class = shift;
          my $self = {@_};
          bless($self, $class);
-         $self->{'code_page'} ||= 'ISO-8859-2';
          my $log = $self->_get_logger;
+         # check if required subclasses are implemented
+         foreach my $subclass (qw/open_db fetch_rec/) {
+                 $log->logdie("missing implementation of $subclass") unless ($self->SUPER::can($subclass));
+         }
+         if ($self->can('init')) {
+                 $log->debug("calling init");
+                 $self->init(@_);
+         }
+         $self->{'code_page'} ||= 'ISO-8859-2';
          # running with low_mem flag? well, use DBM::Deep then.
          if ($self->{'low_mem'}) {
                  $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
-Line 94 
 sub new {
+Line 115 
 sub new {
          $self ? return $self : return undef;
  }
+ =head2 open
+ This function will read whole database in memory and produce lookups.
+  $isis->open(
+         path => '/path/to/database/file',
+         code_page => '852',
+         limit_mfn => 500,
+         start_mfn => 6000,
+         lookup => $lookup_obj,
+  );
+ By default, C<code_page> is assumed to be C<852>.
+ If optional parametar C<start_mfn> is set, this will be first MFN to read
+ from database (so you can skip beginning of your database if you need to).
+ If optional parametar C<limit_mfn> is set, it will read just 500 records
+ from database in example above.
+ Returns size of database, regardless of C<start_mfn> and C<limit_mfn>
+ parametars, see also C<$isis->size>.
+ =cut
+ sub open {
+         my $self = shift;
+         my $arg = {@_};
+         my $log = $self->_get_logger();
+         $log->logcroak("need path") if (! $arg->{'path'});
+         my $code_page = $arg->{'code_page'} || '852';
+         # store data in object
+         $self->{'code_page'} = $code_page;
+         foreach my $v (qw/path start_mfn limit_mfn/) {
+                 $self->{$v} = $arg->{$v} if ($arg->{$v});
+         }
+         # create Text::Iconv object
+         $self->{iconv} = Text::Iconv->new($code_page,$self->{'code_page'});
+         my ($db, $size) = $self->open_db(
+                 path => $arg->{path},
+         );
+         unless ($db) {
+                 $log->logwarn("can't open database $arg->{path}, skipping...");
+                 return;
+         }
+         unless ($size) {
+                 $log->logwarn("no records in database $arg->{path}, skipping...");
+                 return;
+         }
+         my $startmfn = 1;
+         my $maxmfn = $size;
+         if (my $s = $self->{start_mfn}) {
+                 $log->info("skipping to MFN $s");
+                 $startmfn = $s;
+         } else {
+                 $self->{start_mfn} = $startmfn;
+         }
+         if ($self->{limit_mfn}) {
+                 $log->info("limiting to ",$self->{limit_mfn}," records");
+                 $maxmfn = $startmfn + $self->{limit_mfn} - 1;
+                 $maxmfn = $size if ($maxmfn > $size);
+         }
+         # store size for later
+         $self->{size} = ($maxmfn - $startmfn) ? ($maxmfn - $startmfn + 1) : 0;
+         $log->info("processing $self->{size} records in $code_page, convert to $self->{code_page}");
+         # read database
+         for (my $mfn = $startmfn; $mfn <= $maxmfn; $mfn++) {
+                 $log->debug("mfn: $mfn\n");
+                 my $rec = $self->fetch_rec( $db, $mfn );
+                 if (! $rec) {
+                         $log->warn("record $mfn empty? skipping...");
+                         next;
+                 }
+                 # store
+                 if ($self->{'low_mem'}) {
+                         $self->{'db'}->put($mfn, $rec);
+                 } else {
+                         $self->{'data'}->{$mfn} = $rec;
+                 }
+                 # create lookup
+                 $self->{'lookup'}->add( $rec ) if ($rec && $self->{'lookup'});
+                 $self->progress_bar($mfn,$maxmfn);
+         }
+         $self->{'current_mfn'} = -1;
+         $self->{'last_pcnt'} = 0;
+         $log->debug("max mfn: $maxmfn");
+         # store max mfn and return it.
+         $self->{'max_mfn'} = $maxmfn;
+         return $size;
+ }
+ =head2 fetch
+ Fetch next record from database. It will also displays progress bar.
+  my $rec = $isis->fetch;
+ Record from this function should probably go to C<data_structure> for
+ normalisation.
+ =cut
+ sub fetch {
+         my $self = shift;
+         my $log = $self->_get_logger();
+         $log->logconfess("it seems that you didn't load database!") unless ($self->{'current_mfn'});
+         if ($self->{'current_mfn'} == -1) {
+                 $self->{'current_mfn'} = $self->{'start_mfn'};
+         } else {
+                 $self->{'current_mfn'}++;
+         }
+         my $mfn = $self->{'current_mfn'};
+         if ($mfn > $self->{'max_mfn'}) {
+                 $self->{'current_mfn'} = $self->{'max_mfn'};
+                 $log->debug("at EOF");
+                 return;
+         }
+         $self->progress_bar($mfn,$self->{'max_mfn'});
+         my $rec;
+         if ($self->{'low_mem'}) {
+                 $rec = $self->{'db'}->get($mfn);
+         } else {
+                 $rec = $self->{'data'}->{$mfn};
+         }
+         $rec ||= 0E0;
+ }
+ =head2 pos
+ Returns current record number (MFN).
+  print $isis->pos;
+ First record in database has position 1.
+ =cut
+ sub pos {
+         my $self = shift;
+         return $self->{'current_mfn'};
+ }
+ =head2 size
+ Returns number of records in database
+  print $isis->size;
+ Result from this function can be used to loop through all records
+  foreach my $mfn ( 1 ... $isis->size ) { ... }
+ because it takes into account C<start_mfn> and C<limit_mfn>.
+ =cut
+ sub size {
+         my $self = shift;
+         return $self->{'size'};
+ }
+ =head2 seek
+ Seek to specified MFN in file.
+  $isis->seek(42);
+ First record in database has position 1.
+ =cut
+ sub seek {
+         my $self = shift;
+         my $pos = shift || return;
+         my $log = $self->_get_logger();
+         if ($pos < 1) {
+                 $log->warn("seek before first record");
+                 $pos = 1;
+         } elsif ($pos > $self->{'max_mfn'}) {
+                 $log->warn("seek beyond last record");
+                 $pos = $self->{'max_mfn'};
+         }
+         return $self->{'current_mfn'} = (($pos - 1) || -1);
+ }
  =head1 MEMORY USAGE
  C<low_mem> options is double-edged sword. If enabled, WebPAC

 Legend:



Removed from v.11
 


changed lines


 
Added in v.285
 Legend:



Removed from v.11
 


changed lines


 
Added in v.285
-Removed from v.11
+Added in v.285

	ViewVC Help
Powered by ViewVC 1.1.26