--- trunk/lib/WebPAC/Input.pm 2005/07/16 20:54:28 11 +++ trunk/lib/WebPAC/Input.pm 2005/12/18 21:06:39 285 @@ -3,21 +3,29 @@ use warnings; use strict; +use blib; + +use WebPAC::Common; +use base qw/WebPAC::Common/; +use Text::Iconv; + =head1 NAME WebPAC::Input - core module for input file format =head1 VERSION -Version 0.01 +Version 0.02 =cut -our $VERSION = '0.01'; +our $VERSION = '0.02'; =head1 SYNOPSIS -This module will load particular loader module and execute it's functions. +This module is used as base class for all database specific modules +(basically, files which have one handle, fixed size while indexing and some +kind of numeric idefinirier which goes from 1 to filesize). Perhaps a little code snippet. @@ -54,17 +62,30 @@ Default is not to use C options (see L below). +This function will also call low-level C if it exists with same +parametars. + =cut sub new { - my $class = shift; - my $self = {@_}; + my $class = shift; + my $self = {@_}; bless($self, $class); - $self->{'code_page'} ||= 'ISO-8859-2'; - my $log = $self->_get_logger; + # check if required subclasses are implemented + foreach my $subclass (qw/open_db fetch_rec/) { + $log->logdie("missing implementation of $subclass") unless ($self->SUPER::can($subclass)); + } + + if ($self->can('init')) { + $log->debug("calling init"); + $self->init(@_); + } + + $self->{'code_page'} ||= 'ISO-8859-2'; + # running with low_mem flag? well, use DBM::Deep then. if ($self->{'low_mem'}) { $log->info("running with low_mem which impacts performance (<32 Mb memory usage)"); @@ -94,6 +115,229 @@ $self ? return $self : return undef; } +=head2 open + +This function will read whole database in memory and produce lookups. + + $isis->open( + path => '/path/to/database/file', + code_page => '852', + limit_mfn => 500, + start_mfn => 6000, + lookup => $lookup_obj, + ); + +By default, C is assumed to be C<852>. + +If optional parametar C is set, this will be first MFN to read +from database (so you can skip beginning of your database if you need to). + +If optional parametar C is set, it will read just 500 records +from database in example above. + +Returns size of database, regardless of C and C +parametars, see also C<$isis->size>. + +=cut + +sub open { + my $self = shift; + my $arg = {@_}; + + my $log = $self->_get_logger(); + + $log->logcroak("need path") if (! $arg->{'path'}); + my $code_page = $arg->{'code_page'} || '852'; + + # store data in object + $self->{'code_page'} = $code_page; + foreach my $v (qw/path start_mfn limit_mfn/) { + $self->{$v} = $arg->{$v} if ($arg->{$v}); + } + + # create Text::Iconv object + $self->{iconv} = Text::Iconv->new($code_page,$self->{'code_page'}); + + my ($db, $size) = $self->open_db( + path => $arg->{path}, + ); + + unless ($db) { + $log->logwarn("can't open database $arg->{path}, skipping..."); + return; + } + + unless ($size) { + $log->logwarn("no records in database $arg->{path}, skipping..."); + return; + } + + my $startmfn = 1; + my $maxmfn = $size; + + if (my $s = $self->{start_mfn}) { + $log->info("skipping to MFN $s"); + $startmfn = $s; + } else { + $self->{start_mfn} = $startmfn; + } + + if ($self->{limit_mfn}) { + $log->info("limiting to ",$self->{limit_mfn}," records"); + $maxmfn = $startmfn + $self->{limit_mfn} - 1; + $maxmfn = $size if ($maxmfn > $size); + } + + # store size for later + $self->{size} = ($maxmfn - $startmfn) ? ($maxmfn - $startmfn + 1) : 0; + + $log->info("processing $self->{size} records in $code_page, convert to $self->{code_page}"); + + # read database + for (my $mfn = $startmfn; $mfn <= $maxmfn; $mfn++) { + + $log->debug("mfn: $mfn\n"); + + my $rec = $self->fetch_rec( $db, $mfn ); + + if (! $rec) { + $log->warn("record $mfn empty? skipping..."); + next; + } + + # store + if ($self->{'low_mem'}) { + $self->{'db'}->put($mfn, $rec); + } else { + $self->{'data'}->{$mfn} = $rec; + } + + # create lookup + $self->{'lookup'}->add( $rec ) if ($rec && $self->{'lookup'}); + + $self->progress_bar($mfn,$maxmfn); + + } + + $self->{'current_mfn'} = -1; + $self->{'last_pcnt'} = 0; + + $log->debug("max mfn: $maxmfn"); + + # store max mfn and return it. + $self->{'max_mfn'} = $maxmfn; + + return $size; +} + +=head2 fetch + +Fetch next record from database. It will also displays progress bar. + + my $rec = $isis->fetch; + +Record from this function should probably go to C for +normalisation. + +=cut + +sub fetch { + my $self = shift; + + my $log = $self->_get_logger(); + + $log->logconfess("it seems that you didn't load database!") unless ($self->{'current_mfn'}); + + if ($self->{'current_mfn'} == -1) { + $self->{'current_mfn'} = $self->{'start_mfn'}; + } else { + $self->{'current_mfn'}++; + } + + my $mfn = $self->{'current_mfn'}; + + if ($mfn > $self->{'max_mfn'}) { + $self->{'current_mfn'} = $self->{'max_mfn'}; + $log->debug("at EOF"); + return; + } + + $self->progress_bar($mfn,$self->{'max_mfn'}); + + my $rec; + + if ($self->{'low_mem'}) { + $rec = $self->{'db'}->get($mfn); + } else { + $rec = $self->{'data'}->{$mfn}; + } + + $rec ||= 0E0; +} + +=head2 pos + +Returns current record number (MFN). + + print $isis->pos; + +First record in database has position 1. + +=cut + +sub pos { + my $self = shift; + return $self->{'current_mfn'}; +} + + +=head2 size + +Returns number of records in database + + print $isis->size; + +Result from this function can be used to loop through all records + + foreach my $mfn ( 1 ... $isis->size ) { ... } + +because it takes into account C and C. + +=cut + +sub size { + my $self = shift; + return $self->{'size'}; +} + +=head2 seek + +Seek to specified MFN in file. + + $isis->seek(42); + +First record in database has position 1. + +=cut + +sub seek { + my $self = shift; + my $pos = shift || return; + + my $log = $self->_get_logger(); + + if ($pos < 1) { + $log->warn("seek before first record"); + $pos = 1; + } elsif ($pos > $self->{'max_mfn'}) { + $log->warn("seek beyond last record"); + $pos = $self->{'max_mfn'}; + } + + return $self->{'current_mfn'} = (($pos - 1) || -1); +} + + =head1 MEMORY USAGE C options is double-edged sword. If enabled, WebPAC