/[Biblio-Isis]/trunk/IsisDB.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/IsisDB.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 15 by dpavlin, Wed Dec 29 22:46:40 2004 UTC revision 27 by dpavlin, Sat Jan 1 22:29:35 2005 UTC
# Line 2  package IsisDB; Line 2  package IsisDB;
2  use strict;  use strict;
3    
4  use Carp;  use Carp;
5    use File::Glob qw(:globally :nocase);
6    
7  use Data::Dumper;  use Data::Dumper;
8    
9  BEGIN {  BEGIN {
10          use Exporter ();          use Exporter ();
11          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
12          $VERSION     = 0.05;          $VERSION     = 0.08;
13          @ISA         = qw (Exporter);          @ISA         = qw (Exporter);
14          #Give a hoot don't pollute, do not export more than needed by default          #Give a hoot don't pollute, do not export more than needed by default
15          @EXPORT      = qw ();          @EXPORT      = qw ();
# Line 35  IsisDB - Read CDS/ISIS, WinISIS and Isis Line 37  IsisDB - Read CDS/ISIS, WinISIS and Isis
37  =head1 DESCRIPTION  =head1 DESCRIPTION
38    
39  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or
40  IsisMarc. It can be used as perl-only alternative to OpenIsis module.  IsisMarc. It can be used as perl-only alternative to OpenIsis module which
41    seems to depriciate it's old C<XS> bindings for perl.
42    
43  It can create hash values from data in ISIS database (using C<to_hash>),  It can create hash values from data in ISIS database (using C<to_hash>),
44  ASCII dump (using C<to_ascii>) or just hash with field names and packed  ASCII dump (using C<to_ascii>) or just hash with field names and packed
# Line 48  fields which are zero sized will be fill Line 51  fields which are zero sized will be fill
51  It also has support for identifiers (only if ISIS database is created by  It also has support for identifiers (only if ISIS database is created by
52  IsisMarc), see C<to_hash>.  IsisMarc), see C<to_hash>.
53    
54  This will module will always be slower than OpenIsis module which use C  This module will always be slower than OpenIsis module which use C
55  library. However, since it's written in perl, it's platform independent (so  library. However, since it's written in perl, it's platform independent (so
56  you don't need C compiler), and can be easily modified. I hope that it  you don't need C compiler), and can be easily modified. I hope that it
57  creates data structures which are easier to use than ones created by  creates data structures which are easier to use than ones created by
# Line 96  Options are described below: Line 99  Options are described below:
99  =item isisdb  =item isisdb
100    
101  This is full or relative path to ISIS database files which include  This is full or relative path to ISIS database files which include
102  common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.  common prefix of C<.MST>, and C<.XRF> and optionally C<.FDT> (if using
103    C<read_fdt> option) files.
104    
105  In this example it uses C<./cds/cds.MST> and related files.  In this example it uses C<./cds/cds.MST> and related files.
106    
# Line 134  sub new { Line 138  sub new {
138                  $self->{$v} = {@_}->{$v};                  $self->{$v} = {@_}->{$v};
139          }          }
140    
141            my @isis_files = grep(/\.(FDT|MST|XRF|CNT)$/i,glob($self->{isisdb}."*"));
142    
143            foreach my $f (@isis_files) {
144                    my $ext = $1 if ($f =~ m/\.(\w\w\w)$/);
145                    $self->{lc($ext)."_file"} = $f;
146            }
147    
148            my @must_exist = qw(mst xrf);
149            push @must_exist, "fdt" if ($self->{read_fdt});
150    
151            foreach my $ext (@must_exist) {
152                    croak "missing ",uc($ext)," file in ",$self->{isisdb} unless ($self->{$ext."_file"});
153            }
154    
155            print STDERR "## using files: ",join(" ",@isis_files),"\n" if ($self->{debug});
156    
157          # if you want to read .FDT file use read_fdt argument when creating class!          # if you want to read .FDT file use read_fdt argument when creating class!
158          if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {          if ($self->{read_fdt} && -e $self->{fdt_file}) {
159    
160                  # read the $db.FDT file for tags                  # read the $db.FDT file for tags
161                  my $fieldzone=0;                  my $fieldzone=0;
162    
163                  open(fileFDT, $self->{isisdb}.".FDT") || croak "can't read '$self->{isisdb}.FDT': $!";                  open(fileFDT, $self->{fdt_file}) || croak "can't read '$self->{fdt_file}': $!";
164    
165                  while (<fileFDT>) {                  while (<fileFDT>) {
166                          chomp;                          chomp;
# Line 164  sub new { Line 184  sub new {
184    
185          # Get the Maximum MFN from $db.MST          # Get the Maximum MFN from $db.MST
186    
187          open(fileMST,$self->{isisdb}.".MST") || croak "can't read '$self->{isisdb}.MST': $!";          open($self->{'fileMST'}, $self->{mst_file}) || croak "can't open '$self->{mst_file}': $!";
188    
189          # MST format:   (* = 32 bit signed)          # MST format:   (* = 32 bit signed)
190          # CTLMFN*       always 0          # CTLMFN*       always 0
# Line 172  sub new { Line 192  sub new {
192          # NXTMFB*       last block allocated to master file          # NXTMFB*       last block allocated to master file
193          # NXTMFP        offset to next available position in last block          # NXTMFP        offset to next available position in last block
194          # MFTYPE        always 0 for user db file (1 for system)          # MFTYPE        always 0 for user db file (1 for system)
195          seek(fileMST,4,0);          seek($self->{'fileMST'},4,0);
196    
197          my $buff;          my $buff;
198    
199          read(fileMST, $buff, 4);          read($self->{'fileMST'}, $buff, 4);
200          $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";          $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";
201    
202          # save maximum MFN          # save maximum MFN
203          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;
204    
205          close(fileMST);  
206    
207    
208            print STDERR Dumper($self),"\n" if ($self->{debug});
209    
210            # open files for later
211            open($self->{'fileXRF'}, $self->{xrf_file}) || croak "can't open '$self->{xrf_file}': $!";
212    
213            $self ? return $self : return undef;
214    }
215    
216    =head2 read_cnt
217    
218    Read content of C<.CNT> file and return hash containing it.
219    
220      print Dumper($isis->read_cnt);
221    
222    This function is not used by module (C<.CNT> files are not required for this
223    module to work), but it can be useful to examine your index (while debugging
224    for example).
225    
226    =cut
227    
228    sub read_cnt  {
229            my $self = shift;
230    
231            croak "missing CNT file in ",$self->{isisdb} unless ($self->{cnt_file});
232    
233          # Get the index information from $db.CNT          # Get the index information from $db.CNT
234        
235          open(fileCNT, $self->{isisdb}.".CNT") || croak "can't read '$self->{isisdb}.CNT': $!";          open(fileCNT, $self->{cnt_file}) || croak "can't read '$self->{cnt_file}': $!";
236    
237          # There is two 26 Bytes fixed lenght records          # There is two 26 Bytes fixed lenght records
238    
# Line 210  sub new { Line 256  sub new {
256                  my $buff = shift || return;                  my $buff = shift || return;
257                  my @arr = unpack("ssssssllls", $buff);                  my @arr = unpack("ssssssllls", $buff);
258    
259                  print "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});                  print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
260    
261                  my $IDTYPE = shift @arr;                  my $IDTYPE = shift @arr;
262                  foreach (@flds) {                  foreach (@flds) {
263                          $self->{$IDTYPE}->{$_} = abs(shift @arr);                          $self->{cnt}->{$IDTYPE}->{$_} = abs(shift @arr);
264                  }                  }
265          }          }
266    
267            my $buff;
268    
269          read(fileCNT, $buff, 26);          read(fileCNT, $buff, 26);
270          $self->unpack_cnt($buff);          $self->unpack_cnt($buff);
271    
272          read(fileCNT, $buff, 26);          read(fileCNT, $buff, 26);
273          $self->unpack_cnt($buff);          $self->unpack_cnt($buff);
274    
   
275          close(fileCNT);          close(fileCNT);
276    
277          print Dumper($self),"\n" if ($self->{debug});          return $self->{cnt};
   
         # open files for later  
         open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";  
   
         open($self->{'fileMST'}, $self->{isisdb}.".MST") || croak "can't open '$self->{isisdb}.MST': $!";  
   
         $self ? return $self : return undef;  
278  }  }
279    
280  =head2 fetch  =head2 fetch
# Line 258  sub fetch { Line 298  sub fetch {
298    
299          my $mfn = shift || croak "fetch needs MFN as argument!";          my $mfn = shift || croak "fetch needs MFN as argument!";
300    
301          print "fetch: $mfn\n" if ($self->{debug});          # is mfn allready in memory?
302            my $old_mfn = $self->{'current_mfn'} || -1;
303            return $self->{record} if ($mfn == $old_mfn);
304    
305            print STDERR "## fetch: $mfn\n" if ($self->{debug});
306    
307          # XXX check this?          # XXX check this?
308          my $mfnpos=($mfn+int(($mfn-1)/127))*4;          my $mfnpos=($mfn+int(($mfn-1)/127))*4;
309    
310          print "seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});          print STDERR "## seeking to $mfnpos in file '$self->{xrf_file}'\n" if ($self->{debug});
311          seek($self->{'fileXRF'},$mfnpos,0);          seek($self->{'fileXRF'},$mfnpos,0);
312    
313          my $buff;          my $buff;
314    
315            # delete old record
316            delete $self->{record};
317    
318          # read XRFMFB abd XRFMFP          # read XRFMFB abd XRFMFP
319          read($self->{'fileXRF'}, $buff, 4);          read($self->{'fileXRF'}, $buff, 4);
320          my $pointer=unpack("l",$buff) || carp "pointer is null";          my $pointer=unpack("l",$buff) || carp "pointer is null";
321    
322            # check for logically deleted record
323            if ($pointer < 0) {
324                    print STDERR "## record $mfn is logically deleted\n" if ($self->{debug});
325                    $self->{deleted} = $mfn;
326    
327                    return unless $self->{include_deleted};
328    
329                    $pointer = abs($pointer);
330            }
331    
332          my $XRFMFB = int($pointer/2048);          my $XRFMFB = int($pointer/2048);
333          my $XRFMFP = $pointer - ($XRFMFB*2048);          my $XRFMFP = $pointer - ($XRFMFB*2048);
334    
335          print "XRFMFB: $XRFMFB XRFMFP: $XRFMFP\n" if ($self->{debug});          # (XRFMFB - 1) * 512 + XRFMFP
336            # why do i have to do XRFMFP % 1024 ?
337    
338          # XXX fix this to be more readable!!          my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 512);
         # e.g. (XRFMFB - 1) * 512 + XRFMFP  
339    
340          my $offset = $pointer;          print STDERR "## pointer: $pointer XRFMFB: $XRFMFB XRFMFP: $XRFMFP offset: $blk_off\n" if ($self->{'debug'});
         my $offset2=int($offset/2048)-1;  
         my $offset22=int($offset/4096);  
         my $offset3=$offset-($offset22*4096);  
         if ($offset3>512) {  
                 $offset3=$offset3-2048;  
         }  
         my $offset4=($offset2*512)+$offset3;  
   
         print "$offset - $offset2 - $offset3 - $offset4\n" if ($self->{debug});  
341    
342          # Get Record Information          # Get Record Information
343    
344          seek($self->{'fileMST'},$offset4,0);          seek($self->{'fileMST'},$blk_off,0);
345    
346          read($self->{'fileMST'}, $buff, 4);          read($self->{'fileMST'}, $buff, 4);
347          my $value=unpack("l",$buff);          my $value=unpack("l",$buff);
348    
349            print STDERR "## offset for rowid $value is $blk_off (blk $XRFMFB off $XRFMFP)\n" if ($self->{debug});
350    
351          if ($value!=$mfn) {          if ($value!=$mfn) {
352  print ("Error: The MFN:".$mfn." is not found in MST(".$value.")");                      if ($value == 0) {
353                  return -1;      # XXX deleted record?                          print STDERR "## record $mfn is physically deleted\n" if ($self->{debug});
354          }                          $self->{deleted} = $mfn;
355                            return;
356                    }
357    
358  #       $MFRL=$self->Read16($fileMST);                  carp "Error: MFN ".$mfn." not found in MST file, found $value";    
359  #       $MFBWB=$self->Read32($fileMST);                  return;
360  #       $MFBWP=$self->Read16($fileMST);          }
 #       $BASE=$self->Read16($fileMST);  
 #       $NVF=$self->Read16($fileMST);  
 #       $STATUS=$self->Read16($fileMST);  
361    
362          read($self->{'fileMST'}, $buff, 14);          read($self->{'fileMST'}, $buff, 14);
363    
364          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);
365    
366          print "MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});          print STDERR "## MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
367    
368          # delete old record          warn "MFRL $MFRL is not even number" unless ($MFRL % 2 == 0);
         delete $self->{record};  
369    
370          if (! $self->{'include_deleted'} && $MFRL < 0) {          warn "BASE is not 18+6*NVF" unless ($BASE == 18 + 6 * $NVF);
                 print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});  
                 return;  
         }  
371    
372          # Get Directory Format          # Get Directory Format
373    
# Line 332  print ("Error: The MFN:".$mfn." is not f Line 377  print ("Error: The MFN:".$mfn." is not f
377    
378          read($self->{'fileMST'}, $buff, 6 * $NVF);          read($self->{'fileMST'}, $buff, 6 * $NVF);
379    
380          my $fld_len = 0;          my $rec_len = 0;
381    
382          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
383    
 #               $TAG=$self->Read16($fileMST);  
 #               $POS=$self->Read16($fileMST);  
 #               $LEN=$self->Read16($fileMST);  
   
384                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));
385    
386                  print "TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});                  print STDERR "## TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
387    
388                  # The TAG does not exists in .FDT so we set it to 0.                  # The TAG does not exists in .FDT so we set it to 0.
389                  #                  #
# Line 358  print ("Error: The MFN:".$mfn." is not f Line 399  print ("Error: The MFN:".$mfn." is not f
399                  push @FieldPOS,$POS;                  push @FieldPOS,$POS;
400                  push @FieldLEN,$LEN;                  push @FieldLEN,$LEN;
401    
402                  $fld_len += $LEN;                  $rec_len += $LEN;
403          }          }
404    
405          # Get Variable Fields          # Get Variable Fields
406    
407          read($self->{'fileMST'},$buff,$fld_len);          read($self->{'fileMST'},$buff,$rec_len);
408    
409            print STDERR "## rec_len: $rec_len poc: ",tell($self->{'fileMST'})."\n" if ($self->{debug});
410    
411          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
412                  # skip zero-sized fields                  # skip zero-sized fields
# Line 371  print ("Error: The MFN:".$mfn." is not f Line 414  print ("Error: The MFN:".$mfn." is not f
414    
415                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);
416          }          }
         close(fileMST);  
417    
418          print Dumper($self),"\n" if ($self->{debug});          $self->{'current_mfn'} = $mfn;
419    
420            print STDERR Dumper($self),"\n" if ($self->{debug});
421    
422          return $self->{'record'};          return $self->{'record'};
423  }  }
424    
425  =head2 to_ascii  =head2 to_ascii
426    
427  Dump ASCII output of record with specified MFN  Returns ASCII output of record with specified MFN
428    
429    print $isis->to_ascii(42);    print $isis->to_ascii(42);
430    
431  It outputs something like this:  This outputs something like this:
432    
433    210   ^aNew York^cNew York University press^dcop. 1988    210   ^aNew York^cNew York University press^dcop. 1988
434    990   2140    990   2140
# Line 421  Read record with specified MFN and conve Line 465  Read record with specified MFN and conve
465    
466    my $hash = $isis->to_hash($mfn);    my $hash = $isis->to_hash($mfn);
467    
468  It has ability to convert characters (using C<hash_filter> from ISIS  It has ability to convert characters (using C<hash_filter>) from ISIS
469  database before creating structures enabling character re-mapping or quick  database before creating structures enabling character re-mapping or quick
470  fix-up of data.  fix-up of data.
471    
# Line 467  sub to_hash { Line 511  sub to_hash {
511          my $mfn = shift || confess "need mfn!";          my $mfn = shift || confess "need mfn!";
512    
513          # init record to include MFN as field 000          # init record to include MFN as field 000
514          my $rec = { '000' => $mfn };          my $rec = { '000' => [ $mfn ] };
515    
516          my $row = $self->fetch($mfn);          my $row = $self->fetch($mfn);
517    
# Line 480  sub to_hash { Line 524  sub to_hash {
524                          my $val;                          my $val;
525    
526                          # has identifiers?                          # has identifiers?
527                          ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])//);                          ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])\^/\^/);
528    
529                          # has subfields?                          # has subfields?
530                          if ($l =~ m/\^/) {                          if ($l =~ m/\^/) {
# Line 517  sub tag_name { Line 561  sub tag_name {
561    
562  =head1 BUGS  =head1 BUGS
563    
564  This module has been very lightly tested. Use with caution and report bugs.  Some parts of CDS/ISIS documentation are not detailed enough to exmplain
565    some variations in input databases which has been tested with this module.
566    When I was in doubt, I assumed that OpenIsis's implementation was right
567    (except for obvious bugs).
568    
569    However, every effort has been made to test this module with as much
570    databases (and programs that create them) as possible.
571    
572    I would be very greatful for success or failure reports about usage of this
573    module with databases from programs other than WinIsis and IsisMarc. I had
574    tested this against ouput of one C<isis.dll>-based application, but I don't
575    know any details about it's version.
576    
577  =head1 AUTHOR  =head1 AUTHOR
578    
Legend:
Removed from v.15  
changed lines
  Added in v.27
  ViewVC Help
Powered by ViewVC 1.1.26