/[Biblio-Isis]/trunk/IsisDB.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/IsisDB.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 12 by dpavlin, Wed Dec 29 20:10:11 2004 UTC revision 25 by dpavlin, Fri Dec 31 05:43:20 2004 UTC
# Line 2  package IsisDB; Line 2  package IsisDB;
2  use strict;  use strict;
3    
4  use Carp;  use Carp;
5    use File::Glob qw(:globally :nocase);
6    
7  use Data::Dumper;  use Data::Dumper;
8    
9  BEGIN {  BEGIN {
10          use Exporter ();          use Exporter ();
11          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
12          $VERSION     = 0.04;          $VERSION     = 0.08;
13          @ISA         = qw (Exporter);          @ISA         = qw (Exporter);
14          #Give a hoot don't pollute, do not export more than needed by default          #Give a hoot don't pollute, do not export more than needed by default
15          @EXPORT      = qw ();          @EXPORT      = qw ();
# Line 18  BEGIN { Line 20  BEGIN {
20    
21  =head1 NAME  =head1 NAME
22    
23  IsisDB - Read CDS/ISIS database  IsisDB - Read CDS/ISIS, WinISIS and IsisMarc database
24    
25  =head1 SYNOPSIS  =head1 SYNOPSIS
26    
# Line 34  IsisDB - Read CDS/ISIS database Line 36  IsisDB - Read CDS/ISIS database
36    
37  =head1 DESCRIPTION  =head1 DESCRIPTION
38    
39  This module will read CDS/ISIS databases and create hash values out of it.  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or
40  It can be used as perl-only alternative to OpenIsis module.  IsisMarc. It can be used as perl-only alternative to OpenIsis module.
41    
42  This will module will always be slower that OpenIsis module which use C  It can create hash values from data in ISIS database (using C<to_hash>),
43  library. However, since it's written in perl, it's platform independent (so  ASCII dump (using C<to_ascii>) or just hash with field names and packed
44  you don't need C compiler), and can be easily modified.  values (like C<^asomething^belse>).
45    
46  Unique feature of this module is ability to C<include_deleted> records.  Unique feature of this module is ability to C<include_deleted> records.
47  It will also skip zero sized fields (OpenIsis has a bug in XS bindings, so  It will also skip zero sized fields (OpenIsis has a bug in XS bindings, so
48  fields which are zero sized will be filled with random junk from memory).  fields which are zero sized will be filled with random junk from memory).
49    
50    It also has support for identifiers (only if ISIS database is created by
51    IsisMarc), see C<to_hash>.
52    
53    This will module will always be slower than OpenIsis module which use C
54    library. However, since it's written in perl, it's platform independent (so
55    you don't need C compiler), and can be easily modified. I hope that it
56    creates data structures which are easier to use than ones created by
57    OpenIsis, so reduced time in other parts of the code should compensate for
58    slower performance of this module (speed of reading ISIS database is
59    rarely an issue).
60    
61  =head1 METHODS  =head1 METHODS
62    
63  =cut  =cut
# Line 65  fields which are zero sized will be fill Line 78  fields which are zero sized will be fill
78    
79  =head2 new  =head2 new
80    
81  Open CDS/ISIS database  Open ISIS database
82    
83   my $isis = new IsisDB(   my $isis = new IsisDB(
84          isisdb => './cds/cds',          isisdb => './cds/cds',
# Line 84  Options are described below: Line 97  Options are described below:
97    
98  =item isisdb  =item isisdb
99    
100  Prefix path to CDS/ISIS. It should contain full or relative path to database  This is full or relative path to ISIS database files which include
101  and common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.  common prefix of C<.MST>, and C<.XRF> and optionally C<.FDT> (if using
102    C<read_fdt> option) files.
103    
104    In this example it uses C<./cds/cds.MST> and related files.
105    
106  =item read_fdt  =item read_fdt
107    
# Line 121  sub new { Line 137  sub new {
137                  $self->{$v} = {@_}->{$v};                  $self->{$v} = {@_}->{$v};
138          }          }
139    
140            my @isis_files = grep(/\.(FDT|MST|XRF|CNT)$/i,glob($self->{isisdb}."*"));
141    
142            foreach my $f (@isis_files) {
143                    my $ext = $1 if ($f =~ m/\.(\w\w\w)$/);
144                    $self->{lc($ext)."_file"} = $f;
145            }
146    
147            my @must_exist = qw(mst xrf);
148            push @must_exist, "fdt" if ($self->{read_fdt});
149    
150            foreach my $ext (@must_exist) {
151                    croak "missing ",uc($ext)," file in ",$self->{isisdb} unless ($self->{$ext."_file"});
152            }
153    
154            print STDERR "## using files: ",join(" ",@isis_files),"\n" if ($self->{debug});
155    
156          # if you want to read .FDT file use read_fdt argument when creating class!          # if you want to read .FDT file use read_fdt argument when creating class!
157          if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {          if ($self->{read_fdt} && -e $self->{fdt_file}) {
158    
159                  # read the $db.FDT file for tags                  # read the $db.FDT file for tags
160                  my $fieldzone=0;                  my $fieldzone=0;
161    
162                  open(fileFDT, $self->{isisdb}.".FDT") || croak "can't read '$self->{isisdb}.FDT': $!";                  open(fileFDT, $self->{fdt_file}) || croak "can't read '$self->{fdt_file}': $!";
163    
164                  while (<fileFDT>) {                  while (<fileFDT>) {
165                          chomp;                          chomp;
# Line 151  sub new { Line 183  sub new {
183    
184          # Get the Maximum MFN from $db.MST          # Get the Maximum MFN from $db.MST
185    
186          open(fileMST,$self->{isisdb}.".MST") || croak "can't read '$self->{isisdb}.MST': $!";          open($self->{'fileMST'}, $self->{mst_file}) || croak "can't open '$self->{mst_file}': $!";
187    
188          # MST format:   (* = 32 bit signed)          # MST format:   (* = 32 bit signed)
189          # CTLMFN*       always 0          # CTLMFN*       always 0
# Line 159  sub new { Line 191  sub new {
191          # NXTMFB*       last block allocated to master file          # NXTMFB*       last block allocated to master file
192          # NXTMFP        offset to next available position in last block          # NXTMFP        offset to next available position in last block
193          # MFTYPE        always 0 for user db file (1 for system)          # MFTYPE        always 0 for user db file (1 for system)
194          seek(fileMST,4,0);          seek($self->{'fileMST'},4,0);
195    
196          my $buff;          my $buff;
197    
198          read(fileMST, $buff, 4);          read($self->{'fileMST'}, $buff, 4);
199          $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";          $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";
200    
201          # save maximum MFN          # save maximum MFN
202          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;
203    
204          close(fileMST);  
205    
206    
207            print STDERR Dumper($self),"\n" if ($self->{debug});
208    
209            # open files for later
210            open($self->{'fileXRF'}, $self->{xrf_file}) || croak "can't open '$self->{xrf_file}': $!";
211    
212            $self ? return $self : return undef;
213    }
214    
215    =head2 read_cnt
216    
217    This function is not really used by module, but can be useful to find info
218    about your index (if debugging it for example).
219    
220      print Dumper($isis->read_cnt);
221    
222    =cut
223    
224    sub read_cnt  {
225            my $self = shift;
226    
227            croak "missing CNT file in ",$self->{isisdb} unless ($self->{cnt_file});
228    
229          # Get the index information from $db.CNT          # Get the index information from $db.CNT
230        
231          open(fileCNT, $self->{isisdb}.".CNT") || croak "can't read '$self->{isisdb}.CNT': $!";          open(fileCNT, $self->{cnt_file}) || croak "can't read '$self->{cnt_file}': $!";
232    
233          # There is two 26 Bytes fixed lenght records          # There is two 26 Bytes fixed lenght records
234    
# Line 197  sub new { Line 252  sub new {
252                  my $buff = shift || return;                  my $buff = shift || return;
253                  my @arr = unpack("ssssssllls", $buff);                  my @arr = unpack("ssssssllls", $buff);
254    
255                  print "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});                  print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
256    
257                  my $IDTYPE = shift @arr;                  my $IDTYPE = shift @arr;
258                  foreach (@flds) {                  foreach (@flds) {
259                          $self->{$IDTYPE}->{$_} = abs(shift @arr);                          $self->{cnt}->{$IDTYPE}->{$_} = abs(shift @arr);
260                  }                  }
261          }          }
262    
263            my $buff;
264    
265          read(fileCNT, $buff, 26);          read(fileCNT, $buff, 26);
266          $self->unpack_cnt($buff);          $self->unpack_cnt($buff);
267    
268          read(fileCNT, $buff, 26);          read(fileCNT, $buff, 26);
269          $self->unpack_cnt($buff);          $self->unpack_cnt($buff);
270    
   
271          close(fileCNT);          close(fileCNT);
272    
273          print Dumper($self) if ($self->{debug});          return $self->{cnt};
   
         # open files for later  
         open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";  
   
         open($self->{'fileMST'}, $self->{isisdb}.".MST") || croak "can't open '$self->{isisdb}.MST': $!";  
   
         $self ? return $self : return undef;  
274  }  }
275    
276  =head2 fetch  =head2 fetch
# Line 231  Read record with selected MFN Line 280  Read record with selected MFN
280    my $rec = $isis->fetch(55);    my $rec = $isis->fetch(55);
281    
282  Returns hash with keys which are field names and values are unpacked values  Returns hash with keys which are field names and values are unpacked values
283  for that field (like C<^asometing^bsomething else>)  for that field like this:
284    
285      $rec = {
286        '210' => [ '^aNew York^cNew York University press^dcop. 1988' ],
287        '990' => [ '2140', '88', 'HAY' ],
288      };
289    
290  =cut  =cut
291    
# Line 240  sub fetch { Line 294  sub fetch {
294    
295          my $mfn = shift || croak "fetch needs MFN as argument!";          my $mfn = shift || croak "fetch needs MFN as argument!";
296    
297          print "fetch: $mfn\n" if ($self->{debug});          # is mfn allready in memory?
298            my $old_mfn = $self->{'current_mfn'} || -1;
299            return $self->{record} if ($mfn == $old_mfn);
300    
301            print STDERR "## fetch: $mfn\n" if ($self->{debug});
302    
303          # XXX check this?          # XXX check this?
304          my $mfnpos=($mfn+int(($mfn-1)/127))*4;          my $mfnpos=($mfn+int(($mfn-1)/127))*4;
305    
306          print "seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});          print STDERR "## seeking to $mfnpos in file '$self->{xrf_file}'\n" if ($self->{debug});
307          seek($self->{'fileXRF'},$mfnpos,0);          seek($self->{'fileXRF'},$mfnpos,0);
308    
309          my $buff;          my $buff;
310    
311            # delete old record
312            delete $self->{record};
313    
314          # read XRFMFB abd XRFMFP          # read XRFMFB abd XRFMFP
315          read($self->{'fileXRF'}, $buff, 4);          read($self->{'fileXRF'}, $buff, 4);
316          my $pointer=unpack("l",$buff) || carp "pointer is null";          my $pointer=unpack("l",$buff) || carp "pointer is null";
317    
318            # check for logically deleted record
319            if ($pointer < 0) {
320                    print STDERR "## record $mfn is logically deleted\n" if ($self->{debug});
321                    $self->{deleted} = $mfn;
322    
323                    return unless $self->{include_deleted};
324    
325                    $pointer = abs($pointer);
326            }
327    
328          my $XRFMFB = int($pointer/2048);          my $XRFMFB = int($pointer/2048);
329          my $XRFMFP = $pointer - ($XRFMFB*2048);          my $XRFMFP = $pointer - ($XRFMFB*2048);
330    
331          print "XRFMFB: $XRFMFB XRFMFP: $XRFMFP\n" if ($self->{debug});          # (XRFMFB - 1) * 512 + XRFMFP
332            # why do i have to do XRFMFP % 1024 ?
333    
334          # XXX fix this to be more readable!!          my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 1024);
         # e.g. (XRFMFB - 1) * 512 + XRFMFP  
   
         my $offset = $pointer;  
         my $offset2=int($offset/2048)-1;  
         my $offset22=int($offset/4096);  
         my $offset3=$offset-($offset22*4096);  
         if ($offset3>512) {  
                 $offset3=$offset3-2048;  
         }  
         my $offset4=($offset2*512)+$offset3;  
335    
336          print "$offset - $offset2 - $offset3 - $offset4\n" if ($self->{debug});          print STDERR "## pointer: $pointer XRFMFB: $XRFMFB XRFMFP: $XRFMFP offset: $blk_off\n" if ($self->{'debug'});
337    
338          # Get Record Information          # Get Record Information
339    
340          seek($self->{'fileMST'},$offset4,0);          seek($self->{'fileMST'},$blk_off,0);
341    
342          read($self->{'fileMST'}, $buff, 4);          read($self->{'fileMST'}, $buff, 4);
343          my $value=unpack("l",$buff);          my $value=unpack("l",$buff);
344    
345            print STDERR "## offset for rowid $value is $blk_off (blk $XRFMFB off $XRFMFP)\n" if ($self->{debug});
346    
347          if ($value!=$mfn) {          if ($value!=$mfn) {
348  print ("Error: The MFN:".$mfn." is not found in MST(".$value.")");                      carp "Error: MFN ".$mfn." not found in MST(".$value.")";    
349                  return -1;      # XXX deleted record?                  #return;
350          }          }
351    
 #       $MFRL=$self->Read16($fileMST);  
 #       $MFBWB=$self->Read32($fileMST);  
 #       $MFBWP=$self->Read16($fileMST);  
 #       $BASE=$self->Read16($fileMST);  
 #       $NVF=$self->Read16($fileMST);  
 #       $STATUS=$self->Read16($fileMST);  
   
352          read($self->{'fileMST'}, $buff, 14);          read($self->{'fileMST'}, $buff, 14);
353    
354          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);
355    
356          print "MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});          print STDERR "## MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
357    
358          # delete old record          warn "MFRL $MFRL is not even number" unless ($MFRL % 2 == 0);
         delete $self->{record};  
359    
360          if (! $self->{'include_deleted'} && $MFRL < 0) {          warn "BASE is not 18+6*NVF" unless ($BASE == 18 + 6 * $NVF);
                 print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});  
                 return;  
         }  
361    
362          # Get Directory Format          # Get Directory Format
363    
# Line 314  print ("Error: The MFN:".$mfn." is not f Line 367  print ("Error: The MFN:".$mfn." is not f
367    
368          read($self->{'fileMST'}, $buff, 6 * $NVF);          read($self->{'fileMST'}, $buff, 6 * $NVF);
369    
370          my $fld_len = 0;          my $rec_len = 0;
371    
372          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
373    
 #               $TAG=$self->Read16($fileMST);  
 #               $POS=$self->Read16($fileMST);  
 #               $LEN=$self->Read16($fileMST);  
   
374                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));
375    
376                  print "TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});                  print STDERR "## TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
377    
378                  # The TAG does not exists in .FDT so we set it to 0.                  # The TAG does not exists in .FDT so we set it to 0.
379                  #                  #
# Line 340  print ("Error: The MFN:".$mfn." is not f Line 389  print ("Error: The MFN:".$mfn." is not f
389                  push @FieldPOS,$POS;                  push @FieldPOS,$POS;
390                  push @FieldLEN,$LEN;                  push @FieldLEN,$LEN;
391    
392                  $fld_len += $LEN;                  $rec_len += $LEN;
393          }          }
394    
395          # Get Variable Fields          # Get Variable Fields
396    
397          read($self->{'fileMST'},$buff,$fld_len);          read($self->{'fileMST'},$buff,$rec_len);
398    
399            print STDERR "## rec_len: $rec_len poc: ",tell($self->{'fileMST'})."\n" if ($self->{debug});
400    
401          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
402                  # skip zero-sized fields                  # skip zero-sized fields
# Line 353  print ("Error: The MFN:".$mfn." is not f Line 404  print ("Error: The MFN:".$mfn." is not f
404    
405                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);
406          }          }
         close(fileMST);  
407    
408          print Dumper($self) if ($self->{debug});          $self->{'current_mfn'} = $mfn;
409    
410            print STDERR Dumper($self),"\n" if ($self->{debug});
411    
412          return $self->{'record'};          return $self->{'record'};
413  }  }
414    
415  =head2 to_ascii  =head2 to_ascii
416    
417  Dump ascii output of selected MFN  Dump ASCII output of record with specified MFN
418    
419      print $isis->to_ascii(42);
420    
421    It outputs something like this:
422    
423    print $isis->to_ascii(55);    210   ^aNew York^cNew York University press^dcop. 1988
424      990   2140
425      990   88
426      990   HAY
427    
428    If C<read_fdt> is specified when calling C<new> it will display field names
429    from C<.FDT> file instead of numeric tags.
430    
431  =cut  =cut
432    
# Line 378  sub to_ascii { Line 440  sub to_ascii {
440          my $out = "0\t$mfn";          my $out = "0\t$mfn";
441    
442          foreach my $f (sort keys %{$rec}) {          foreach my $f (sort keys %{$rec}) {
443                  $out .= "\n$f\t".join("\n$f\t",@{$self->{record}->{$f}});                  my $fn = $self->tag_name($f);
444                    $out .= "\n$fn\t".join("\n$fn\t",@{$self->{record}->{$f}});
445          }          }
446    
447          $out .= "\n";          $out .= "\n";
# Line 388  sub to_ascii { Line 451  sub to_ascii {
451    
452  =head2 to_hash  =head2 to_hash
453    
454  Read mfn and convert it to hash  Read record with specified MFN and convert it to hash
455    
456    my $hash = $isis->to_hash($mfn);    my $hash = $isis->to_hash($mfn);
457    
458  It has ability to convert characters (using C<hash_filter> from ISIS  It has ability to convert characters (using C<hash_filter> from ISIS
459  database before creating structures enabling character remapping or quick  database before creating structures enabling character re-mapping or quick
460  fixup of data.  fix-up of data.
461    
462  This function returns hash which is like this:  This function returns hash which is like this:
463    
# Line 413  This function returns hash which is like Line 476  This function returns hash which is like
476               ],               ],
477    };    };
478    
479  You can later use that has to produce any output from ISIS data.  You can later use that hash to produce any output from ISIS data.
480    
481    If database is created using IsisMarc, it will also have to special fields
482    which will be used for identifiers, C<i1> and C<i2> like this:
483    
484      '200' => [
485                 {
486                   'i1' => '1',
487                   'i2' => ' '
488                   'a' => 'Goa',
489                   'f' => 'Valdo D\'Arienzo',
490                   'e' => 'tipografie e tipografi nel XVI secolo',
491                 }
492               ],
493    
494    This method will also create additional field C<000> with MFN.
495    
496  =cut  =cut
497    
# Line 422  sub to_hash { Line 500  sub to_hash {
500    
501          my $mfn = shift || confess "need mfn!";          my $mfn = shift || confess "need mfn!";
502    
503          my $rec;          # init record to include MFN as field 000
504            my $rec = { '000' => [ $mfn ] };
505    
506          my $row = $self->fetch($mfn);          my $row = $self->fetch($mfn);
507    
508          foreach my $k (keys %{$row}) {          foreach my $k (keys %{$row}) {
# Line 431  sub to_hash { Line 511  sub to_hash {
511                          # filter output                          # filter output
512                          $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});                          $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});
513    
                         # has subfields?  
514                          my $val;                          my $val;
515    
516                            # has identifiers?
517                            ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])\^/\^/);
518    
519                            # has subfields?
520                          if ($l =~ m/\^/) {                          if ($l =~ m/\^/) {
521                                  foreach my $t (split(/\^/,$l)) {                                  foreach my $t (split(/\^/,$l)) {
522                                          next if (! $t);                                          next if (! $t);
# Line 449  sub to_hash { Line 533  sub to_hash {
533          return $rec;          return $rec;
534  }  }
535    
536  #  =head2 tag_name
 # XXX porting from php left-over:  
 #  
 # do I *REALLY* need those methods, or should I use  
 # $self->{something} directly?  
 #  
 # Probably direct usage is better!  
 #  
537    
538  sub TagName {  Return name of selected tag
         my $self = shift;  
         return $self->{TagName};  
 }  
539    
540  sub NextMFN {   print $isis->tag_name('200');
541    
542    =cut
543    
544    sub tag_name {
545          my $self = shift;          my $self = shift;
546          return $self->{NXTMFN};          my $tag = shift || return;
547            return $self->{'TagName'}->{$tag} || $tag;
548  }  }
549    
550  1;  1;
# Line 481  This module has been very lightly tested Line 560  This module has been very lightly tested
560          dpavlin@rot13.org          dpavlin@rot13.org
561          http://www.rot13.org/~dpavlin/          http://www.rot13.org/~dpavlin/
562    
563  This module is based heavily on code from LIBISIS.PHP - Library to read ISIS files V0.1.1  This module is based heavily on code from C<LIBISIS.PHP> library to read ISIS files V0.1.1
564  written in php and (c) 2000 Franck Martin - <franck@sopac.org> released under LGPL.  written in php and (c) 2000 Franck Martin <franck@sopac.org> and released under LGPL.
565    
566  =head1 COPYRIGHT  =head1 COPYRIGHT
567    
# Line 495  LICENSE file included with this module. Line 574  LICENSE file included with this module.
574    
575  =head1 SEE ALSO  =head1 SEE ALSO
576    
577  L<http://www.openisis.org|OpenIsis>, perl(1).  OpenIsis web site L<http://www.openisis.org>
578    
579    perl4lib site L<http://perl4lib.perl.org>
580    
Legend:
Removed from v.12  
changed lines
  Added in v.25
  ViewVC Help
Powered by ViewVC 1.1.26