/[Biblio-Isis]/trunk/lib/Biblio/Isis.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/lib/Biblio/Isis.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 7 by dpavlin, Wed Dec 29 15:10:34 2004 UTC revision 15 by dpavlin, Wed Dec 29 22:46:40 2004 UTC
# Line 7  use Data::Dumper; Line 7  use Data::Dumper;
7  BEGIN {  BEGIN {
8          use Exporter ();          use Exporter ();
9          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
10          $VERSION     = 0.02;          $VERSION     = 0.05;
11          @ISA         = qw (Exporter);          @ISA         = qw (Exporter);
12          #Give a hoot don't pollute, do not export more than needed by default          #Give a hoot don't pollute, do not export more than needed by default
13          @EXPORT      = qw ();          @EXPORT      = qw ();
# Line 18  BEGIN { Line 18  BEGIN {
18    
19  =head1 NAME  =head1 NAME
20    
21  IsisDB - Read CDS/ISIS database  IsisDB - Read CDS/ISIS, WinISIS and IsisMarc database
22    
23  =head1 SYNOPSIS  =head1 SYNOPSIS
24    
25    use IsisDB    use IsisDB;
26    
27    my $isis = new IsisDB(    my $isis = new IsisDB(
28          isisdb => './cds/cds',          isisdb => './cds/cds',
29    );    );
30    
31      for(my $mfn = 1; $mfn <= $isis->{'maxmfn'}; $mfn++) {
32            print $isis->to_ascii($mfn),"\n";
33      }
34    
35  =head1 DESCRIPTION  =head1 DESCRIPTION
36    
37  This module will read CDS/ISIS databases and create hash values out of it.  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or
38  It can be used as perl-only alternative to OpenIsis module.  IsisMarc. It can be used as perl-only alternative to OpenIsis module.
39    
40    It can create hash values from data in ISIS database (using C<to_hash>),
41    ASCII dump (using C<to_ascii>) or just hash with field names and packed
42    values (like C<^asomething^belse>).
43    
44    Unique feature of this module is ability to C<include_deleted> records.
45    It will also skip zero sized fields (OpenIsis has a bug in XS bindings, so
46    fields which are zero sized will be filled with random junk from memory).
47    
48    It also has support for identifiers (only if ISIS database is created by
49    IsisMarc), see C<to_hash>.
50    
51    This will module will always be slower than OpenIsis module which use C
52    library. However, since it's written in perl, it's platform independent (so
53    you don't need C compiler), and can be easily modified. I hope that it
54    creates data structures which are easier to use than ones created by
55    OpenIsis, so reduced time in other parts of the code should compensate for
56    slower performance of this module (speed of reading ISIS database is
57    rarely an issue).
58    
59  =head1 METHODS  =head1 METHODS
60    
# Line 50  It can be used as perl-only alternative Line 74  It can be used as perl-only alternative
74  # some binary reads  # some binary reads
75  #  #
76    
 sub Read32 {  
         my $self = shift;  
   
         my $f = shift || die "Read32 needs file handle";  
         read($$f,$b,4) || die "can't read 4 bytes from $$f from position ".tell($f);  
         return unpack("l",$b);  
 }  
   
77  =head2 new  =head2 new
78    
79  Open CDS/ISIS database  Open ISIS database
80    
81   my $isis = new IsisDB(   my $isis = new IsisDB(
82          isisdb => './cds/cds',          isisdb => './cds/cds',
83          read_fdt => 1,          read_fdt => 1,
84            include_deleted => 1,
85            hash_filter => sub {
86                    my $v = shift;
87                    $v =~ s#foo#bar#g;
88            },
89          debug => 1,          debug => 1,
90   );   );
91    
# Line 74  Options are described below: Line 95  Options are described below:
95    
96  =item isisdb  =item isisdb
97    
98  Prefix path to CDS/ISIS. It should contain full or relative path to database  This is full or relative path to ISIS database files which include
99  and common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.  common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.
100    
101    In this example it uses C<./cds/cds.MST> and related files.
102    
103  =item read_fdt  =item read_fdt
104    
105  Boolean flag to specify if field definition table should be read. It's off  Boolean flag to specify if field definition table should be read. It's off
106  by default.  by default.
107    
108    =item include_deleted
109    
110    Don't skip logically deleted records in ISIS.
111    
112    =item hash_filter
113    
114    Filter code ref which will be used before data is converted to hash.
115    
116  =item debug  =item debug
117    
118  Dump a C<lot> of debugging output.  Dump a B<lot> of debugging output.
119    
120  =back  =back
121    
# Line 97  sub new { Line 128  sub new {
128          my $self = {};          my $self = {};
129          bless($self, $class);          bless($self, $class);
130    
131          $self->{isisdb} = {@_}->{isisdb} || croak "new needs database name as argument!";          croak "new needs database name (isisdb) as argument!" unless ({@_}->{isisdb});
132    
133          $self->{debug} = {@_}->{debug};          foreach my $v (qw{isisdb debug include_deleted hash_filter}) {
134                    $self->{$v} = {@_}->{$v};
135            }
136    
137          # if you want to read .FDT file use read_fdt argument when creating class!          # if you want to read .FDT file use read_fdt argument when creating class!
138          if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {          if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {
# Line 140  sub new { Line 173  sub new {
173          # NXTMFP        offset to next available position in last block          # NXTMFP        offset to next available position in last block
174          # MFTYPE        always 0 for user db file (1 for system)          # MFTYPE        always 0 for user db file (1 for system)
175          seek(fileMST,4,0);          seek(fileMST,4,0);
176          $self->{'NXTMFN'}=$self->Read32(\*fileMST) || carp "NXTNFN is zero";  
177            my $buff;
178    
179            read(fileMST, $buff, 4);
180            $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";
181    
182          # save maximum MFN          # save maximum MFN
183          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;
# Line 173  sub new { Line 210  sub new {
210                  my $buff = shift || return;                  my $buff = shift || return;
211                  my @arr = unpack("ssssssllls", $buff);                  my @arr = unpack("ssssssllls", $buff);
212    
213                    print "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
214    
215                  my $IDTYPE = shift @arr;                  my $IDTYPE = shift @arr;
216                  foreach (@flds) {                  foreach (@flds) {
217                          $self->{$IDTYPE}->{$_} = abs(shift @arr);                          $self->{$IDTYPE}->{$_} = abs(shift @arr);
218                  }                  }
219          }          }
220    
         my $buff;  
221          read(fileCNT, $buff, 26);          read(fileCNT, $buff, 26);
222          $self->unpack_cnt($buff);          $self->unpack_cnt($buff);
223    
# Line 189  sub new { Line 227  sub new {
227    
228          close(fileCNT);          close(fileCNT);
229    
230          print Dumper($self) if ($self->{debug});          print Dumper($self),"\n" if ($self->{debug});
231    
232          # open files for later          # open files for later
233          open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";          open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";
# Line 206  Read record with selected MFN Line 244  Read record with selected MFN
244    my $rec = $isis->fetch(55);    my $rec = $isis->fetch(55);
245    
246  Returns hash with keys which are field names and values are unpacked values  Returns hash with keys which are field names and values are unpacked values
247  for that field.  for that field like this:
248    
249      $rec = {
250        '210' => [ '^aNew York^cNew York University press^dcop. 1988' ],
251        '990' => [ '2140', '88', 'HAY' ],
252      };
253    
254  =cut  =cut
255    
# Line 223  sub fetch { Line 266  sub fetch {
266          print "seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});          print "seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});
267          seek($self->{'fileXRF'},$mfnpos,0);          seek($self->{'fileXRF'},$mfnpos,0);
268    
269            my $buff;
270    
271          # read XRFMFB abd XRFMFP          # read XRFMFB abd XRFMFP
272          my $pointer=$self->Read32(\*{$self->{'fileXRF'}});          read($self->{'fileXRF'}, $buff, 4);
273            my $pointer=unpack("l",$buff) || carp "pointer is null";
274    
275          my $XRFMFB = int($pointer/2048);          my $XRFMFB = int($pointer/2048);
276          my $XRFMFP = $pointer - ($XRFMFB*2048);          my $XRFMFP = $pointer - ($XRFMFB*2048);
# Line 249  sub fetch { Line 295  sub fetch {
295    
296          seek($self->{'fileMST'},$offset4,0);          seek($self->{'fileMST'},$offset4,0);
297    
298          my $value=$self->Read32(\*{$self->{'fileMST'}});          read($self->{'fileMST'}, $buff, 4);
299            my $value=unpack("l",$buff);
300    
301          if ($value!=$mfn) {          if ($value!=$mfn) {
302  print ("Error: The MFN:".$mfn." is not found in MST(".$value.")");      print ("Error: The MFN:".$mfn." is not found in MST(".$value.")");    
# Line 263  print ("Error: The MFN:".$mfn." is not f Line 310  print ("Error: The MFN:".$mfn." is not f
310  #       $NVF=$self->Read16($fileMST);  #       $NVF=$self->Read16($fileMST);
311  #       $STATUS=$self->Read16($fileMST);  #       $STATUS=$self->Read16($fileMST);
312    
         my $buff;  
313          read($self->{'fileMST'}, $buff, 14);          read($self->{'fileMST'}, $buff, 14);
314    
315          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);
316    
317          print "MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});          print "MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
318    
319            # delete old record
320            delete $self->{record};
321    
322            if (! $self->{'include_deleted'} && $MFRL < 0) {
323                    print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});
324                    return;
325            }
326    
327          # Get Directory Format          # Get Directory Format
328    
329          my @FieldPOS;          my @FieldPOS;
330          my @FieldLEN;          my @FieldLEN;
331          my @FieldTAG;          my @FieldTAG;
332    
333            read($self->{'fileMST'}, $buff, 6 * $NVF);
334    
335            my $fld_len = 0;
336    
337          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
338    
339  #               $TAG=$self->Read16($fileMST);  #               $TAG=$self->Read16($fileMST);
340  #               $POS=$self->Read16($fileMST);  #               $POS=$self->Read16($fileMST);
341  #               $LEN=$self->Read16($fileMST);  #               $LEN=$self->Read16($fileMST);
342    
343                  read($self->{'fileMST'}, $buff, 6);                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));
                 my ($TAG,$POS,$LEN) = unpack("sss", $buff);  
344    
345                  print "TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});                  print "TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
346    
# Line 300  print ("Error: The MFN:".$mfn." is not f Line 357  print ("Error: The MFN:".$mfn." is not f
357                  push @FieldTAG,$TAG;                  push @FieldTAG,$TAG;
358                  push @FieldPOS,$POS;                  push @FieldPOS,$POS;
359                  push @FieldLEN,$LEN;                  push @FieldLEN,$LEN;
360    
361                    $fld_len += $LEN;
362          }          }
363    
364          # Get Variable Fields          # Get Variable Fields
365    
366          delete $self->{record};          read($self->{'fileMST'},$buff,$fld_len);
367    
368          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
369                  my $rec;                  # skip zero-sized fields
370                  read($self->{'fileMST'},$rec,$FieldLEN[$i]);                  next if ($FieldLEN[$i] == 0);
                 push @{$self->{record}->{$FieldTAG[$i]}}, $rec;  
         }  
         close(fileMST);  
371    
372          # The record is marked for deletion                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);
         if ($STATUS==1) {  
                 return -1;  
373          }          }
374            close(fileMST);
375    
376          print Dumper($self) if ($self->{debug});          print Dumper($self),"\n" if ($self->{debug});
377    
378          return $self->{'record'};          return $self->{'record'};
379  }  }
380    
381  =head2 to_ascii  =head2 to_ascii
382    
383  Dump ascii output of selected MFN  Dump ASCII output of record with specified MFN
384    
385    print $isis->to_ascii(55);    print $isis->to_ascii(42);
386    
387    It outputs something like this:
388    
389      210   ^aNew York^cNew York University press^dcop. 1988
390      990   2140
391      990   88
392      990   HAY
393    
394    If C<read_fdt> is specified when calling C<new> it will display field names
395    from C<.FDT> file instead of numeric tags.
396    
397  =cut  =cut
398    
# Line 341  sub to_ascii { Line 406  sub to_ascii {
406          my $out = "0\t$mfn";          my $out = "0\t$mfn";
407    
408          foreach my $f (sort keys %{$rec}) {          foreach my $f (sort keys %{$rec}) {
409                  $out .= "\n$f\t".join("\n$f\t",@{$self->{record}->{$f}});                  my $fn = $self->tag_name($f);
410                    $out .= "\n$fn\t".join("\n$fn\t",@{$self->{record}->{$f}});
411          }          }
412    
413          $out .= "\n";          $out .= "\n";
# Line 349  sub to_ascii { Line 415  sub to_ascii {
415          return $out;          return $out;
416  }  }
417    
418  #  =head2 to_hash
419  # XXX porting from php left-over:  
420  #  Read record with specified MFN and convert it to hash
 # do I *REALLY* need those methods, or should I use  
 # $self->{something} directly?  
 #  
 # Probably direct usage is better!  
 #  
421    
422  sub TagName {    my $hash = $isis->to_hash($mfn);
423    
424    It has ability to convert characters (using C<hash_filter> from ISIS
425    database before creating structures enabling character re-mapping or quick
426    fix-up of data.
427    
428    This function returns hash which is like this:
429    
430      $hash = {
431        '210' => [
432                   {
433                     'c' => 'New York University press',
434                     'a' => 'New York',
435                     'd' => 'cop. 1988'
436                   }
437                 ],
438        '990' => [
439                   '2140',
440                   '88',
441                   'HAY'
442                 ],
443      };
444    
445    You can later use that hash to produce any output from ISIS data.
446    
447    If database is created using IsisMarc, it will also have to special fields
448    which will be used for identifiers, C<i1> and C<i2> like this:
449    
450      '200' => [
451                 {
452                   'i1' => '1',
453                   'i2' => ' '
454                   'a' => 'Goa',
455                   'f' => 'Valdo D\'Arienzo',
456                   'e' => 'tipografie e tipografi nel XVI secolo',
457                 }
458               ],
459    
460    This method will also create additional field C<000> with MFN.
461    
462    =cut
463    
464    sub to_hash {
465          my $self = shift;          my $self = shift;
466          return $self->{TagName};  
467            my $mfn = shift || confess "need mfn!";
468    
469            # init record to include MFN as field 000
470            my $rec = { '000' => $mfn };
471    
472            my $row = $self->fetch($mfn);
473    
474            foreach my $k (keys %{$row}) {
475                    foreach my $l (@{$row->{$k}}) {
476    
477                            # filter output
478                            $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});
479    
480                            my $val;
481    
482                            # has identifiers?
483                            ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])//);
484    
485                            # has subfields?
486                            if ($l =~ m/\^/) {
487                                    foreach my $t (split(/\^/,$l)) {
488                                            next if (! $t);
489                                            $val->{substr($t,0,1)} = substr($t,1);
490                                    }
491                            } else {
492                                    $val = $l;
493                            }
494    
495                            push @{$rec->{$k}}, $val;
496                    }
497            }
498    
499            return $rec;
500  }  }
501    
502  sub NextMFN {  =head2 tag_name
503    
504    Return name of selected tag
505    
506     print $isis->tag_name('200');
507    
508    =cut
509    
510    sub tag_name {
511          my $self = shift;          my $self = shift;
512          return $self->{NXTMFN};          my $tag = shift || return;
513            return $self->{'TagName'}->{$tag} || $tag;
514  }  }
515    
516  1;  1;
# Line 381  This module has been very lightly tested Line 526  This module has been very lightly tested
526          dpavlin@rot13.org          dpavlin@rot13.org
527          http://www.rot13.org/~dpavlin/          http://www.rot13.org/~dpavlin/
528    
529  This module is based heavily on code from LIBISIS.PHP - Library to read ISIS files V0.1.1  This module is based heavily on code from C<LIBISIS.PHP> library to read ISIS files V0.1.1
530  written in php and (c) 2000 Franck Martin - <franck@sopac.org> released under LGPL.  written in php and (c) 2000 Franck Martin <franck@sopac.org> and released under LGPL.
531    
532  =head1 COPYRIGHT  =head1 COPYRIGHT
533    
# Line 395  LICENSE file included with this module. Line 540  LICENSE file included with this module.
540    
541  =head1 SEE ALSO  =head1 SEE ALSO
542    
543  L<http://www.openisis.org|OpenIsis>, perl(1).  OpenIsis web site L<http://www.openisis.org>
544    
545    perl4lib site L<http://perl4lib.perl.org>
546    
Legend:
Removed from v.7  
changed lines
  Added in v.15
  ViewVC Help
Powered by ViewVC 1.1.26