WebPAC/Input/ISIS.pm

package WebPAC::Input::ISIS;

use warnings;
use strict;

use WebPAC::Common;
use base qw/WebPAC::Input WebPAC::Common/;
use Text::Iconv;

=head1 NAME

WebPAC::Input::ISIS - support for CDS/ISIS source files

=head1 VERSION

Version 0.01

=cut

our $VERSION = '0.01';


# auto-configure

my ($have_biblio_isis, $have_openisis) = (0,0);

eval "use Biblio::Isis 0.13;";
unless ($@) { 
        $have_biblio_isis = 1
} else {
        eval "use OpenIsis;";
        $have_openisis = 1 unless ($@);
}

=head1 SYNOPSIS

Open CDS/ISIS, WinISIS or IsisMarc database using Biblio::Isis or OpenIsis
module and read all records to memory.

 my $isis = new WebPAC::Input::ISIS();
 $isis->open( filename => '/path/to/ISIS/ISIS' );

=head1 FUNCTIONS

=head2 open

This function will read whole database in memory and produce lookups.

 $isis->open(
        filename => '/data/ISIS/ISIS',
        code_page => '852',
        limit_mfn => 500,
        start_mfn => 6000,
        lookup => $lookup_obj,
 );

By default, ISIS code page is assumed to be C<852>.

If optional parametar C<start_mfn> is set, this will be first MFN to read
from database (so you can skip beginning of your database if you need to).

If optional parametar C<limit_mfn> is set, it will read just 500 records
from database in example above.

Returns size of database, regardless of C<start_mfn> and C<limit_mfn>
parametars, see also C<$isis->size>.

=cut

sub open {
        my $self = shift;
        my $arg = {@_};

        my $log = $self->_get_logger();

        $log->logcroak("need filename") if (! $arg->{'filename'});
        my $code_page = $arg->{'code_page'} || '852';

        #$log->logdie("can't find database ",$arg->{'filename'}) unless (glob($arg->{'filename'}.'.*'));

        # store data in object
        $self->{'isis_code_page'} = $code_page;
        foreach my $v (qw/isis_filename start_mfn limit_mfn/) {
                $self->{$v} = $arg->{$v} if ($arg->{$v});
        }

        # create Text::Iconv object
        my $cp = Text::Iconv->new($code_page,$self->{'code_page'});

        $log->info("reading ISIS database '",$arg->{'filename'},"'");
        $log->debug("isis code page: $code_page");

        my ($isis_db,$db_size);

        if ($have_openisis) {
                $log->debug("using OpenIsis perl bindings");
                $isis_db = OpenIsis::open($arg->{'filename'});
                $db_size = OpenIsis::maxRowid( $isis_db ) || 1;
        } elsif ($have_biblio_isis) {
                $log->debug("using Biblio::Isis");
                use Biblio::Isis;
                $isis_db = new Biblio::Isis(
                        isisdb => $arg->{'filename'},
                        include_deleted => 1,
                        hash_filter => sub {
                                my $l = shift || return;
                                $l = $cp->convert($l);
                                return $l;
                        },
                );
                $db_size = $isis_db->count;

                unless ($db_size) {
                        $log->logwarn("no records in database ", $arg->{'filename'}, ", skipping...");
                        return;
                }

        } else {
                $log->logdie("Can't find supported ISIS library for perl. I suggent that you install Bilbio::Isis from CPAN.");
        }


        my $startmfn = 1;
        my $maxmfn = $db_size;

        if (my $s = $self->{'start_mfn'}) {
                $log->info("skipping to MFN $s");
                $startmfn = $s;
        } else {
                $self->{'start_mfn'} = $startmfn;
        }

        if ($self->{limit_mfn}) {
                $log->info("limiting to ",$self->{limit_mfn}," records");
                $maxmfn = $startmfn + $self->{limit_mfn} - 1;
                $maxmfn = $db_size if ($maxmfn > $db_size);
        }

        # store size for later
        $self->{'size'} = ($maxmfn - $startmfn) ? ($maxmfn - $startmfn + 1) : 0;

        $log->info("processing ",($maxmfn-$startmfn)." records using ",( $have_openisis ? 'OpenIsis' : 'Biblio::Isis'));


        # read database
        for (my $mfn = $startmfn; $mfn <= $maxmfn; $mfn++) {

                $log->debug("mfn: $mfn\n");

                my $rec;

                if ($have_openisis) {

                        # read record using OpenIsis
                        my $row = OpenIsis::read( $isis_db, $mfn );
                        foreach my $k (keys %{$row}) {
                                if ($k ne "mfn") {
                                        foreach my $l (@{$row->{$k}}) {
                                                $l = $cp->convert($l);
                                                # has subfields?
                                                my $val;
                                                if ($l =~ m/\^/) {
                                                        foreach my $t (split(/\^/,$l)) {
                                                                next if (! $t);
                                                                $val->{substr($t,0,1)} = substr($t,1);
                                                        }
                                                } else {
                                                        $val = $l;
                                                }

                                                push @{$rec->{$k}}, $val;
                                        }
                                } else {
                                        push @{$rec->{'000'}}, $mfn;
                                }
                        }

                } elsif ($have_biblio_isis) {
                        $rec = $isis_db->to_hash($mfn);
                } else {
                        $log->logdie("hum? implementation missing?");
                }

                if (! $rec) {
                        $log->warn("record $mfn empty? skipping...");
                        next;
                }

                # store
                if ($self->{'low_mem'}) {
                        $self->{'db'}->put($mfn, $rec);
                } else {
                        $self->{'data'}->{$mfn} = $rec;
                }

                # create lookup
                $self->{'lookup'}->add( $rec ) if ($rec && $self->{'lookup'});

                $self->progress_bar($mfn,$maxmfn);

        }

        $self->{'current_mfn'} = -1;
        $self->{'last_pcnt'} = 0;

        $log->debug("max mfn: $maxmfn");

        # store max mfn and return it.
        $self->{'max_mfn'} = $maxmfn;

        return $db_size;
}

=head2 fetch

Fetch next record from database. It will also displays progress bar.

 my $rec = $isis->fetch;

Record from this function should probably go to C<data_structure> for
normalisation.

=cut

sub fetch {
        my $self = shift;

        my $log = $self->_get_logger();

        $log->logconfess("it seems that you didn't load database!") unless ($self->{'current_mfn'});

        if ($self->{'current_mfn'} == -1) {
                $self->{'current_mfn'} = $self->{'start_mfn'};
        } else {
                $self->{'current_mfn'}++;
        }

        my $mfn = $self->{'current_mfn'};

        if ($mfn > $self->{'max_mfn'}) {
                $self->{'current_mfn'} = $self->{'max_mfn'};
                $log->debug("at EOF");
                return;
        }

        $self->progress_bar($mfn,$self->{'max_mfn'});

        my $rec;

        if ($self->{'low_mem'}) {
                $rec = $self->{'db'}->get($mfn);
        } else {
                $rec = $self->{'data'}->{$mfn};
        }

        $rec ||= 0E0;
}

=head2 pos

Returns current record number (MFN).

 print $isis->pos;

First record in database has position 1.

=cut

sub pos {
        my $self = shift;
        return $self->{'current_mfn'};
}


=head2 size

Returns number of records in database

 print $isis->size;

Result from this function can be used to loop through all records

 foreach my $mfn ( 1 ... $isis->size ) { ... }

because it takes into account C<start_mfn> and C<limit_mfn>.

=cut

sub size {
        my $self = shift;
        return $self->{'size'};
}

=head2 seek

Seek to specified MFN in file.

 $isis->seek(42);

First record in database has position 1.

=cut

sub seek {
        my $self = shift;
        my $pos = shift || return;

        my $log = $self->_get_logger();

        if ($pos < 1) {
                $log->warn("seek before first record");
                $pos = 1;
        } elsif ($pos > $self->{'max_mfn'}) {
                $log->warn("seek beyond last record");
                $pos = $self->{'max_mfn'};
        }

        return $self->{'current_mfn'} = (($pos - 1) || -1);
}

=head1 AUTHOR

Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>

=head1 COPYRIGHT & LICENSE

Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.

This program is free software; you can redistribute it and/or modify it
under the same terms as Perl itself.

=cut

1; # End of WebPAC::Input::ISIS
1	dpavlin	6	package WebPAC::Input::ISIS;
2
3			use warnings;
4			use strict;
5
6			use WebPAC::Common;
7			use base qw/WebPAC::Input WebPAC::Common/;
8	dpavlin	9	use Text::Iconv;
9	dpavlin	6
10			=head1 NAME
11
12			WebPAC::Input::ISIS - support for CDS/ISIS source files
13
14			=head1 VERSION
15
16			Version 0.01
17
18			=cut
19
20			our $VERSION = '0.01';
21
22
23			# auto-configure
24
25			my ($have_biblio_isis, $have_openisis) = (0,0);
26
27			eval "use Biblio::Isis 0.13;";
28			unless ($@) {
29			$have_biblio_isis = 1
30			} else {
31			eval "use OpenIsis;";
32			$have_openisis = 1 unless ($@);
33			}
34
35			=head1 SYNOPSIS
36
37			Open CDS/ISIS, WinISIS or IsisMarc database using Biblio::Isis or OpenIsis
38			module and read all records to memory.
39
40			my $isis = new WebPAC::Input::ISIS();
41			$isis->open( filename => '/path/to/ISIS/ISIS' );
42
43			=head1 FUNCTIONS
44
45			=head2 open
46
47			This function will read whole database in memory and produce lookups.
48
49			$isis->open(
50			filename => '/data/ISIS/ISIS',
51			code_page => '852',
52			limit_mfn => 500,
53			start_mfn => 6000,
54			lookup => $lookup_obj,
55			);
56
57			By default, ISIS code page is assumed to be C<852>.
58
59			If optional parametar C<start_mfn> is set, this will be first MFN to read
60			from database (so you can skip beginning of your database if you need to).
61
62			If optional parametar C<limit_mfn> is set, it will read just 500 records
63			from database in example above.
64
65	dpavlin	21	Returns size of database, regardless of C<start_mfn> and C<limit_mfn>
66			parametars, see also C<$isis->size>.
67	dpavlin	6
68			=cut
69
70			sub open {
71			my $self = shift;
72			my $arg = {@_};
73
74			my $log = $self->_get_logger();
75
76			$log->logcroak("need filename") if (! $arg->{'filename'});
77			my $code_page = $arg->{'code_page'} \|\| '852';
78
79	dpavlin	251	#$log->logdie("can't find database ",$arg->{'filename'}) unless (glob($arg->{'filename'}.'.*'));
80	dpavlin	6
81			# store data in object
82			$self->{'isis_code_page'} = $code_page;
83	dpavlin	21	foreach my $v (qw/isis_filename start_mfn limit_mfn/) {
84			$self->{$v} = $arg->{$v} if ($arg->{$v});
85			}
86	dpavlin	6
87			# create Text::Iconv object
88			my $cp = Text::Iconv->new($code_page,$self->{'code_page'});
89
90			$log->info("reading ISIS database '",$arg->{'filename'},"'");
91			$log->debug("isis code page: $code_page");
92
93	dpavlin	21	my ($isis_db,$db_size);
94	dpavlin	6
95			if ($have_openisis) {
96			$log->debug("using OpenIsis perl bindings");
97			$isis_db = OpenIsis::open($arg->{'filename'});
98	dpavlin	21	$db_size = OpenIsis::maxRowid( $isis_db ) \|\| 1;
99	dpavlin	6	} elsif ($have_biblio_isis) {
100			$log->debug("using Biblio::Isis");
101			use Biblio::Isis;
102			$isis_db = new Biblio::Isis(
103			isisdb => $arg->{'filename'},
104			include_deleted => 1,
105			hash_filter => sub {
106			my $l = shift \|\| return;
107			$l = $cp->convert($l);
108			return $l;
109			},
110			);
111	dpavlin	21	$db_size = $isis_db->count;
112	dpavlin	6
113	dpavlin	21	unless ($db_size) {
114	dpavlin	6	$log->logwarn("no records in database ", $arg->{'filename'}, ", skipping...");
115			return;
116			}
117
118			} else {
119			$log->logdie("Can't find supported ISIS library for perl. I suggent that you install Bilbio::Isis from CPAN.");
120			}
121
122
123			my $startmfn = 1;
124	dpavlin	21	my $maxmfn = $db_size;
125	dpavlin	6
126			if (my $s = $self->{'start_mfn'}) {
127			$log->info("skipping to MFN $s");
128			$startmfn = $s;
129			} else {
130			$self->{'start_mfn'} = $startmfn;
131			}
132
133	dpavlin	21	if ($self->{limit_mfn}) {
134			$log->info("limiting to ",$self->{limit_mfn}," records");
135			$maxmfn = $startmfn + $self->{limit_mfn} - 1;
136			$maxmfn = $db_size if ($maxmfn > $db_size);
137			}
138	dpavlin	6
139	dpavlin	21	# store size for later
140			$self->{'size'} = ($maxmfn - $startmfn) ? ($maxmfn - $startmfn + 1) : 0;
141
142	dpavlin	6	$log->info("processing ",($maxmfn-$startmfn)." records using ",( $have_openisis ? 'OpenIsis' : 'Biblio::Isis'));
143
144
145			# read database
146			for (my $mfn = $startmfn; $mfn <= $maxmfn; $mfn++) {
147
148			$log->debug("mfn: $mfn\n");
149
150			my $rec;
151
152			if ($have_openisis) {
153
154			# read record using OpenIsis
155			my $row = OpenIsis::read( $isis_db, $mfn );
156			foreach my $k (keys %{$row}) {
157			if ($k ne "mfn") {
158			foreach my $l (@{$row->{$k}}) {
159			$l = $cp->convert($l);
160			# has subfields?
161			my $val;
162			if ($l =~ m/\^/) {
163			foreach my $t (split(/\^/,$l)) {
164			next if (! $t);
165			$val->{substr($t,0,1)} = substr($t,1);
166			}
167			} else {
168			$val = $l;
169			}
170
171			push @{$rec->{$k}}, $val;
172			}
173			} else {
174			push @{$rec->{'000'}}, $mfn;
175			}
176			}
177
178			} elsif ($have_biblio_isis) {
179			$rec = $isis_db->to_hash($mfn);
180			} else {
181			$log->logdie("hum? implementation missing?");
182			}
183
184	dpavlin	113	if (! $rec) {
185			$log->warn("record $mfn empty? skipping...");
186			next;
187			}
188	dpavlin	6
189			# store
190			if ($self->{'low_mem'}) {
191			$self->{'db'}->put($mfn, $rec);
192			} else {
193			$self->{'data'}->{$mfn} = $rec;
194			}
195
196			# create lookup
197	dpavlin	251	$self->{'lookup'}->add( $rec ) if ($rec && $self->{'lookup'});
198	dpavlin	6
199			$self->progress_bar($mfn,$maxmfn);
200
201			}
202
203			$self->{'current_mfn'} = -1;
204			$self->{'last_pcnt'} = 0;
205
206			$log->debug("max mfn: $maxmfn");
207
208			# store max mfn and return it.
209	dpavlin	21	$self->{'max_mfn'} = $maxmfn;
210
211			return $db_size;
212	dpavlin	6	}
213
214	dpavlin	10	=head2 fetch
215	dpavlin	6
216			Fetch next record from database. It will also displays progress bar.
217
218	dpavlin	10	my $rec = $isis->fetch;
219	dpavlin	6
220	dpavlin	10	Record from this function should probably go to C<data_structure> for
221			normalisation.
222	dpavlin	8
223	dpavlin	6	=cut
224
225	dpavlin	10	sub fetch {
226	dpavlin	6	my $self = shift;
227
228			my $log = $self->_get_logger();
229
230			$log->logconfess("it seems that you didn't load database!") unless ($self->{'current_mfn'});
231
232			if ($self->{'current_mfn'} == -1) {
233			$self->{'current_mfn'} = $self->{'start_mfn'};
234			} else {
235			$self->{'current_mfn'}++;
236			}
237
238			my $mfn = $self->{'current_mfn'};
239
240			if ($mfn > $self->{'max_mfn'}) {
241			$self->{'current_mfn'} = $self->{'max_mfn'};
242			$log->debug("at EOF");
243			return;
244			}
245
246			$self->progress_bar($mfn,$self->{'max_mfn'});
247
248	dpavlin	113	my $rec;
249
250	dpavlin	6	if ($self->{'low_mem'}) {
251	dpavlin	113	$rec = $self->{'db'}->get($mfn);
252	dpavlin	6	} else {
253	dpavlin	113	$rec = $self->{'data'}->{$mfn};
254	dpavlin	6	}
255	dpavlin	113
256			$rec \|\|= 0E0;
257	dpavlin	6	}
258
259	dpavlin	10	=head2 pos
260	dpavlin	8
261	dpavlin	10	Returns current record number (MFN).
262	dpavlin	8
263	dpavlin	10	print $isis->pos;
264	dpavlin	8
265	dpavlin	10	First record in database has position 1.
266
267	dpavlin	8	=cut
268
269	dpavlin	10	sub pos {
270	dpavlin	8	my $self = shift;
271	dpavlin	10	return $self->{'current_mfn'};
272			}
273	dpavlin	8
274	dpavlin	10
275			=head2 size
276
277			Returns number of records in database
278
279			print $isis->size;
280
281	dpavlin	21	Result from this function can be used to loop through all records
282
283			foreach my $mfn ( 1 ... $isis->size ) { ... }
284
285			because it takes into account C<start_mfn> and C<limit_mfn>.
286
287	dpavlin	10	=cut
288
289			sub size {
290			my $self = shift;
291	dpavlin	21	return $self->{'size'};
292	dpavlin	8	}
293
294	dpavlin	10	=head2 seek
295	dpavlin	6
296	dpavlin	10	Seek to specified MFN in file.
297	dpavlin	6
298	dpavlin	10	$isis->seek(42);
299	dpavlin	6
300	dpavlin	10	First record in database has position 1.
301
302	dpavlin	6	=cut
303
304	dpavlin	10	sub seek {
305	dpavlin	6	my $self = shift;
306	dpavlin	10	my $pos = shift \|\| return;
307
308			my $log = $self->_get_logger();
309
310			if ($pos < 1) {
311			$log->warn("seek before first record");
312			$pos = 1;
313			} elsif ($pos > $self->{'max_mfn'}) {
314			$log->warn("seek beyond last record");
315			$pos = $self->{'max_mfn'};
316			}
317
318			return $self->{'current_mfn'} = (($pos - 1) \|\| -1);
319	dpavlin	6	}
320
321			=head1 AUTHOR
322
323			Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
324
325			=head1 COPYRIGHT & LICENSE
326
327			Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.
328
329			This program is free software; you can redistribute it and/or modify it
330			under the same terms as Perl itself.
331
332			=cut
333
334			1; # End of WebPAC::Input::ISIS