/[webpac2]/trunk/run.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/run.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 217 - (show annotations)
Mon Dec 5 17:47:51 2005 UTC (18 years, 3 months ago) by dpavlin
File MIME type: text/plain
File size: 2533 byte(s)
 r11536@llin:  dpavlin | 2005-12-05 15:29:47 +0100
 change on load_ds and save_ds which not accept ONLY hash (and optional
 database name if not specified when calling new WebPAC::Store)

1 #!/usr/bin/perl -w
2
3 use strict;
4
5 use Cwd qw/abs_path/;
6 use File::Temp qw/tempdir/;
7 use Data::Dumper;
8 use lib './lib';
9
10 use WebPAC::Lookup;
11 use WebPAC::Input::ISIS;
12 use WebPAC::Store 0.03;
13 use WebPAC::Normalize::XML;
14 use WebPAC::Output::TT;
15 use WebPAC::Output::Estraier 0.02;
16 use YAML qw/LoadFile/;
17 use LWP::Simple;
18
19 my $limit = shift @ARGV;
20
21 my $config = LoadFile('conf/config.yml');
22
23 print "config = ",Dumper($config);
24
25 die "no databases in config file!\n" unless ($config->{databases});
26
27 my $total_rows = 0;
28
29 while (my ($database, $db_config) = each %{ $config->{databases} }) {
30
31 my $type = lc($db_config->{input}->{type});
32
33 die "I know only how to handle input type isis, not '$type'!\n" unless ($type eq 'isis');
34
35 my $abs_path = abs_path($0);
36 $abs_path =~ s#/[^/]*$#/#;
37
38 my $lookup = new WebPAC::Lookup(
39 lookup_file => $db_config->{input}->{lookup},
40 );
41
42 my $db_path = $config->{webpac}->{db_path} . '/' . $database;
43
44
45 my $log = $lookup->_get_logger;
46 $log->info("working on $database in $db_path");
47
48 my $db = new WebPAC::Store(
49 path => $db_path,
50 database => $database,
51 );
52
53 my $est_config = $config->{hyperestraier} || $log->logdie("can't find 'hyperestraier' part in confguration");
54 $est_config->{database} = $database;
55
56 $log->info("using HyperEstraier URL $est_config->{masterurl}");
57
58 my $est = new WebPAC::Output::Estraier(
59 %{ $est_config },
60 );
61
62 #
63 # now, iterate through input formats
64 #
65
66 my @inputs;
67 if (ref($db_config->{input}) eq 'ARRAY') {
68 @inputs = @{ $db_config->{input} };
69 } else {
70 push @inputs, $db_config->{input};
71 }
72
73 foreach my $input (@inputs) {
74 $log->info("working on input $input->{path} [$input->{type}]");
75
76 my $isis = new WebPAC::Input::ISIS(
77 code_page => $config->{webpac}->{webpac_encoding},
78 limit_mfn => $input->{limit},
79 );
80
81 my $maxmfn = $isis->open(
82 filename => $input->{path},
83 code_page => $input->{encoding}, # database encoding
84 );
85
86 my $n = new WebPAC::Normalize::XML(
87 # filter => { 'foo' => sub { shift } },
88 db => $db,
89 lookup_regex => $lookup->regex,
90 lookup => $lookup,
91 );
92
93 $n->open(
94 tag => $input->{normalize}->{tag},
95 xml_file => $input->{normalize}->{path},
96 );
97
98 for ( 0 ... $isis->size ) {
99
100 my $row = $isis->fetch || next;
101
102 my $mfn = $row->{'000'}->[0] || die "can't find MFN";
103
104 my $ds = $n->data_structure($row);
105
106 $est->add(
107 id => $input->{name} . "#" . $mfn,
108 ds => $ds,
109 type => $config->{hyperestraier}->{type},
110 );
111
112 $total_rows++;
113 }
114
115 };
116
117 $log->info("$total_rows records indexed");
118 }
119

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26