Annotation of /cvs-head/script/index_ora

#!/usr/bin/perl -w
#                              -*- Mode: Perl -*- 
# $Basename$
# $Revision: 1.9 $
# Author          : Ulrich Pfeifer
# Created On      : Mon Dec 31 13:57:11 2001
# Last Modified By: Ulrich Pfeifer
# Last Modified On: Fri Jan  4 15:59:20 2002
# Language        : CPerl
# 
# (C) Copyright 2001, UUNET Deutschland GmbH, Germany
# 

use 5.007;

use strict;
use Devel::Peek qw(Dump);

use File::Path;
use DB_File;
use Getopt::Long;
use Cwd;

BEGIN {require WAIT::Config;}
use WAIT::Database;
use WAIT::Parse::Ora;
use WAIT::Document::Ora;
use WAIT::InvertedIndex;


$DB_BTREE->{'cachesize'} = 200_000 ;

my %OPT = (
           database => 'DB',
           dir      => $WAIT::Config->{WAIT_home} || '/tmp',
           table    => 'ora',
          );

GetOptions(\%OPT,
           'database=s',
           'dir=s',
           'table=s',
          ) || die "Usage: ...\n";

my @localtime = localtime;
$localtime[5] += 1900;
$localtime[4]++;
my $jobid = sprintf "%04s-%02s-%02s_%02s:%02s_%d", @localtime[5,4,3,2,1], $$;
my $db = WAIT::Database->create(name      => "$OPT{database}-$jobid",
                                directory => $OPT{dir})
    or die "Could not create database $OPT{database}: $@\n";

my $layout = new WAIT::Parse::Ora;

use lib "/usr/local/apache/lib";
use oreilly_de_catalog::wait_handler;

my $stem  = ['OR_tr_20020124', 'OR_lc_20020124', 'split2', 'stop', 'Stem'];
my $text  = [{
              'prefix'    => ['OR_tr_20020124', 'OR_lc_20020124'],
              'intervall' => ['OR_tr_20020124', 'OR_lc_20020124'],
             },
             'OR_tr_20020124', 'OR_lc_20020124', 'split2', 'stop'];
my $sound = ['OR_tr_20020124', 'OR_lc_20020124', 'split2', 'Soundex'];
my $trigr = ['OR_lc_20020124', 'OR_trigrams_20020125'];
my $isbn  = ['split6', 'OR_isbn_20020127'];

my $cwd = cwd;

my %D;
my $access = tie %D, 'WAIT::Document::Ora', @ARGV,
  or die "Couldn't tie to file: $!\n";

my $tb = $db->create_table(name     => $OPT{table},
                           attr     => ['author', 'isbn', 'title',
                                        'headline', 'docid'],
                           layout   => $layout,
                           access   => $access,
                           invindex =>
                           [
                            'title'  => $text,
                            # 'title'  => $stem,
                            'aboutauthor'  => $text,
                            # 'aboutauthor'  => $stem,
                            'desc'   => $text,
                            'abstract' => $text,
                            'author' => $text,
                            # 'author' => $sound,
                            'colophon' => $text,
                            'isbn'   => $isbn,
                           ]
                          );
die "Couldn't create table $OPT{table}: $@\n" unless $tb;

my ($did, $value);
binmode STDOUT, ":utf8";
while (($did, $value) = each %D) {
  my $record   = $layout->split($value);
  my $headline = $record->{title};
  $headline =~ s/\s+/ /sg;
  printf "%15s %s\n", $record->{isbn}, substr($headline,0,60);
  $tb->insert('docid'  => $did,
              headline => $headline,
              %{$record});
}
$tb->set(top=>1);

my $tritb = $db->create_table(
                              name => "$OPT{table}_fallback",
                              attr => [qw(docid headline)], # name
                                                            # "headline"
                                                            # only for
                                                            # sman
                              invindex => [ headline => $trigr ],
                             );
my %dict;
for my $f ($tb->fields) {
  my(@idx) = @{$tb->table->{inverted}{$f} || []};
  for my $idx (@idx) {
    my $name = $idx->name;
    next if $name =~ /(_|\b)(Stem|Soundex)(\b|_)/; # irrelevant for alternatives
    my @keys = $idx->keys;
    @dict{@keys} = ();
  }
}
my @dictkeys = grep s/^p//, keys %dict;
my $maxdebug = 5;
for my $headline (@dictkeys) {
  if ($maxdebug && $headline =~ /[^\040-\177]/) {
    Dump $headline;
    $maxdebug--;
  }
  # printf "%s\n", substr($headline,0,60);
  $tritb->insert(docid => $headline, headline => $headline);
}
$tritb->set(top=>1);
$tritb->close;
$tb->close();
$db->close();

# Now we have a new database with a very long name and we want that
# database to be accessible with the $OPT{database} name

use File::Spec;
my $long_dir   = "$OPT{database}-$jobid";
my $want_dir   = File::Spec->catdir($OPT{dir}, $OPT{database});
my $prel_slink = File::Spec->catdir($OPT{dir}, "$OPT{database}-$$");
unlink $prel_slink; # may fail
symlink $long_dir, $prel_slink or die "Could not symlink $long_dir, $prel_slink: $!";
rename $prel_slink, $want_dir or die "Could not rename $prel_slink, $want_dir: $!";

system("chmod 777 $want_dir/*/read")==0 or die;

$WAIT::Config = $WAIT::Config; # make perl -w happy


__END__
## ###################################################################
## pod
## ###################################################################

=head1 NAME

index_ora - generate an WAIT index for O'Reilly catalog

=head1 SYNOPSIS

B<index_ora>
[B<-database> I<dbname>]
[B<-dir> I<directory>]
[B<-table> I<table name>]
I<directory>

=head1 DESCRIPTION

=head1 OPTIONS

=over 5

=item B<-database> I<dbname>

Specify database name. Default is F<DB>.

=item B<-dir> I<directory>

Alternate directory where databases are located. Default is the
directory specified during configuration of WAIT.

=item B<-table> I<table name>

Specify an alternate table name. Default is C<ora>.

=head1 AUTHOR

Ulrich Pfeifer E<lt>F<pfeifer@wait.de>E<gt>

1	ulpfr	55	#!/usr/bin/perl -w
2			# -- Mode: Perl --
3			# $Basename$
4	laperla	72	# $Revision: 1.9 $
5	ulpfr	55	# Author : Ulrich Pfeifer
6			# Created On : Mon Dec 31 13:57:11 2001
7			# Last Modified By: Ulrich Pfeifer
8	ulpfr	62	# Last Modified On: Fri Jan 4 15:59:20 2002
9	ulpfr	55	# Language : CPerl
10			#
11			# (C) Copyright 2001, UUNET Deutschland GmbH, Germany
12			#
13
14	laperla	69	use 5.007;
15
16	ulpfr	55	use strict;
17	laperla	71	use Devel::Peek qw(Dump);
18	laperla	69
19	ulpfr	55	use File::Path;
20			use DB_File;
21			use Getopt::Long;
22			use Cwd;
23
24	laperla	69	BEGIN {require WAIT::Config;}
25			use WAIT::Database;
26			use WAIT::Parse::Ora;
27			use WAIT::Document::Ora;
28			use WAIT::InvertedIndex;
29	ulpfr	55
30
31			$DB_BTREE->{'cachesize'} = 200_000 ;
32
33	laperla	67	my %OPT = (
34	ulpfr	55	database => 'DB',
35			dir => $WAIT::Config->{WAIT_home} \|\| '/tmp',
36			table => 'ora',
37			);
38
39			GetOptions(\%OPT,
40			'database=s',
41			'dir=s',
42			'table=s',
43			) \|\| die "Usage: ...\n";
44
45	laperla	68	my @localtime = localtime;
46			$localtime[5] += 1900;
47			$localtime[4]++;
48			my $jobid = sprintf "%04s-%02s-%02s_%02s:%02s_%d", @localtime[5,4,3,2,1], $$;
49			my $db = WAIT::Database->create(name => "$OPT{database}-$jobid",
50			directory => $OPT{dir})
51			or die "Could not create database $OPT{database}: $@\n";
52	ulpfr	55
53			my $layout = new WAIT::Parse::Ora;
54
55	laperla	69	use lib "/usr/local/apache/lib";
56			use oreilly_de_catalog::wait_handler;
57
58			my $stem = ['OR_tr_20020124', 'OR_lc_20020124', 'split2', 'stop', 'Stem'];
59	ulpfr	55	my $text = [{
60	laperla	69	'prefix' => ['OR_tr_20020124', 'OR_lc_20020124'],
61			'intervall' => ['OR_tr_20020124', 'OR_lc_20020124'],
62	ulpfr	55	},
63	laperla	69	'OR_tr_20020124', 'OR_lc_20020124', 'split2', 'stop'];
64	laperla	71	my $sound = ['OR_tr_20020124', 'OR_lc_20020124', 'split2', 'Soundex'];
65			my $trigr = ['OR_lc_20020124', 'OR_trigrams_20020125'];
66	laperla	72	my $isbn = ['split6', 'OR_isbn_20020127'];
67	ulpfr	55
68			my $cwd = cwd;
69
70			my %D;
71			my $access = tie %D, 'WAIT::Document::Ora', @ARGV,
72			or die "Couldn't tie to file: $!\n";
73
74			my $tb = $db->create_table(name => $OPT{table},
75	ulpfr	62	attr => ['author', 'isbn', 'title',
76	ulpfr	55	'headline', 'docid'],
77			layout => $layout,
78			access => $access,
79			invindex =>
80			[
81	laperla	72	'title' => $text,
82			# 'title' => $stem,
83			'aboutauthor' => $text,
84			# 'aboutauthor' => $stem,
85			'desc' => $text,
86			'abstract' => $text,
87	ulpfr	55	'author' => $text,
88	laperla	72	# 'author' => $sound,
89	laperla	65	'colophon' => $text,
90	laperla	72	'isbn' => $isbn,
91	ulpfr	55	]
92			);
93			die "Couldn't create table $OPT{table}: $@\n" unless $tb;
94
95			my ($did, $value);
96	laperla	69	binmode STDOUT, ":utf8";
97	ulpfr	55	while (($did, $value) = each %D) {
98			my $record = $layout->split($value);
99			my $headline = $record->{title};
100			$headline =~ s/\s+/ /sg;
101	ulpfr	62	printf "%15s %s\n", $record->{isbn}, substr($headline,0,60);
102	ulpfr	55	$tb->insert('docid' => $did,
103			headline => $headline,
104			%{$record});
105			}
106			$tb->set(top=>1);
107	laperla	71
108			my $tritb = $db->create_table(
109			name => "$OPT{table}_fallback",
110	laperla	72	attr => [qw(docid headline)], # name
111			# "headline"
112			# only for
113			# sman
114	laperla	71	invindex => [ headline => $trigr ],
115			);
116			my %dict;
117			for my $f ($tb->fields) {
118			my(@idx) = @{$tb->table->{inverted}{$f} \|\| []};
119			for my $idx (@idx) {
120	laperla	72	my $name = $idx->name;
121			next if $name =~ /(_\|\b)(Stem\|Soundex)(\b\|_)/; # irrelevant for alternatives
122	laperla	71	my @keys = $idx->keys;
123			@dict{@keys} = ();
124			}
125			}
126			my @dictkeys = grep s/^p//, keys %dict;
127			my $maxdebug = 5;
128			for my $headline (@dictkeys) {
129			if ($maxdebug && $headline =~ /[^\040-\177]/) {
130			Dump $headline;
131			$maxdebug--;
132			}
133	laperla	72	# printf "%s\n", substr($headline,0,60);
134	laperla	71	$tritb->insert(docid => $headline, headline => $headline);
135			}
136			$tritb->set(top=>1);
137			$tritb->close;
138	ulpfr	55	$tb->close();
139			$db->close();
140
141	laperla	68	# Now we have a new database with a very long name and we want that
142			# database to be accessible with the $OPT{database} name
143
144			use File::Spec;
145			my $long_dir = "$OPT{database}-$jobid";
146			my $want_dir = File::Spec->catdir($OPT{dir}, $OPT{database});
147			my $prel_slink = File::Spec->catdir($OPT{dir}, "$OPT{database}-$$");
148			unlink $prel_slink; # may fail
149			symlink $long_dir, $prel_slink or die "Could not symlink $long_dir, $prel_slink: $!";
150			rename $prel_slink, $want_dir or die "Could not rename $prel_slink, $want_dir: $!";
151
152	laperla	72	system("chmod 777 $want_dir/*/read")==0 or die;
153
154	ulpfr	55	$WAIT::Config = $WAIT::Config; # make perl -w happy
155
156
157			__END__
158			## ###################################################################
159			## pod
160			## ###################################################################
161
162			=head1 NAME
163
164			index_ora - generate an WAIT index for O'Reilly catalog
165
166			=head1 SYNOPSIS
167
168			B<index_ora>
169			[B<-database> I<dbname>]
170			[B<-dir> I<directory>]
171			[B<-table> I<table name>]
172			I<directory>
173
174			=head1 DESCRIPTION
175
176			=head1 OPTIONS
177
178			=over 5
179
180			=item B<-database> I<dbname>
181
182			Specify database name. Default is F<DB>.
183
184			=item B<-dir> I<directory>
185
186			Alternate directory where databases are located. Default is the
187			directory specified during configuration of WAIT.
188
189			=item B<-table> I<table name>
190
191			Specify an alternate table name. Default is C<ora>.
192
193			=head1 AUTHOR
194
195			Ulrich Pfeifer E<lt>F<pfeifer@wait.de>E<gt>
196