1 |
#!/usr/bin/perl -w |
2 |
|
3 |
=head1 NAME |
4 |
|
5 |
cpanest - generate an Hyper Estraier index for CPAN |
6 |
|
7 |
=head1 SYNOPSIS |
8 |
|
9 |
B<cpanest> |
10 |
[B<-clean>] [B<-noclean>] |
11 |
[B<-cpan> I<url or directory>] |
12 |
[B<-node> I<node_uri>] |
13 |
[B<-force>] [B<-noforce>] |
14 |
[B<-keep> I<directory>] |
15 |
[B<-match> I<regexp>] |
16 |
[B<-test> I<level>] |
17 |
[B<-trust_mtime>] [B<-notrust_mtime>] |
18 |
|
19 |
=head1 DESCRIPTION |
20 |
|
21 |
This is a port of C<cpanwait> from L<WAIT> perl search engine to node API of |
22 |
Hyper Estraier. |
23 |
|
24 |
All the hard work was done by Ulrich Pfeifer who wrote all parsers and |
25 |
formatters. I just added support for Hyper Estraier back-end after. |
26 |
|
27 |
B<This documentation is somewhat incomplete and off-the-sync with code.> |
28 |
|
29 |
=head1 OPTIONS |
30 |
|
31 |
=over 5 |
32 |
|
33 |
=item B<-clean> / B<-noclean> |
34 |
|
35 |
Clean the table befor indexing. Default is B<off>. |
36 |
|
37 |
=item B<-cpan> I<url or directory> |
38 |
|
39 |
Default directory or URL for indexing. If an URL is given, there |
40 |
currently must be a file F<indices/find-ls.gz> relative to it which |
41 |
contains the output of C<find . -ls | gzip>. |
42 |
Default is F<ftp://ftp.rz.ruhr-uni-bochum.de/pub/CPAN>. |
43 |
|
44 |
|
45 |
=item B<-node> I<http://localhost:1978/node/cpan> |
46 |
|
47 |
Specify node URI |
48 |
|
49 |
=item B<-force> |
50 |
|
51 |
Force reindexing, even if B<cpan> thinks files are up to date. |
52 |
Default is B<off> |
53 |
|
54 |
=item B<-keep> I<directory> |
55 |
|
56 |
If fetching from a remote server, keep files in I<directory>. Default is |
57 |
F</app/unido-i06/src/share/lang/perl/96a/CPAN/sources>. |
58 |
|
59 |
=item B<-match> I<regexp> |
60 |
|
61 |
Limit to patches matching I<regexp>. Default is F<authors/id/>. |
62 |
|
63 |
=item B<-test> I<level> |
64 |
|
65 |
Set test level, were B<0> means normal operation, B<1> means, don't |
66 |
really index and B<2> means, don't even get archives and examine them. |
67 |
|
68 |
=item B<-trust_mtime> / B<-notrust_mtime> |
69 |
|
70 |
If B<on>, the files mtimes are used to decide, which version of an |
71 |
archive is the newest. If b<off>, the version extracted is used |
72 |
(beware, there are far more version numbering schemes than B<cpan> can |
73 |
parse). |
74 |
|
75 |
=back |
76 |
|
77 |
=head1 AUTHORS |
78 |
|
79 |
Ulrich Pfeifer E<lt>F<pfeifer@ls6.informatik.uni-dortumund.de>E<gt> |
80 |
|
81 |
Dobrica Pavlinusic E<lt>F<dpavlin@rot13.org>E<gt> |
82 |
|
83 |
=head1 COPYRIGHT |
84 |
|
85 |
Copyright (c) 1996-1997, Ulrich Pfeifer |
86 |
|
87 |
Copyright (c) 2005, Dobrica Pavlinusic |
88 |
|
89 |
=cut |
90 |
|
91 |
use strict; |
92 |
|
93 |
use File::Path; |
94 |
use Getopt::Long; |
95 |
use File::Find; |
96 |
use File::Basename; |
97 |
use IO::File; |
98 |
use IO::Zlib; |
99 |
use POSIX qw/strftime/; |
100 |
|
101 |
use lib '/data/wait/lib'; |
102 |
|
103 |
use WAIT::Parse::Base; |
104 |
use WAIT::Parse::Pod; |
105 |
use WAIT::Document::Tar; |
106 |
use WAIT::Document::Find; |
107 |
|
108 |
sub fname($); |
109 |
|
110 |
# maximum number of archives to index (set to -1 for unlimited) |
111 |
my $max = -1; |
112 |
|
113 |
my %OPT = ( |
114 |
node => 'http://localhost:1978/node/cpan', |
115 |
clean => 0, |
116 |
remove => [], |
117 |
force => 0, |
118 |
# cpan => '/usr/src/perl/CPAN/sources', |
119 |
cpan => '/rest/cpan/CPAN/', |
120 |
trust_mtime => 1, |
121 |
match => 'authors/id/', |
122 |
test => 0, |
123 |
# cpan => 'ftp://ftp.rz.ruhr-uni-bochum.de/pub/CPAN', |
124 |
# cpan => 'ftp://ftp.uni-hamburg.de:/pub/soft/lang/perl/CPAN', |
125 |
keep => '/tmp/CPAN/', |
126 |
); |
127 |
|
128 |
GetOptions(\%OPT, |
129 |
'node=s', |
130 |
'cpan=s', |
131 |
'keep=s', |
132 |
'match=s', |
133 |
'clean!', |
134 |
'test=i', # test level 0: normal |
135 |
# 1: don't change db |
136 |
# 2: don't look at archives even |
137 |
|
138 |
'remove=s@', |
139 |
'force!', # force indexing even if seen |
140 |
'trust_mtime!', # use mtime instead of version number |
141 |
'max=i', |
142 |
'debug!', |
143 |
) || die "Usage: ...\n"; |
144 |
|
145 |
if ($OPT{max}) { |
146 |
$max = $OPT{max}; |
147 |
print STDERR "processing just first $max modules\n"; |
148 |
} |
149 |
|
150 |
# FIXME |
151 |
#clean_node( |
152 |
# node => $OPT{node}, |
153 |
# ) if $OPT{clean}; |
154 |
|
155 |
my $tb = new HyperEstraier::WAIT::Table( |
156 |
uri => $OPT{node}, |
157 |
attr => ['docid', 'headline', 'source', 'size', 'parent', 'version'], |
158 |
key => 'docid', |
159 |
invindex => [ qw/name synopsis bugs description text environment example author/ ], |
160 |
debug => $OPT{debug}, |
161 |
) or die "Could not open node '$OPT{node}'"; |
162 |
|
163 |
my $layout= new WAIT::Parse::Pod; |
164 |
|
165 |
# Map e.g. '.../latest' to 'perl'. Used in wanted(). Effects version |
166 |
# considerations. Value *must* match common prefix. Aliasing should be |
167 |
# used if CPAN contains serveral distributions with different name but |
168 |
# same root directory. |
169 |
# We still have a problem if there are different root directories! |
170 |
|
171 |
my %ALIAS = (# tar name real (root) name |
172 |
'Games-Scrabble' => 'Games', |
173 |
'HTML-ParseBrowser' => 'HTML', |
174 |
'iodbc_ext' => 'iodbc-ext-0.1', |
175 |
'sol-inst' => 'Solaris', |
176 |
'WebService-Validator-CSS-223C' => 'WebService-Validator-CSS-W3C-0.02', |
177 |
'MPEG-ID3212Tag' => 'MPEG-ID3v2Tag-0.36', |
178 |
'WebService-GoogleHack' => 'WebService', |
179 |
'Db-Mediasurface-ReadConfig' => 'ReadConfig', |
180 |
'Tie-Array-RestrictUpdates' => 'Tie', |
181 |
'HTML-Lister' => 'HTML', |
182 |
'Net-253950-AsyncZ' => 'Net-Z3950-AsyncZ-0.08', |
183 |
'ChildExit_0' => 'ChildExit-0.1', |
184 |
'Tie-TieConstant' => 'TieConstant.pm', |
185 |
'Crypt-OpenSSL-23509' => 'Crypt-OpenSSL-X509-0.2', |
186 |
'subclustv' => 'blib', |
187 |
'finance-yahooquote' => 'Finance-YahooQuote-0.20', |
188 |
'HPUX-FS' => 'FS', |
189 |
'Business-DE-Konto' => 'Business', |
190 |
'Digest-MD5-124p' => 'Digest-MD5-M4p-0.01', |
191 |
'AKDB_Okewo_de' => 'AKDB', |
192 |
'ExtUtils-0577' => 'ExtUtils-F77-1.14', |
193 |
'LispFmt' => 'Lisp::Fmt-0.00', |
194 |
'Acme-Stegano' => 'Acme', |
195 |
'Acme-RTB' => 'Acme', |
196 |
'WWW-Search-PRWire' => 'work', |
197 |
'Video-Capture-214l' => 'Video-Capture-V4l-0.224', |
198 |
'Tie-DirHandle' => 'Tie', |
199 |
'DB2' => 'DBD-DB2-0.71a', |
200 |
'Tie-Scalar-RestrictUpdates' => 'Tie', |
201 |
'Math-MVPoly' => 'MVPoly', |
202 |
'PlugIn' => 'PlugIn.pm', |
203 |
'Lingua-ID-Nums2Words' => 'Nums2Words-0.01', |
204 |
'chronos-1.' => 'Chronos', |
205 |
'jp_beta' => 'jperl_beta_r1', |
206 |
'Bundle-223C-Validator' => 'Bundle-W3C-Validator-0.6.5', |
207 |
'Text-199' => 'Text-T9-1.0', |
208 |
'Games-Literati' => 'Games', |
209 |
'VMS-IndexedFile' => 'VMS', |
210 |
'authen-rbac' => 'Authen', |
211 |
'Graphics-EPS' => 'EPS.pm', |
212 |
'new.spirit-2.' => 'new.spirit', |
213 |
'Tk-MListbox' => 'MListbox-1.11', |
214 |
'DBD-SQLrelay' => 'SQLRelay.pm', |
215 |
'Tie-RDBM-Cached' => 'RDBM', |
216 |
'PDL_IO_HDF' => 'HDF', |
217 |
'HPUX-LVM' => 'LVM', |
218 |
'Parse-Nibbler' => 'Parse', |
219 |
'Digest-Perl-MD4' => 'MD4', |
220 |
'Crypt-Imail' => 'Imail', |
221 |
'ubertext' => 'Text-UberText-0.95', |
222 |
'MP3-123U' => 'M3U', |
223 |
'Qmail-Control' => 'Qmail', |
224 |
'T-LXS' => 'Text-LevenshteinXS-0.02', |
225 |
'HTML-Paginator' => 'HTML', |
226 |
'swig' => 'SWIG1.1p5', |
227 |
'MIDI-Realtime' => 'MIDI', |
228 |
'sparky-public' => 'Sparky-Public-1.06', |
229 |
'Chemistry-MolecularMass' => 'Chemistry', |
230 |
'Net-253950-SimpleServer' => 'Net-Z3950-SimpleServer-0.08', |
231 |
'NewsClipper-OpenSource' => 'NewsClipper-1.32-OpenSource', |
232 |
'Win32API-Resources' => 'Resources.pm', |
233 |
'Unicode-Collate-Standard-2131_1' => 'Unicode-Collate-Standard-V3_1_1-0.1', |
234 |
'Net-026Term' => 'Net-C6Term-0.11', |
235 |
'BitArray1' => 'BitArray', |
236 |
'Audio-Radio-214L' => 'Audio-Radio-V4L-0.01', |
237 |
'Devel-AutoProfiler' => 'Devel', |
238 |
'Brasil-Checar-CGC' => 'Brasil', |
239 |
'AI-NeuralNet-SOM' => 'SOM.pm', |
240 |
'Net-BitTorrent-File-fix' => 'Net-BitTorrent-File-1.01', |
241 |
'VMS-FindFile' => 'VMS', |
242 |
'LoadHtml.' => 'README', |
243 |
'Time-Compare' => 'Time', |
244 |
'ShiftJIS-230213-MapUTF' => 'ShiftJIS-X0213-MapUTF-0.21', |
245 |
'Image-WMF' => 'Image', |
246 |
'sdf-2.0.eta' => 'sdf-2.001beta1', |
247 |
'Math-Expr-LATEST' => 'Math-Expr-0.4', |
248 |
'MP3-Player-PktConcert' => 'MP3', |
249 |
'Apache-OWA' => 'OWA', |
250 |
'Audio-Gramofile' => 'Audio', |
251 |
'DBIx-Copy' => 'Copy', |
252 |
'P4-024' => 'P4-C4-2.021', |
253 |
'Disassemble-2386' => 'Disassemble-X86-0.13', |
254 |
'Proc-Swarm' => 'Swarm-0.5', |
255 |
'Smil' => 'perlysmil', |
256 |
'Net-SSH-2232Perl' => 'Net-SSH-W32Perl-0.05', |
257 |
'Win32-SerialPort' => 'SerialPort-0.19', |
258 |
'Lingua-ID-Words2Nums' => 'Words2Nums-0.01', |
259 |
'Parse-Text' => 'Text', |
260 |
'DBIx-HTMLView-LATEST' => 'DBIx-HTMLView-0.9', |
261 |
'Apache-NNTPGateway' => 'NNTPGateway-0.9', |
262 |
'XPathToXML' => 'XPathToXML.pm', |
263 |
'XML-WMM-ASX' => 'XML', |
264 |
'CGISession' => 'CGI', |
265 |
'Net-SMS-142' => 'Net-SMS-O2-0.019', |
266 |
'Search-253950' => 'Search-Z3950-0.05', |
267 |
'Date-Christmas' => 'Christmas', |
268 |
'Win32-InternetExplorer-Window' => 'Win32', |
269 |
'Apache-WAP-MailPeek' => 'MailPeek', |
270 |
'Statistics-Table-F' => 'Statistics', |
271 |
'BerkeleyDB_Locks' => 'BerkeleyDB-Locks-0_2', |
272 |
'HookPrePostCall' => 'PrePostCall-1.2', |
273 |
'Oak-AAS-Service-DBI_13_PAM' => 'Oak-AAS-Service-DBI_N_PAM-1.8', |
274 |
'Math-Vector' => 'Vector.pm', |
275 |
'Audio-124pDecrypt' => 'Audio-M4pDecrypt-0.04', |
276 |
'libao-perl_0.03' => 'libao-perl-0.03', |
277 |
'CGI-EZForm' => 'EZForm', |
278 |
'Data-Locations-fixed' => 'Data-Locations-5.2-fixed', |
279 |
'HTML-Template-Filter-Dreamweaver' => 'Dreamweaver', |
280 |
'LineByLine' => 'LineByLine.pm', |
281 |
'Geo-0400' => 'Geo-E00-0.05', |
282 |
'WebService-Validator-HTML-223C' => 'WebService-Validator-HTML-W3C-0.03', |
283 |
'DateTime-Format-223CDTF' => 'DateTime-Format-W3CDTF-0.04', |
284 |
'DBD_SQLFLEX' => 'DBD-Sqlflex', |
285 |
'Text-Number' => 'Number', |
286 |
'DBIx-DataLookup' => 'DBIx', |
287 |
'MP3-ID3211Tag' => 'MP3-ID3v1Tag-1.11', |
288 |
'Text-Striphigh' => 'Striphigh-0.02', |
289 |
'Tie-SortHash' => 'SortHash', |
290 |
'Apache-AccessAbuse' => 'AccessAbuse', |
291 |
'MP3-123U-Parser' => 'MP3-M3U-Parser', |
292 |
'Net-253950' => 'Net-Z3950-0.44', |
293 |
'Net-RBLClient' => 'RBLCLient-0.2', |
294 |
'CGI-EasyCGI' => 'CGI', |
295 |
'http-handle' => 'HTTP::Handle', |
296 |
'JPEG-Comment' => 'JPEG', |
297 |
'router-lg' => 'Router', |
298 |
'Db-Mediasurface' => 'Mediasurface', |
299 |
'Text-BarGraph' => 'bargraph', |
300 |
'TL' => 'Text-Levenshtein-0.04', |
301 |
'Config-Vars' => 'Config-0.01', |
302 |
'Tie-PerfectHash' => 'Tie', |
303 |
'DNS-TinyDNS' => 'DNS', |
304 |
'DesignPattern-Factory' => 'Factory', |
305 |
'WWW-01_Rail' => 'WWW-B_Rail-0.01', |
306 |
'Win32-Exchange' => 'blib', |
307 |
'Math-RPN' => 'Math', |
308 |
'Db-Mediasurface-Cache' => 'Cache', |
309 |
'perl_archie.' => 'Archie.pm', |
310 |
'Acme-PGPSign' => 'Acme', |
311 |
'HTML-Widget-sideBar' => 'HTML-Widget-SideBar-1.00', |
312 |
'log' => 'Games', |
313 |
'File-List' => 'File', |
314 |
'Schedule-Cronchik' => 'Schedule', |
315 |
'Curses-Devkit' => 'Cdk', |
316 |
'Pod-PalmDoc' => 'Pod', |
317 |
'Easy-WML' => 'Easy WML 0.1', |
318 |
'Interval.' => 'Date', |
319 |
'Brasil-Checar-CPF' => 'Brasil', |
320 |
'Apache-WAP-AutoIndex' => 'AutoIndex', |
321 |
|
322 |
'SOM.pm' => 'SOM.pm', |
323 |
'PlugIn.pm' => 'PlugIn.pm', |
324 |
'XPathToXML.pm' => 'XPathToXML.pm', |
325 |
'Vector.pm' => 'Vector.pm', |
326 |
'LineByLine.pm' => 'LineByLine.pm', |
327 |
'Archie.pm' => 'Archie.pm', |
328 |
'TieConstant.pm' => 'TieConstant.pm', |
329 |
'EPS.pm' => 'EPS.pm', |
330 |
'SQLRelay.pm' => 'SQLRelay.pm', |
331 |
'Resources.pm' => 'Resources.pm', |
332 |
'README' => 'README', |
333 |
|
334 |
); |
335 |
my %NEW_ALIAS; # found in this pass |
336 |
|
337 |
# Map module names to pathes. Generated by wanted() doing alisaing. |
338 |
my %ARCHIVE; |
339 |
|
340 |
# Map module names to latest version. Generated by wanted() |
341 |
my %VERSION; |
342 |
|
343 |
|
344 |
# Mapping for modules with common root not matching modules name that |
345 |
# are not aliased. This is just for prefix stripping and not strictly |
346 |
# necessary. Takes effect after version considerations. |
347 |
my %TR = (# tar name root to strip |
348 |
'Net_SSLeay.pm' => 'SSLeay/', |
349 |
'EventDrivenServer' => 'Server/', |
350 |
'bio_lib.pl.' => '', |
351 |
'AlarmCall' => 'Sys/', |
352 |
'Cdk-ext' => 'Cdk/', |
353 |
'Sx' => '\d.\d/', |
354 |
'DumpStack' => 'Devel/', |
355 |
'StatisticsDescriptive'=> 'Statistics/', |
356 |
'Term-Gnuplot' => 'Gnuplot/', |
357 |
'iodbc_ext' => 'iodbc-ext-\d.\d/', |
358 |
'UNIVERSAL' => '', |
359 |
'Term-Query' => 'Query/', |
360 |
'SelfStubber' => 'Devel/', |
361 |
'CallerItem' => 'Devel/', |
362 |
); |
363 |
|
364 |
my $LWP; |
365 |
|
366 |
# FIXME |
367 |
my $DIR = '/rest/estseek/cpan/'; |
368 |
my $DATA = $DIR . '/data'; |
369 |
|
370 |
|
371 |
if (@{$OPT{remove}}) { |
372 |
my $pod; |
373 |
for $pod (@{$OPT{remove}}) { |
374 |
unless (-e $pod) { |
375 |
$pod = "$DIR/$pod"; |
376 |
} |
377 |
index_pod(file => $pod, remove => 1) if -f $pod; |
378 |
unlink $pod or warn "Could not unlink '$pod': $!\n"; |
379 |
} |
380 |
exit; |
381 |
} |
382 |
|
383 |
# Now get the beef |
384 |
if ($OPT{cpan} =~ /^(http|ftp):/) { |
385 |
$LWP = 1; |
386 |
require LWP::Simple; |
387 |
LWP::Simple->import(); |
388 |
|
389 |
mkpath($DATA,1,0755) or |
390 |
die "Could not generate '$DATA/': $!" |
391 |
unless -d $DATA; |
392 |
|
393 |
if (! -f "$DATA/find-ls.gz" or -M "$DATA/find-ls.gz" > 0.5) { |
394 |
my $status = mirror("$OPT{cpan}/indices/find-ls.gz", "$DATA/find-ls.gz"); |
395 |
if ($status != &RC_OK and $status != &RC_NOT_MODIFIED) { |
396 |
# we could use Net:FTP here ... |
397 |
die "Was unable to mirror '$OPT{cpan}/indices/find-ls.gz'\n"; |
398 |
} |
399 |
} |
400 |
my $fh = new IO::File "gzip -cd $DATA/find-ls.gz |"; |
401 |
die "Could not open 'gzip -cd $DATA/find-ls.gz': !$\n" unless $fh; |
402 |
|
403 |
my $line; |
404 |
while (defined ($line = <$fh>)) { |
405 |
chomp($line); |
406 |
my ($mon, $mday, $time, $file, $is_link) = (split ' ', $line)[7..11]; |
407 |
|
408 |
next if defined $is_link; |
409 |
my $mtime = mtime($mon, $mday, $time); |
410 |
|
411 |
$file =~ s:^\./::; |
412 |
($_) = fileparse($file); |
413 |
$File::Find::name = $file; |
414 |
wanted($mtime); |
415 |
} |
416 |
} else { |
417 |
find(sub {&wanted((stat($_))[9])}, $OPT{cpan}); |
418 |
} |
419 |
|
420 |
ARCHIVE: |
421 |
for my $tar (sort keys %ARCHIVE) { |
422 |
next if $OPT{match} and $ARCHIVE{$tar} !~ /$OPT{match}/o; |
423 |
my $base = (split /\//, $ARCHIVE{$tar})[-1]; |
424 |
my $parent; |
425 |
|
426 |
my %attr; |
427 |
|
428 |
# logging |
429 |
if ($OPT{trust_mtime}) { |
430 |
$attr{'@mdate'} = strftime('%Y-%m-%dT%H:%M:%S+00:00', gmtime($VERSION{$tar})); |
431 |
$parent->{'@mdate'} = $attr{'@mdate'}; |
432 |
printf "%-20s %10s %s\t", $tar, $attr{'@mdate'}, $base; |
433 |
} else { |
434 |
$attr{'version'} = $VERSION{$tar}; |
435 |
printf "%-20s %10.5f %s\t", $tar, $attr{'version'}, $base; |
436 |
} |
437 |
|
438 |
# Remember the archive |
439 |
# We should have an extra table for the tar file data ... |
440 |
if (!$OPT{force} and $tb->have(docid => $base)) { |
441 |
print "skipping\n"; |
442 |
next ARCHIVE; |
443 |
} else { |
444 |
$parent->{_id} = $tb->insert(docid => $base, |
445 |
headline => $ARCHIVE{$tar}, |
446 |
%attr |
447 |
) unless $OPT{test}; |
448 |
print "indexing\n"; |
449 |
} |
450 |
|
451 |
next ARCHIVE if $OPT{test} > 1; |
452 |
|
453 |
my $TAR = myget($tar); |
454 |
|
455 |
next ARCHIVE unless $TAR; # not able to fetch it |
456 |
|
457 |
my %tar; |
458 |
tie (%tar, |
459 |
'WAIT::Document::Tar', |
460 |
sub { $_[0] =~ /\.(pm|pod|PL)$/i or $_[0] =~ /readme/i}, |
461 |
#sub { $_[0] !~ m:/$: }, |
462 |
$TAR) |
463 |
or warn "Could not tie '$TAR'\n"; |
464 |
|
465 |
my $sloppy; |
466 |
my ($key, $val); |
467 |
|
468 |
FILE: |
469 |
while (($key, $val) = each %tar) { |
470 |
my $file = fname($key); |
471 |
|
472 |
# don't index directories |
473 |
next if $file =~ /\/$/; |
474 |
|
475 |
# is it a POD file? |
476 |
next FILE unless $file =~ /readme/i or $val =~ /\n=head/; |
477 |
|
478 |
# remove directory prefix |
479 |
unless ($sloppy # no common root |
480 |
or $file =~ s:^\Q$tar\E[^/]*/:: # common root, maybe alias |
481 |
or ($TR{$tar} # common root, not aliased |
482 |
and $file =~ s:^\Q$TR{$tar}\E::) |
483 |
) { |
484 |
# try to determine an alias |
485 |
warn "Bad directory prefix: '$file'\n"; |
486 |
my ($prefix) = split /\//, $file; |
487 |
|
488 |
while ($key = (tied %tar)->NEXTKEY) { |
489 |
my $file = fname($key); |
490 |
|
491 |
next if $file =~ /\/$/; |
492 |
unless ($file =~ m:^$prefix/: or $file eq $prefix) { |
493 |
warn "Archive contains different prefixes: $prefix,$file\n"; |
494 |
$prefix = ''; |
495 |
last; |
496 |
} |
497 |
} |
498 |
if ($prefix) { |
499 |
print "Please alias '$tar' to '$prefix' next time!\n"; |
500 |
print "See alias table later.\n"; |
501 |
$NEW_ALIAS{$tar} = $prefix; |
502 |
$tb->delete_by_key($parent->{_id}); |
503 |
next ARCHIVE; |
504 |
} else { |
505 |
print "Assuming that tar file name $tar is a valid prefix\n"; |
506 |
$sloppy = 1; |
507 |
|
508 |
# We may reset too much here! But that this is not exact |
509 |
# science anyway. Maybe we should ignore using 'next ARCHIVE'. |
510 |
|
511 |
$key = (tied %tar)->FIRSTKEY; |
512 |
redo FILE; |
513 |
} |
514 |
} |
515 |
|
516 |
# remove /lib prefix |
517 |
$file =~ s:^lib/::; |
518 |
|
519 |
# generate new path |
520 |
my $path = "$DATA/$tar/$file"; |
521 |
|
522 |
my ($sbase, $sdir) = fileparse($path); |
523 |
my $fh; |
524 |
|
525 |
unless ($OPT{test}) { |
526 |
if (-f $path) { |
527 |
index_pod(file => $path, remove => 1); |
528 |
unlink $path or warn "Could not unlink '$path' $!\n"; |
529 |
} elsif (!-d $sdir) { |
530 |
mkpath($sdir,1,0755) or die "Could not mkpath($sdir): $!\n"; |
531 |
} |
532 |
# $fh = new IO::File "> $path"; |
533 |
$fh = new IO::Zlib "$path.gz","wb"; |
534 |
die "Could not write '$path': $!\n" unless $fh; |
535 |
} |
536 |
|
537 |
if ($file =~ /readme|install/i) { # make READMEs verbatim pods |
538 |
$val =~ s/\n/\n /g; |
539 |
$val = "=head1 NAME\n\n$tar $file\n\n=head1 DESCRIPTION\n\n $val" |
540 |
unless $val =~ /^=head/m; |
541 |
} else { # remove non-pod stuff |
542 |
my $nval = $val; $val = ''; |
543 |
my $cutting = 1; |
544 |
|
545 |
for (split /\n/, $nval) { |
546 |
if (/^=cut|!NO!SUBS!/) { |
547 |
$cutting = 1; |
548 |
} elsif ($cutting and /^=head/) { |
549 |
$cutting = 0; |
550 |
} |
551 |
unless ($cutting) { |
552 |
$val .= $_ . "\n"; |
553 |
} |
554 |
} |
555 |
} |
556 |
unless ($OPT{test}) { |
557 |
$fh->print($val); |
558 |
index_pod(file => $path, parent => $parent, |
559 |
text => $val, source => $ARCHIVE{$tar}, |
560 |
); |
561 |
} |
562 |
} |
563 |
|
564 |
if ($LWP and !$OPT{keep}) { |
565 |
unlink $TAR or warn |
566 |
"Could not unlink '$TAR': $!\n"; |
567 |
} |
568 |
} |
569 |
|
570 |
if (%NEW_ALIAS) { |
571 |
print "\%ALIAS = (\n"; |
572 |
for (keys %NEW_ALIAS) { |
573 |
print "\t'$_'\t=> '$NEW_ALIAS{$_}',\n"; |
574 |
} |
575 |
print "\t);\n"; |
576 |
} |
577 |
|
578 |
exit; |
579 |
|
580 |
sub fname ($) { |
581 |
my $key = shift; |
582 |
my ($ntar, $file) = split $;, $key; |
583 |
|
584 |
# remove leading './' - shudder |
585 |
$file =~ s/^\.\///; |
586 |
|
587 |
return($file); |
588 |
} |
589 |
|
590 |
sub myget { |
591 |
my $tar = shift; |
592 |
my $TAR; |
593 |
|
594 |
if ($LWP) { # fetch the archive |
595 |
if ($OPT{keep}) { |
596 |
$TAR = "$OPT{keep}/$ARCHIVE{$tar}"; |
597 |
print "Keeping in '$TAR'\n" unless -e $TAR; |
598 |
my ($base, $path) = fileparse($TAR); |
599 |
unless (-d $path) { |
600 |
mkpath($path,1,0755) or |
601 |
die "Could not mkpath($path)\n"; |
602 |
} |
603 |
} else { |
604 |
$TAR = "/tmp/$tar.tar.gz"; |
605 |
} |
606 |
unless (-e $TAR) { # lwp mirror seems to fetch ftp: in any case? |
607 |
print "Fetching $OPT{cpan}/$ARCHIVE{$tar}\n"; |
608 |
my $status = mirror("$OPT{cpan}/$ARCHIVE{$tar}", $TAR); |
609 |
if ($status != &RC_OK and $status != &RC_NOT_MODIFIED) { |
610 |
warn "Was unable to mirror '$ARCHIVE{$tar}, skipping'\n"; |
611 |
return; |
612 |
} |
613 |
} |
614 |
} else { |
615 |
$TAR = $ARCHIVE{$tar}; |
616 |
} |
617 |
$TAR; |
618 |
} |
619 |
|
620 |
sub index_pod { |
621 |
my %parm = @_; |
622 |
my $did = $parm{file}; |
623 |
my $rel_did = $did; |
624 |
my $abs_did = $did; |
625 |
|
626 |
if ($rel_did =~ s:$DIR/::) { |
627 |
$abs_did = "$DIR/$rel_did"; |
628 |
} |
629 |
|
630 |
undef $did; |
631 |
|
632 |
# check for both variants |
633 |
if ($tb->have('docid' => $rel_did)) { |
634 |
$did = $rel_did; |
635 |
} elsif ($tb->have('docid' => $abs_did)) { |
636 |
$did = $abs_did; |
637 |
} |
638 |
if ($did) { # have it version |
639 |
if (!$parm{remove} and !$OPT{force}) { |
640 |
warn "duplicate: $did\n"; |
641 |
return; |
642 |
} |
643 |
} else { # not seen yet |
644 |
$did = $rel_did; |
645 |
if ($parm{remove}) { |
646 |
print "missing: $did\n"; |
647 |
return; |
648 |
} |
649 |
} |
650 |
|
651 |
$parm{'text'} ||= WAIT::Document::Find->FETCH($abs_did); |
652 |
|
653 |
unless (defined $parm{'text'}) { |
654 |
print "unavailable: $did\n"; |
655 |
return; |
656 |
} |
657 |
|
658 |
my $record = $layout->split($parm{'text'}); |
659 |
|
660 |
if (! $record) { |
661 |
print "empty pod: $did\n"; |
662 |
return; |
663 |
} |
664 |
|
665 |
$record->{size} = length($parm{'text'}); |
666 |
my $headline = $record->{name} || $did; |
667 |
|
668 |
# additional fields for Hyper Estraier |
669 |
$record->{'@mdate'} = $parm{'mdate'} if ($parm{'mdate'}); |
670 |
|
671 |
$headline =~ s/^$DATA//o; # $did |
672 |
$headline =~ s/\s+/ /g; $headline =~ s/^\s+//; |
673 |
|
674 |
printf "%s %s\n", ($parm{remove})?'-':'+', substr($headline,0,70); |
675 |
if ($parm{remove}) { |
676 |
$tb->delete('docid' => $did, |
677 |
headline => $headline, |
678 |
%{$record}); |
679 |
} else { |
680 |
foreach (keys %{$parm{parent}}) { |
681 |
next if (/^_/); |
682 |
$record->{$_} = $parm{parent}->{$_} if ($parm{parent}->{$_}); |
683 |
} |
684 |
$tb->insert('docid' => $did, |
685 |
headline => $headline, |
686 |
source => $parm{source}, |
687 |
parent => $parm{parent}->{_id}, |
688 |
%{$record}); |
689 |
} |
690 |
} |
691 |
|
692 |
# This *must* remove the version in *any* case. It should compute a |
693 |
# resonable version number - but usually mtimes should be used. |
694 |
sub version { |
695 |
local ($_) = @_; |
696 |
|
697 |
# remove alpha/beta postfix |
698 |
s/([-_\d])(a|b|alpha|beta|src)$/$1/; |
699 |
|
700 |
# jperl1.3@4.019.tar.gz |
701 |
s/@\d.\d+//; |
702 |
|
703 |
# oraperl-v2.4-gk.tar.gz |
704 |
s/-v(\d)/$1/; |
705 |
|
706 |
# lettered versions - shudder |
707 |
s/([-_\d\.])([a-z])([\d\._])/sprintf "$1%02d$3", ord(lc $2) - ord('a') /ei; |
708 |
s/([-_\d\.])([a-z])$/sprintf "$1%02d", ord(lc $2) - ord('a') /ei; |
709 |
|
710 |
# thanks libwww-5b12 ;-) |
711 |
s/(\d+)b/($1-1).'.'/e; |
712 |
s/(\d+)a/($1-2).'.'/e; |
713 |
|
714 |
# replace '-pre' by '0.' |
715 |
s/-pre([\.\d])/-0.$1/; |
716 |
s/\.\././g; |
717 |
s/(\d)_(\d)/$1$2/g; |
718 |
|
719 |
# chop '[-.]' and thelike |
720 |
s/\W$//; |
721 |
|
722 |
# ram's versions Storable-0.4@p |
723 |
s/\@/./; |
724 |
|
725 |
if (s/[-_]?(\d+)\.(0\d+)\.(\d+)$//) { |
726 |
return($_, $1 + "0.$2" + $3 / 1000000); |
727 |
} elsif (s/[-_]?(\d+)\.(\d+)\.(\d+)$//) { |
728 |
return($_, $1 + $2/1000 + $3 / 1000000); |
729 |
} elsif (s/[-_]?(\d+\.[\d_]+)$//) { |
730 |
return($_, $1); |
731 |
} elsif (s/[-_]?([\d_]+)$//) { |
732 |
return($_, $1); |
733 |
} elsif (s/-(\d+.\d+)-/-/) { # perl-4.019-ref-guide |
734 |
return($_, $1); |
735 |
} else { |
736 |
if ($_ =~ /\d/) { # smells like an unknown scheme |
737 |
warn "Odd version Numbering: '$File::Find::name'\n"; |
738 |
return($_, undef); |
739 |
} else { # assume version 0 |
740 |
warn "No version Numbering: '$File::Find::name'\n"; |
741 |
return($_, 0); |
742 |
} |
743 |
|
744 |
} |
745 |
} |
746 |
|
747 |
sub wanted { |
748 |
my $mtime = shift; # called by parse_file_ls(); |
749 |
|
750 |
return if (! $max); |
751 |
$max--; |
752 |
|
753 |
return unless /^(.*)\.(tar\.(gz|Z)|tgz)$/; |
754 |
|
755 |
my ($archive, $version) = version($1); |
756 |
|
757 |
unless (defined $version) { |
758 |
warn "Skipping $1\n"; |
759 |
return; |
760 |
} |
761 |
|
762 |
# Check for file alias |
763 |
$archive = $ALIAS{$archive} if $ALIAS{$archive}; |
764 |
|
765 |
# Check for path alias. |
766 |
if ($File::Find::name =~ m(/CPAN/(?:source/)?(.*\Q$archive\E))) { |
767 |
if ($ALIAS{$1}) { |
768 |
$archive = $ALIAS{$1}; |
769 |
} |
770 |
} |
771 |
|
772 |
if ($OPT{trust_mtime}) { |
773 |
$version = $mtime; |
774 |
} else { |
775 |
$version =~ s/(\d)_/$1/; |
776 |
$version ||= $mtime; # mtime |
777 |
} |
778 |
|
779 |
if (!exists $ARCHIVE{$archive} |
780 |
or $VERSION{$archive} < $version) { |
781 |
$ARCHIVE{$archive} = $File::Find::name; |
782 |
$VERSION{$archive} = $version; |
783 |
} |
784 |
} |
785 |
|
786 |
my %MON; |
787 |
my $YEAR; |
788 |
|
789 |
BEGIN { |
790 |
my $i = 1; |
791 |
for (qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)) { |
792 |
$MON{$_} = $i++; |
793 |
} |
794 |
$YEAR = (localtime(time))[5]; |
795 |
} |
796 |
|
797 |
# We could/should use Date::GetDate here |
798 |
use Time::Local; |
799 |
sub mtime { |
800 |
my ($mon, $mday, $time) = @_; |
801 |
my ($hour, $min, $year, $monn) = (0,0); |
802 |
|
803 |
if ($time =~ /(\d+):(\d+)/) { |
804 |
($hour, $min) = ($1, $2); |
805 |
$year = $YEAR; |
806 |
} else { |
807 |
$year = $time; |
808 |
} |
809 |
$monn = $MON{$mon} || $MON{ucfirst lc $mon} || warn "Unknown month: '$mon'"; |
810 |
my $guess = timelocal(0,$min,$hour,$mday,$monn-1,$year); |
811 |
if ($guess > time) { |
812 |
$guess = timelocal(0,$min,$hour,$mday,$monn-1,$year-1); |
813 |
} |
814 |
$guess; |
815 |
} |
816 |
|
817 |
package HyperEstraier::WAIT::Table; |
818 |
|
819 |
use Search::Estraier; |
820 |
use Text::Iconv; |
821 |
|
822 |
=head1 NAME |
823 |
|
824 |
HyperEstraier::WAIT::Table |
825 |
|
826 |
=head1 DESCRIPTION |
827 |
|
828 |
This is a mode that emulates C<WAIT::Table> functionality somewhat. |
829 |
|
830 |
There are some limitations and only one key attribute is supported (and used |
831 |
for C<@uri>). |
832 |
|
833 |
=head2 Porting from WAIT to this module. |
834 |
|
835 |
Since only one key is supported (and used as C<@uri> attribute), |
836 |
use first parametar of C<keyset> as C<key>. |
837 |
|
838 |
Full text index is specified as C<invindex>, but you need just name of fields. |
839 |
|
840 |
You will probably need to add |
841 |
|
842 |
use WAIT::Parse::Base; |
843 |
|
844 |
to your code after you remove C<WAIT::Config> and C<WAIT::Database>. |
845 |
|
846 |
=head1 METHODS |
847 |
|
848 |
=head2 new |
849 |
|
850 |
my $tb = new HyperEstraier::WAIT::Table( |
851 |
uri => 'http://localhost:1978/node/cpan', |
852 |
attr => qw/docid headline source size parent/, |
853 |
key => 'docid', |
854 |
invindex => qw/name synopsis bugs description text environment example author/, |
855 |
); |
856 |
|
857 |
=cut |
858 |
|
859 |
sub new { |
860 |
my $class = shift; |
861 |
my $self = {@_}; |
862 |
bless($self, $class); |
863 |
|
864 |
foreach my $p (qw/uri attr key invindex/) { |
865 |
die "need $p" unless ($self->{$p}); |
866 |
} |
867 |
|
868 |
$self->{'iso2utf'} = Text::Iconv->new('ISO-8859-1','UTF-8'); |
869 |
|
870 |
my $node = Search::Estraier::Node->new($self->{'uri'}); |
871 |
$node->set_auth('admin', 'admin'); |
872 |
|
873 |
$self->{'node'} = $node; |
874 |
|
875 |
$self ? return $self : return undef; |
876 |
} |
877 |
|
878 |
=head2 have |
879 |
|
880 |
if ( $tb->have(docid => $something) ) ... |
881 |
|
882 |
=cut |
883 |
|
884 |
sub have { |
885 |
my $self = shift; |
886 |
my $args = {@_}; |
887 |
my $key = $self->{'key'} || die "no key in object"; |
888 |
my $key_v = $args->{$key} || die "no key $key in data"; |
889 |
|
890 |
my $id = $self->{'node'}->uri_to_id('file://' . $key_v); |
891 |
|
892 |
return unless($id); |
893 |
|
894 |
return ($id == -1 ? undef : $id); |
895 |
} |
896 |
|
897 |
=head2 insert |
898 |
|
899 |
my $key = $tb->insert( |
900 |
docid => $base, |
901 |
headline => 'Something', |
902 |
... |
903 |
); |
904 |
|
905 |
=cut |
906 |
|
907 |
sub insert { |
908 |
my $self = shift; |
909 |
my $args = {@_}; |
910 |
|
911 |
my $uri = 'file://'; |
912 |
$uri .= $args->{'docid'} or die "no docid"; |
913 |
|
914 |
my $doc = Search::Estraier::Document->new; |
915 |
|
916 |
$doc->add_attr('@uri', $uri); |
917 |
$doc->add_attr('@title', $args->{'headline'}) if ($args->{'headline'}); |
918 |
$doc->add_attr('@size', $args->{'size'}) if ($args->{'size'}); |
919 |
|
920 |
my @attr = $self->{'attr'} || die "no attr in object"; |
921 |
my @invindex = $self->{'invindex'} || die "no invindex in object"; |
922 |
|
923 |
foreach my $attr (keys %{$args}) { |
924 |
if (grep(/^$attr$/, @{ $self->{'attr'} }) or $attr =~ m/^@/o) { |
925 |
$doc->add_attr($attr, $args->{$attr}); |
926 |
} |
927 |
if (grep(/^$attr$/, @{ $self->{'invindex'} })) { |
928 |
$doc->add_text($args->{$attr}); |
929 |
} |
930 |
} |
931 |
|
932 |
print STDERR $doc->dump_draft if ($self->{'debug'}); |
933 |
|
934 |
my $id; |
935 |
unless ($self->{'node'}->put_doc($doc)) { |
936 |
printf STDERR "ERROR: %d\n", $self->{'node'}->status; |
937 |
} else { |
938 |
$id = $self->{'node'}->uri_to_id( $uri ); |
939 |
if ($id != -1) { |
940 |
print STDERR "id: $id\n" if ($self->{'debug'}) |
941 |
} else { |
942 |
print STDERR "ERROR: can't find id for newly insrted document $uri\n"; |
943 |
} |
944 |
} |
945 |
|
946 |
return $id; |
947 |
} |
948 |
|
949 |
=head2 delete_by_key |
950 |
|
951 |
$tb->delete_by_key($key); |
952 |
|
953 |
=cut |
954 |
|
955 |
sub delete_by_key { |
956 |
my $self = shift; |
957 |
my $key_v = shift || die "no key?"; |
958 |
|
959 |
unless ($self->{'node'}->out_doc_by_uri( 'file://' . $key_v )) { |
960 |
print STDERR "WARNING: can't delete document $key_v\n"; |
961 |
} |
962 |
} |
963 |
|
964 |
=head2 delete |
965 |
|
966 |
$tb->delete( docid => $did, ... ); |
967 |
|
968 |
=cut |
969 |
|
970 |
sub delete { |
971 |
my $self = shift; |
972 |
my $args = {@_}; |
973 |
|
974 |
my $key = $self->{'key'} || die "no key in object"; |
975 |
|
976 |
die "no $key in data" unless (my $key_v = $args->{$key}); |
977 |
|
978 |
$self->delete_by_key($key_v); |
979 |
|
980 |
} |