trunk/data/indexer.pl

#!/usr/bin/perl -w

use strict;
use HyperEstraier;

# score for words in title
my $title_rank = 3;

open(my $t, "gzip -cd trivia.list.gz |") || die "can't open trivia.list.gz: $!";

# open the database
my $db = HyperEstraier::Database->new();
$db->open('casket', $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);


my $header = 1;
my ($title, $year) = ('',undef);
my $in = 0;
my $trivia = '';
my @qv;

my ($all_years,$all_titles,$all_quotes);

sub qv {
        my $t = shift || return;

        sub qv_print {
                my $v = shift || return '';
#               $v =~ s/(.*)\s*,\s*(.+)/$2 $1/g;
                push @qv, $v;
                $all_quotes->{$v}++;
                return '';
        }
        $t =~ s#([_'])([^_']+?)\1 \(qv\)#qv_print($2)#ge;
}

my $i = 0;

while(<$t>) {
        if ($header && /^=====+/) {
                $header = 0;
                next;
        }
        next if $header;

        if (/^#\s+(.*)\s*$/) {
                $title = $1;
                if ($title =~ m#^("*)(.*)\1\s*\((\d+)\)(:?\s*\(\w+\))*$#) {
                        $year = $3;
                        $all_titles->{$2}++;
                        $all_years->{$3}++;
                } else {
                        $year = undef;
                }

                print "# $title ", ( $year ? "[$year]" : "" ), "\n";
                next;

        } elsif (/^-\s(.*)\s*$/) {
                $in = 1;
                $trivia = "$1\n";
                qv($1);
        } elsif (/^\s\s(.*)\s*$/) {
                $trivia .= "$1\n";
                qv($1);
        } elsif (/^$/ && $in) {
                $i++;
                print "$i ";

                # create a document object 
                my $doc = HyperEstraier::Document->new;

                # add attributes to the document object 
                $doc->add_attr('@uri', "file://localhost/trivia/$i");
                $doc->add_attr('@title', $title);
                $doc->add_attr('@size', length($trivia));

                $doc->add_attr('year', $year) if ($year);
                foreach my $q (@qv) {
                        $doc->add_attr('quote', $q);
                        $doc->add_hidden_text($q);
                }

                # add the body text to the document object 
                $doc->add_text($trivia);
                $doc->add_hidden_text(
                        ($title x $title_rank)
                );


                # register the document object to the database
                $db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);

                $trivia = '';
                @qv = ();
                $in = 0;
        } else {
                print "#$_\n";
        }

#       last if ($i > 1000);    # XXX remove this!
}

sub dump_data($$) {
        my ($name,$hash) = @_;

        open(my $fh, "> $name") || die "can't open $name: $!";

        foreach my $k (sort keys %{$hash}) {
                print $fh "$k\t",$hash->{$k},"\n";
        }

        close($fh);
}

dump_data('titles.data', $all_titles);
dump_data('quotes.data', $all_quotes);
dump_data('years.data', $all_years);

#print "\noptimizing...\n";
#$db->optimize(0);
1	#!/usr/bin/perl -w
2
3	use strict;
4	use HyperEstraier;
5
6	# score for words in title
7	my $title_rank = 3;
8
9	open(my $t, "gzip -cd trivia.list.gz \|") \|\| die "can't open trivia.list.gz: $!";
10
11	# open the database
12	my $db = HyperEstraier::Database->new();
13	$db->open('casket', $HyperEstraier::Database::DBWRITER \| $HyperEstraier::Database::DBCREAT);
14
15
16	my $header = 1;
17	my ($title, $year) = ('',undef);
18	my $in = 0;
19	my $trivia = '';
20	my @qv;
21
22	my ($all_years,$all_titles,$all_quotes);
23
24	sub qv {
25	my $t = shift \|\| return;
26
27	sub qv_print {
28	my $v = shift \|\| return '';
29	# $v =~ s/(.)\s,\s*(.+)/$2 $1/g;
30	push @qv, $v;
31	$all_quotes->{$v}++;
32	return '';
33	}
34	$t =~ s#([_'])([^_']+?)\1 \(qv\)#qv_print($2)#ge;
35	}
36
37	my $i = 0;
38
39	while(<$t>) {
40	if ($header && /^=====+/) {
41	$header = 0;
42	next;
43	}
44	next if $header;
45
46	if (/^#\s+(.)\s$/) {
47	$title = $1;
48	if ($title =~ m#^(")(.)\1\s\((\d+)\)(:?\s\(\w+\))*$#) {
49	$year = $3;
50	$all_titles->{$2}++;
51	$all_years->{$3}++;
52	} else {
53	$year = undef;
54	}
55
56	print "# $title ", ( $year ? "[$year]" : "" ), "\n";
57	next;
58
59	} elsif (/^-\s(.)\s$/) {
60	$in = 1;
61	$trivia = "$1\n";
62	qv($1);
63	} elsif (/^\s\s(.)\s$/) {
64	$trivia .= "$1\n";
65	qv($1);
66	} elsif (/^$/ && $in) {
67	$i++;
68	print "$i ";
69
70	# create a document object
71	my $doc = HyperEstraier::Document->new;
72
73	# add attributes to the document object
74	$doc->add_attr('@uri', "file://localhost/trivia/$i");
75	$doc->add_attr('@title', $title);
76	$doc->add_attr('@size', length($trivia));
77
78	$doc->add_attr('year', $year) if ($year);
79	foreach my $q (@qv) {
80	$doc->add_attr('quote', $q);
81	$doc->add_hidden_text($q);
82	}
83
84	# add the body text to the document object
85	$doc->add_text($trivia);
86	$doc->add_hidden_text(
87	($title x $title_rank)
88	);
89
90
91	# register the document object to the database
92	$db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
93
94	$trivia = '';
95	@qv = ();
96	$in = 0;
97	} else {
98	print "#$_\n";
99	}
100
101	# last if ($i > 1000); # XXX remove this!
102	}
103
104	sub dump_data($$) {
105	my ($name,$hash) = @_;
106
107	open(my $fh, "> $name") \|\| die "can't open $name: $!";
108
109	foreach my $k (sort keys %{$hash}) {
110	print $fh "$k\t",$hash->{$k},"\n";
111	}
112
113	close($fh);
114	}
115
116	dump_data('titles.data', $all_titles);
117	dump_data('quotes.data', $all_quotes);
118	dump_data('years.data', $all_years);
119
120	#print "\noptimizing...\n";
121	#$db->optimize(0);