trunk/data/indexer.pl

#!/usr/bin/perl -w

use strict;
use Search::Estraier 0.06;

# score for words in title
my $title_rank = 3;

open(my $t, "gzip -cd trivia.list.gz |") || die "can't open trivia.list.gz: $!";

# open node
my $node = Search::Estraier::Node->new(
        url => 'http://localhost:1978/node/trivia',
        user => 'admin',
        passwd => 'admin',
        create => 1,
);


my $header = 1;
my ($title, $year) = ('',undef);
my $in = 0;
my $trivia = '';
my @qv;

my ($all_years,$all_titles,$all_quotes);

sub qv {
        my $t = shift || return;

        sub qv_print {
                my $v = shift || return '';
#               $v =~ s/(.*)\s*,\s*(.+)/$2 $1/g;
                push @qv, $v;
                $all_quotes->{$v}++;
                return '';
        }
        $t =~ s#([_'"])([^_'"]+?)\1 \(qv\)#qv_print($2)#ge;
}

my $i = 0;

while(<$t>) {
        if ($header && /^=====+/) {
                $header = 0;
                next;
        }
        next if $header;

        if (/^#\s+(.*)\s*$/) {
                $title = $1;
                if ($title =~ m#^("*)(.*)\1\s*\((\d+)\)(:?\s*\(\w+\))*$#) {
                        ($title, $year) = ($2,$3);
                        $all_titles->{$title}++;
                        $all_years->{$3}++;
                } else {
                        $year = undef;
                }

                print "# $title ", ( $year ? "[$year]" : "" ), "\n";
                next;

        } elsif (/^-\s(.*)\s*$/) {
                $in = 1;
                $trivia = "$1\n";
                qv($1);
        } elsif (/^\s\s(.*)\s*$/) {
                $trivia .= "$1\n";
                qv($1);
        } elsif (/^$/ && $in) {
                $i++;
                print "$i ";

                # create a document object 
                my $doc = Search::Estraier::Document->new;

                # add attributes to the document object 
                $doc->add_attr('@uri', "file://localhost/trivia/$i");

                $doc->add_attr('@title', $title);
                $doc->add_hidden_text(
                        ($title x $title_rank)
                );

                $doc->add_attr('@size', length($trivia));

                $doc->add_attr('year', $year) if ($year);
                foreach my $q (@qv) {
                        $doc->add_attr('quote', $q);
                        $doc->add_hidden_text($q);
                }

                # add the body text to the document object 
                $doc->add_text($trivia);


                # register the document object to the database
                $node->put_doc($doc);

                $trivia = '';
                @qv = ();
                $in = 0;
        } else {
                print "#$_\n";
        }

#       last if ($i > 1000);    # XXX remove this!
}

sub dump_data($$) {
        my ($name,$hash) = @_;

        open(my $fh, "> $name") || die "can't open $name: $!";

        foreach my $k (sort keys %{$hash}) {
                print $fh "$k\t",$hash->{$k},"\n";
        }

        close($fh);
}

dump_data('titles.data', $all_titles);
dump_data('quotes.data', $all_quotes);
dump_data('years.data', $all_years);

1	#!/usr/bin/perl -w
2
3	use strict;
4	use Search::Estraier 0.06;
5
6	# score for words in title
7	my $title_rank = 3;
8
9	open(my $t, "gzip -cd trivia.list.gz \|") \|\| die "can't open trivia.list.gz: $!";
10
11	# open node
12	my $node = Search::Estraier::Node->new(
13	url => 'http://localhost:1978/node/trivia',
14	user => 'admin',
15	passwd => 'admin',
16	create => 1,
17	);
18
19
20	my $header = 1;
21	my ($title, $year) = ('',undef);
22	my $in = 0;
23	my $trivia = '';
24	my @qv;
25
26	my ($all_years,$all_titles,$all_quotes);
27
28	sub qv {
29	my $t = shift \|\| return;
30
31	sub qv_print {
32	my $v = shift \|\| return '';
33	# $v =~ s/(.)\s,\s*(.+)/$2 $1/g;
34	push @qv, $v;
35	$all_quotes->{$v}++;
36	return '';
37	}
38	$t =~ s#([_'"])([^_'"]+?)\1 \(qv\)#qv_print($2)#ge;
39	}
40
41	my $i = 0;
42
43	while(<$t>) {
44	if ($header && /^=====+/) {
45	$header = 0;
46	next;
47	}
48	next if $header;
49
50	if (/^#\s+(.)\s$/) {
51	$title = $1;
52	if ($title =~ m#^(")(.)\1\s\((\d+)\)(:?\s\(\w+\))*$#) {
53	($title, $year) = ($2,$3);
54	$all_titles->{$title}++;
55	$all_years->{$3}++;
56	} else {
57	$year = undef;
58	}
59
60	print "# $title ", ( $year ? "[$year]" : "" ), "\n";
61	next;
62
63	} elsif (/^-\s(.)\s$/) {
64	$in = 1;
65	$trivia = "$1\n";
66	qv($1);
67	} elsif (/^\s\s(.)\s$/) {
68	$trivia .= "$1\n";
69	qv($1);
70	} elsif (/^$/ && $in) {
71	$i++;
72	print "$i ";
73
74	# create a document object
75	my $doc = Search::Estraier::Document->new;
76
77	# add attributes to the document object
78	$doc->add_attr('@uri', "file://localhost/trivia/$i");
79
80	$doc->add_attr('@title', $title);
81	$doc->add_hidden_text(
82	($title x $title_rank)
83	);
84
85	$doc->add_attr('@size', length($trivia));
86
87	$doc->add_attr('year', $year) if ($year);
88	foreach my $q (@qv) {
89	$doc->add_attr('quote', $q);
90	$doc->add_hidden_text($q);
91	}
92
93	# add the body text to the document object
94	$doc->add_text($trivia);
95
96
97	# register the document object to the database
98	$node->put_doc($doc);
99
100	$trivia = '';
101	@qv = ();
102	$in = 0;
103	} else {
104	print "#$_\n";
105	}
106
107	# last if ($i > 1000); # XXX remove this!
108	}
109
110	sub dump_data($$) {
111	my ($name,$hash) = @_;
112
113	open(my $fh, "> $name") \|\| die "can't open $name: $!";
114
115	foreach my $k (sort keys %{$hash}) {
116	print $fh "$k\t",$hash->{$k},"\n";
117	}
118
119	close($fh);
120	}
121
122	dump_data('titles.data', $all_titles);
123	dump_data('quotes.data', $all_quotes);
124	dump_data('years.data', $all_years);
125