trunk/data/convert.pl

#!/usr/bin/perl -w

use strict;

# score for words in title
my $title_rank = 3;

open(my $t, "gzip -cd trivia.list.gz |") || die "can't open trivia.list.gz: $!";


my $header = 1;
my ($title, $year) = ('',undef);
my $in = 0;
my $trivia = '';
my @qv;

my ($all_years,$all_titles,$all_quotes);

sub qv {
        my $t = shift || return;

        sub qv_print {
                my $v = shift || return '';
#               $v =~ s/(.*)\s*,\s*(.+)/$2 $1/g;
                push @qv, $v;
                $all_quotes->{$v}++;
                return '';
        }
        $t =~ s#([_'])([^_']+?)\1 \(qv\)#qv_print($2)#ge;
}

my $i = 0;

while(<$t>) {
        if ($header && /^=====+/) {
                $header = 0;
                next;
        }
        next if $header;

        if (/^#\s+(.*)\s*$/) {
                $title = $1;
                if ($title =~ m#^("*)(.*)\1\s*\((\d+)\)(:?\s*\(\w+\))*$#) {
                        $year = $3;
                        $all_titles->{$2}++;
                        $all_years->{$3}++;
                } else {
                        $year = undef;
                }

                print "# $title ", ( $year ? "[$year]" : "" ), "\n";
                next;

        } elsif (/^-\s(.*)\s*$/) {
                $in = 1;
                $trivia = "$1\n";
                qv($1);
        } elsif (/^\s\s(.*)\s*$/) {
                $trivia .= "$1\n";
                qv($1);
        } elsif (/^$/ && $in) {
                $i++;
                print "$i ";
                my $out = '@title=' . $title . "\n" .
                        '@size=' . length($trivia) . "\n";
                $out .= "year=$year\n" if ($year);
                $out .= 'quote=' . join(",", @qv) . "\n" if (@qv);
                $out .= "\n$trivia\n" .
                        ("\t$title\n" x $title_rank) .
                        "\t" . join("\n\t", @qv) . "\n";

                open(my $t, "> trivia/$i.est") || die "can't open $i.est: $!";
                print $t $out;
                close($t);
                $trivia = '';
                @qv = ();
                $in = 0;
        } else {
                print "#$_\n";
        }

#       last if ($i > 1000);    # XXX remove this!
}

sub dump_data($$) {
        my ($name,$hash) = @_;

        open(my $fh, "> $name") || die "can't open $name: $!";

        foreach my $k (sort keys %{$hash}) {
                print $fh "$k\t",$hash->{$k},"\n";
        }

        close($fh);
}

dump_data('titles.data', $all_titles);
dump_data('quotes.data', $all_quotes);
dump_data('years.data', $all_years);
1	#!/usr/bin/perl -w
2
3	use strict;
4
5	# score for words in title
6	my $title_rank = 3;
7
8	open(my $t, "gzip -cd trivia.list.gz \|") \|\| die "can't open trivia.list.gz: $!";
9
10
11	my $header = 1;
12	my ($title, $year) = ('',undef);
13	my $in = 0;
14	my $trivia = '';
15	my @qv;
16
17	my ($all_years,$all_titles,$all_quotes);
18
19	sub qv {
20	my $t = shift \|\| return;
21
22	sub qv_print {
23	my $v = shift \|\| return '';
24	# $v =~ s/(.)\s,\s*(.+)/$2 $1/g;
25	push @qv, $v;
26	$all_quotes->{$v}++;
27	return '';
28	}
29	$t =~ s#([_'])([^_']+?)\1 \(qv\)#qv_print($2)#ge;
30	}
31
32	my $i = 0;
33
34	while(<$t>) {
35	if ($header && /^=====+/) {
36	$header = 0;
37	next;
38	}
39	next if $header;
40
41	if (/^#\s+(.)\s$/) {
42	$title = $1;
43	if ($title =~ m#^(")(.)\1\s\((\d+)\)(:?\s\(\w+\))*$#) {
44	$year = $3;
45	$all_titles->{$2}++;
46	$all_years->{$3}++;
47	} else {
48	$year = undef;
49	}
50
51	print "# $title ", ( $year ? "[$year]" : "" ), "\n";
52	next;
53
54	} elsif (/^-\s(.)\s$/) {
55	$in = 1;
56	$trivia = "$1\n";
57	qv($1);
58	} elsif (/^\s\s(.)\s$/) {
59	$trivia .= "$1\n";
60	qv($1);
61	} elsif (/^$/ && $in) {
62	$i++;
63	print "$i ";
64	my $out = '@title=' . $title . "\n" .
65	'@size=' . length($trivia) . "\n";
66	$out .= "year=$year\n" if ($year);
67	$out .= 'quote=' . join(",", @qv) . "\n" if (@qv);
68	$out .= "\n$trivia\n" .
69	("\t$title\n" x $title_rank) .
70	"\t" . join("\n\t", @qv) . "\n";
71
72	open(my $t, "> trivia/$i.est") \|\| die "can't open $i.est: $!";
73	print $t $out;
74	close($t);
75	$trivia = '';
76	@qv = ();
77	$in = 0;
78	} else {
79	print "#$_\n";
80	}
81
82	# last if ($i > 1000); # XXX remove this!
83	}
84
85	sub dump_data($$) {
86	my ($name,$hash) = @_;
87
88	open(my $fh, "> $name") \|\| die "can't open $name: $!";
89
90	foreach my $k (sort keys %{$hash}) {
91	print $fh "$k\t",$hash->{$k},"\n";
92	}
93
94	close($fh);
95	}
96
97	dump_data('titles.data', $all_titles);
98	dump_data('quotes.data', $all_quotes);
99	dump_data('years.data', $all_years);