trunk/lib/MWS.pm

#!/usr/bin/perl -w

package MWS;

use strict;
use warnings;

use lib '.';

use Carp;
use Mail::Box::Manager;
use Config::IniFiles;
use POSIX qw(strftime);
use Text::Autoformat;
use Text::Iconv;
use Text::Unaccent;
use Date::Parse;
use POSIX qw(strftime);
use MIME::Base64;

our $VERSION = '1.00';

my $debug = 1;

my @counters = qw(from to cc bcc folder);

sub new {
        my $class = shift;
        my $self = {@_};
        bless($self, $class);


        my $config_name = $self->{config_file} || croak "need config_file";
        $config_name =~ s/\.conf.*$//;
        $self->{config_name} = $config_name;

        $self->{config} = new Config::IniFiles( -file => $self->{config_file} );

        my $index_dir = $self->{config}->val('global', 'index') || croak "can't find [index] section in config file with path of index";

        $self->{mgr} = Mail::Box::Manager->new(access => 'r');
        $self->{index_dir} = $index_dir;

        # placeholder for opened folders
        $self->{folder} = {};

        $self->{wrap_margin} = $self->{config}->val('global', 'wrap_margin');
        $self->{max_results} = $self->{config}->val('global', 'max_results') || 100;
        $self->reset_counters;

        return $self;
}

sub normalize_string {
        my $self = shift;

        my $v = shift || return;

        $v = unac_string('ISO-8859-2', $v);
        $v = join('',sort split(/\s+/,$v));
        $v =~ s/\W+//g;

        return lc($v);
}

# reset tables for search results
sub reset_counters {
        my $self = shift;

        $self->{counter} = {};

#       foreach my $c (qw(thread from to cc bcc lists links att)) {
#               $self->{counter}->{$c} = {};
#       }

}

sub add_counter($$) {
        my $self = shift;

        my ($c,$v) = @_;
        my $k = $self->normalize_string($v);

        $self->{counter}->{$c}->{$k}->{name} = $v || return;
        return $self->{counter}->{$c}->{$k}->{usage}++;
}

sub yyyymmdd {
        my $self = shift;

        my $t = shift || time;

        my (undef,undef,undef,$dd,$mm,$yyyy) = localtime($t);
        $mm++;
        $yyyy+=1900;
        return ($yyyy,$mm,$dd);
}

sub fmtdate {
        my $self = shift;

        my @out;
        my @formats = qw(%04d %02d %02d);
        while (my $v = shift) {
                my $f = shift @formats;
                push @out, sprintf($f, $v);
        }

        print STDERR "fmtdate: ",join('|',@out),"\n" if ($debug == 2);

        return (wantarray ? @out : join("-",@out));
}

sub add_counter_calendar($) {
        my $self = shift;

        my $t = shift || return;
        
        my ($yyyy,$mm,$dd) = $self->fmtdate($self->yyyymmdd($t));

        return $self->{counter}->{calendar}->{"$yyyy-$mm"}->{$dd}++;
}


sub counter {
        my $self = shift;

        my $c = shift || return;

        return if (! $self->{counter}->{$c});

        return $self->{counter}->{$c};
}

sub mbox_name2path {
        my $self = shift;

        my $mbox = shift || croak "folder_name2path needs mbox name";

        return $self->{config}->val('folders', $mbox) || croak "comeone removed folder $mbox from config?";
}

sub open_folder {
        my $self = shift;

        my $mbox = shift || croak "open_folder needs mbox name";

        if (! $self->{folder}->{$mbox}) {
                my $mbox_path = $self->mbox_name2path($mbox);

                print STDERR "about to open_folder($mbox)\n" if ($debug == 2);

                my $folder;

                # is this maildir (no locking anyway)
                if (-d $mbox_path) {
                        $folder = $self->{mgr}->open($mbox_path);
                } else {
                        $folder = $self->{mgr}->open($mbox_path, lock_type=>'none' );
                }

                $self->{folder}->{$mbox} = $folder || croak "can't open folder $mbox at '$mbox_path': $!";

                print STDERR "open_folder($mbox) ok\n" if ($debug);
        }

        $self->{fetch_count} = 0;

        return $self->{folder}->{$mbox};

}

sub close_folder {
        my $self = shift;

        my $mbox = shift || croak "open_folder needs mbox name";

        $self->{folder}->{$mbox}->close(write => 'NEVER') || croak "can't close folder $mbox";

        # XXX this is rather agressive!!!
        $self->{folder} = {};
        return 
}

sub fetch_message {
        my $self = shift;

        my $mbox_id = shift || die "need mbox_id!";
        my ($mbox,$id) = split(/ /,$mbox_id);

        # return message with ID
        print STDERR "fetch $id from $mbox\n" if ($debug);

        if ($self->{fetch_count}++ > 100) {
                $self->close_folder($mbox);
                print STDERR "close_folder($mbox) forced on ",$self->{fetch_count},"iteration\n";
        }
        
        my $msg = $self->open_folder($mbox)->find($id);
        if ($msg) {
                return $msg;
        } else {
                print STDERR "can't find message $id in $mbox. Time to re-index?\n";
                return;
        }
}


sub search {
        my $self = shift;

        carp "search called without argument!" if (! @_);

        $self->reset_counters;

        print STDERR "search(",join(" ",@_),")\n" if ($debug == 2);
        my @index_ids = $self->search_index(@_);

        $self->{'index_ids'} = \@index_ids;

        #my $results = $#index_ids + 1;
        #$self->{'results'} = $results;
        
        my $results = $self->{'total_hits'} || ($#index_ids + 1);

        $self->{'curr_result'} = 0;

        $self->reset_counters;

        print STDERR "$results results\n" if ($debug == 2);

        return $results || 'error';
}

sub decode_qp($) {
        my $self = shift;

        my $tmp = shift || return;

        sub decode($$$) {
                my ($cp,$enc,$qp) = @_;

                print STDERR "decode($cp,$qp) -> " if ($debug == 2);

                if (uc($enc) eq "Q") {
                        $qp =~ s/=([a-f0-9][a-f0-9])/chr(hex($1))/ieg;
                        $qp =~ s/_/ /g;
                } elsif (uc($enc) eq "B") {
                        $qp = decode_base64($qp);
                } else {
                        croak "unsupported encoding '$enc' in decode_qp\n";
                        return $qp;
                }

                print STDERR "$qp\n" if ($debug == 2);

                my $iconv;
                eval '$iconv = Text::Iconv->new($cp,"ISO-8859-2")';
                if ($iconv) {
                        return $iconv->convert($qp) || '';
                } else {
                        return $qp;
                }
        }

        $tmp =~ s/=\?([^\?]+)\?([QB])\?(.+?)\?=/decode($1,$2,$3)/ige;
        $tmp =~ s/^\s*["']+(.*?)["']+\s*$/$1/g;
        #print STDERR "$tmp\n" if ($debug == 2);
        return $tmp;
}

sub unroll($$$) {
        my $self = shift;

        my ($message,$part,$sub) = @_;

        my @arr;

        return if (! $message->$part);

        foreach my $from ($message->$part) {
                my $tmp = $from->$sub || next;

                $tmp = $self->decode_qp($tmp);
                push @arr, $tmp;
        }

        return @arr;
}

sub fetch_all_results {
        my $self = shift;

        croak "results called before search!" if (! $self->{'index_ids'});

        print STDERR "fetch_all_results [",scalar @{$self->{'index_ids'}},"]\n" if ($debug == 2);

        my @arr;

        foreach my $id (@{$self->{'index_ids'}}) {
                push @arr, $self->fetch_result_by_id($id);
        }


        return @arr;
}

sub fetch_result {
        my $self = shift;

        my $args = {@_};

        croak "results called before search!" if (! $self->{'index_ids'});

        my $curr = $self->{'curr_result'}++;

        my $id = $self->{'index_ids'}->[$curr];
        
        print STDERR "fetch_result: $curr = $id\n" if ($debug == 2);

        return $self->fetch_result_by_id($id);
}

sub plain_text_body {
        my $self = shift;
        my $message = shift || croak "plain_text_body needs message!";

        my $body;

        if (! $message->isMultipart) {
                $body = $message->decoded->string;
        } else {
                foreach my $part ($message->parts) {
                        if ($part->body->mimeType eq 'text/plain') {
                                $body = $part->decoded->string;
                                last;
                        }
                }
        }

        if (! $body) {
                $body = "[plain/text body not found]" if ($debug == 2);
                print STDERR "plain/text body not found\n" if ($debug);
                return;
        }

        # reformat with Text::Autoformat
        my $wrap = $self->{wrap_margin};
        if ($wrap && $body && $body =~ m/^.{$wrap}..*$/m) {
                eval '$body = autoformat($body, {right=>$wrap, all=>1});';
                $body .="\n[reformated using autoformat, margin at $wrap]" if ($debug == 2);
        }

        return $body;
}


sub fetch_result_by_id {
        my $self = shift;

        my $id = shift || return;

        my $row = $self->{cache}->{$id};

        if (! $row) {

                print STDERR "fetch_result_by_id($id) not in cache, hitting disk\n" if ($debug == 2);

                my $message = $self->fetch_message($id) || return;

                $row->{'id'} = $id;

                foreach my $p (@counters) {
                        foreach my $v ($self->unroll($message,$p,'phrase')) {
                                push @{$row->{$p}},$v;
                                $self->add_counter($p,$v);
                        }
                }

                $self->add_counter('folder', $1) if ($id =~ m/^(\S+)\s/);

                $row->{'subject'} = $self->decode_qp($message->subject);
                $row->{'body'} = $self->plain_text_body($message);
                my $utime = str2time($message->date);

                $row->{'date_utime'} = $utime;

                $row->{'date'} = strftime("%Y-%m-%d %H:%M:%S", localtime($utime));
                $self->add_counter_calendar($utime);

                # XXX store in cache?
                $self->{cache}->{$id} = $row;
                print STDERR "$id stored in cache\n" if ($debug == 2);
        } else {
                print STDERR "fetch_result_by_id($id) in cache\n" if ($debug == 2);
                foreach my $p (@counters) {
                        foreach my $v (@{$row->{$p}}) {
                                $self->add_counter($p,$v);
                        }
                }

                $self->add_counter('folder', $1) if ($id =~ m/^(\S+)\s/);

                $self->add_counter_calendar($row->{date_utime});
        }

        return $row;
        
}
1	dpavlin	2	#!/usr/bin/perl -w
2
3			package MWS;
4
5			use strict;
6			use warnings;
7	dpavlin	41
8			use lib '.';
9
10	dpavlin	2	use Carp;
11	dpavlin	3	use Mail::Box::Manager;
12	dpavlin	4	use Config::IniFiles;
13	dpavlin	13	use POSIX qw(strftime);
14	dpavlin	14	use Text::Autoformat;
15	dpavlin	16	use Text::Iconv;
16	dpavlin	19	use Text::Unaccent;
17	dpavlin	20	use Date::Parse;
18			use POSIX qw(strftime);
19	dpavlin	21	use MIME::Base64;
20	dpavlin	14
21	dpavlin	2	our $VERSION = '1.00';
22
23	dpavlin	30	my $debug = 1;
24	dpavlin	2
25	dpavlin	42	my @counters = qw(from to cc bcc folder);
26
27	dpavlin	2	sub new {
28			my $class = shift;
29	dpavlin	41	my $self = {@_};
30	dpavlin	2	bless($self, $class);
31
32
33	dpavlin	41	my $config_name = $self->{config_file} \|\| croak "need config_file";
34			$config_name =~ s/\.conf.*$//;
35			$self->{config_name} = $config_name;
36	dpavlin	2
37	dpavlin	41	$self->{config} = new Config::IniFiles( -file => $self->{config_file} );
38	dpavlin	4
39	dpavlin	27	my $index_dir = $self->{config}->val('global', 'index') \|\| croak "can't find [index] section in config file with path of index";
40
41	dpavlin	12	$self->{mgr} = Mail::Box::Manager->new(access => 'r');
42	dpavlin	27	$self->{index_dir} = $index_dir;
43	dpavlin	4
44	dpavlin	2	# placeholder for opened folders
45			$self->{folder} = {};
46
47	dpavlin	14	$self->{wrap_margin} = $self->{config}->val('global', 'wrap_margin');
48	dpavlin	17	$self->{max_results} = $self->{config}->val('global', 'max_results') \|\| 100;
49	dpavlin	19	$self->reset_counters;
50	dpavlin	14
51	dpavlin	2	return $self;
52			}
53
54	dpavlin	19	sub normalize_string {
55			my $self = shift;
56
57			my $v = shift \|\| return;
58
59			$v = unac_string('ISO-8859-2', $v);
60			$v = join('',sort split(/\s+/,$v));
61			$v =~ s/\W+//g;
62
63	dpavlin	23	return lc($v);
64	dpavlin	19	}
65
66			# reset tables for search results
67			sub reset_counters {
68			my $self = shift;
69
70			$self->{counter} = {};
71
72			# foreach my $c (qw(thread from to cc bcc lists links att)) {
73			# $self->{counter}->{$c} = {};
74			# }
75
76			}
77
78			sub add_counter($$) {
79			my $self = shift;
80
81			my ($c,$v) = @_;
82			my $k = $self->normalize_string($v);
83
84	dpavlin	42	$self->{counter}->{$c}->{$k}->{name} = $v \|\| return;
85	dpavlin	19	return $self->{counter}->{$c}->{$k}->{usage}++;
86			}
87
88	dpavlin	20	sub yyyymmdd {
89			my $self = shift;
90
91			my $t = shift \|\| time;
92
93			my (undef,undef,undef,$dd,$mm,$yyyy) = localtime($t);
94			$mm++;
95			$yyyy+=1900;
96			return ($yyyy,$mm,$dd);
97			}
98
99			sub fmtdate {
100			my $self = shift;
101
102			my @out;
103			my @formats = qw(%04d %02d %02d);
104			while (my $v = shift) {
105			my $f = shift @formats;
106			push @out, sprintf($f, $v);
107			}
108
109	dpavlin	30	print STDERR "fmtdate: ",join('\|',@out),"\n" if ($debug == 2);
110	dpavlin	20
111			return (wantarray ? @out : join("-",@out));
112			}
113
114			sub add_counter_calendar($) {
115			my $self = shift;
116
117	dpavlin	51	my $t = shift \|\| return;
118	dpavlin	20
119			my ($yyyy,$mm,$dd) = $self->fmtdate($self->yyyymmdd($t));
120
121			return $self->{counter}->{calendar}->{"$yyyy-$mm"}->{$dd}++;
122			}
123
124
125	dpavlin	19	sub counter {
126			my $self = shift;
127
128			my $c = shift \|\| return;
129
130			return if (! $self->{counter}->{$c});
131
132			return $self->{counter}->{$c};
133			}
134
135	dpavlin	14	sub mbox_name2path {
136			my $self = shift;
137
138			my $mbox = shift \|\| croak "folder_name2path needs mbox name";
139
140			return $self->{config}->val('folders', $mbox) \|\| croak "comeone removed folder $mbox from config?";
141			}
142
143	dpavlin	7	sub open_folder {
144	dpavlin	2	my $self = shift;
145
146	dpavlin	7	my $mbox = shift \|\| croak "open_folder needs mbox name";
147	dpavlin	2
148			if (! $self->{folder}->{$mbox}) {
149	dpavlin	14	my $mbox_path = $self->mbox_name2path($mbox);
150
151	dpavlin	13	print STDERR "about to open_folder($mbox)\n" if ($debug == 2);
152	dpavlin	14
153	dpavlin	47	my $folder;
154	dpavlin	14
155	dpavlin	47	# is this maildir (no locking anyway)
156			if (-d $mbox_path) {
157			$folder = $self->{mgr}->open($mbox_path);
158			} else {
159			$folder = $self->{mgr}->open($mbox_path, lock_type=>'none' );
160			}
161
162			$self->{folder}->{$mbox} = $folder \|\| croak "can't open folder $mbox at '$mbox_path': $!";
163
164	dpavlin	14	print STDERR "open_folder($mbox) ok\n" if ($debug);
165	dpavlin	2	}
166
167	dpavlin	16	$self->{fetch_count} = 0;
168
169	dpavlin	7	return $self->{folder}->{$mbox};
170
171			}
172
173	dpavlin	14	sub close_folder {
174			my $self = shift;
175
176			my $mbox = shift \|\| croak "open_folder needs mbox name";
177
178	dpavlin	16	$self->{folder}->{$mbox}->close(write => 'NEVER') \|\| croak "can't close folder $mbox";
179
180			# XXX this is rather agressive!!!
181			$self->{folder} = {};
182			return
183	dpavlin	14	}
184
185	dpavlin	7	sub fetch_message {
186			my $self = shift;
187
188			my $mbox_id = shift \|\| die "need mbox_id!";
189			my ($mbox,$id) = split(/ /,$mbox_id);
190
191			# return message with ID
192	dpavlin	13	print STDERR "fetch $id from $mbox\n" if ($debug);
193	dpavlin	16
194			if ($self->{fetch_count}++ > 100) {
195			$self->close_folder($mbox);
196			print STDERR "close_folder($mbox) forced on ",$self->{fetch_count},"iteration\n";
197			}
198
199	dpavlin	17	my $msg = $self->open_folder($mbox)->find($id);
200			if ($msg) {
201			return $msg;
202			} else {
203	dpavlin	2	print STDERR "can't find message $id in $mbox. Time to re-index?\n";
204	dpavlin	17	return;
205			}
206	dpavlin	2	}
207
208
209			sub search {
210			my $self = shift;
211
212	dpavlin	21	carp "search called without argument!" if (! @_);
213	dpavlin	2
214	dpavlin	19	$self->reset_counters;
215
216	dpavlin	21	print STDERR "search(",join(" ",@_),")\n" if ($debug == 2);
217			my @index_ids = $self->search_index(@_);
218	dpavlin	2
219			$self->{'index_ids'} = \@index_ids;
220
221	dpavlin	17	#my $results = $#index_ids + 1;
222			#$self->{'results'} = $results;
223
224			my $results = $self->{'total_hits'} \|\| ($#index_ids + 1);
225	dpavlin	2
226			$self->{'curr_result'} = 0;
227
228	dpavlin	23	$self->reset_counters;
229
230	dpavlin	13	print STDERR "$results results\n" if ($debug == 2);
231
232	dpavlin	2	return $results \|\| 'error';
233			}
234
235	dpavlin	16	sub decode_qp($) {
236			my $self = shift;
237
238			my $tmp = shift \|\| return;
239
240	dpavlin	21	sub decode($$$) {
241			my ($cp,$enc,$qp) = @_;
242
243			print STDERR "decode($cp,$qp) -> " if ($debug == 2);
244
245			if (uc($enc) eq "Q") {
246			$qp =~ s/=([a-f0-9][a-f0-9])/chr(hex($1))/ieg;
247			$qp =~ s/_/ /g;
248			} elsif (uc($enc) eq "B") {
249			$qp = decode_base64($qp);
250			} else {
251			croak "unsupported encoding '$enc' in decode_qp\n";
252			return $qp;
253			}
254
255	dpavlin	22	print STDERR "$qp\n" if ($debug == 2);
256	dpavlin	21
257	dpavlin	50	my $iconv;
258			eval '$iconv = Text::Iconv->new($cp,"ISO-8859-2")';
259			if ($iconv) {
260			return $iconv->convert($qp) \|\| '';
261			} else {
262			return $qp;
263			}
264	dpavlin	16	}
265
266	dpavlin	21	$tmp =~ s/=\?([^\?]+)\?([QB])\?(.+?)\?=/decode($1,$2,$3)/ige;
267	dpavlin	17	$tmp =~ s/^\s["']+(.?)["']+\s*$/$1/g;
268	dpavlin	20	#print STDERR "$tmp\n" if ($debug == 2);
269	dpavlin	16	return $tmp;
270			}
271
272	dpavlin	2	sub unroll($$$) {
273	dpavlin	13	my $self = shift;
274
275	dpavlin	2	my ($message,$part,$sub) = @_;
276
277			my @arr;
278
279	dpavlin	17	return if (! $message->$part);
280
281	dpavlin	2	foreach my $from ($message->$part) {
282	dpavlin	16	my $tmp = $from->$sub \|\| next;
283
284			$tmp = $self->decode_qp($tmp);
285			push @arr, $tmp;
286	dpavlin	2	}
287	dpavlin	14
288			return @arr;
289	dpavlin	2	}
290	dpavlin	4
291			sub fetch_all_results {
292			my $self = shift;
293
294			croak "results called before search!" if (! $self->{'index_ids'});
295
296	dpavlin	14	print STDERR "fetch_all_results [",scalar @{$self->{'index_ids'}},"]\n" if ($debug == 2);
297	dpavlin	13
298	dpavlin	4	my @arr;
299
300			foreach my $id (@{$self->{'index_ids'}}) {
301			push @arr, $self->fetch_result_by_id($id);
302			}
303
304	dpavlin	19
305	dpavlin	4	return @arr;
306			}
307
308	dpavlin	2	sub fetch_result {
309			my $self = shift;
310
311	dpavlin	3	my $args = {@_};
312
313	dpavlin	2	croak "results called before search!" if (! $self->{'index_ids'});
314
315			my $curr = $self->{'curr_result'}++;
316
317			my $id = $self->{'index_ids'}->[$curr];
318	dpavlin	14
319			print STDERR "fetch_result: $curr = $id\n" if ($debug == 2);
320	dpavlin	2
321	dpavlin	4	return $self->fetch_result_by_id($id);
322			}
323	dpavlin	2
324	dpavlin	7	sub plain_text_body {
325			my $self = shift;
326			my $message = shift \|\| croak "plain_text_body needs message!";
327
328	dpavlin	14	my $body;
329
330	dpavlin	7	if (! $message->isMultipart) {
331	dpavlin	14	$body = $message->decoded->string;
332	dpavlin	7	} else {
333			foreach my $part ($message->parts) {
334			if ($part->body->mimeType eq 'text/plain') {
335	dpavlin	14	$body = $part->decoded->string;
336			last;
337	dpavlin	7	}
338			}
339			}
340	dpavlin	14
341	dpavlin	25	if (! $body) {
342	dpavlin	26	$body = "[plain/text body not found]" if ($debug == 2);
343			print STDERR "plain/text body not found\n" if ($debug);
344			return;
345	dpavlin	25	}
346
347	dpavlin	14	# reformat with Text::Autoformat
348			my $wrap = $self->{wrap_margin};
349			if ($wrap && $body && $body =~ m/^.{$wrap}..*$/m) {
350	dpavlin	50	eval '$body = autoformat($body, {right=>$wrap, all=>1});';
351	dpavlin	14	$body .="\n[reformated using autoformat, margin at $wrap]" if ($debug == 2);
352			}
353
354			return $body;
355	dpavlin	7	}
356
357
358	dpavlin	4	sub fetch_result_by_id {
359			my $self = shift;
360
361			my $id = shift \|\| return;
362
363	dpavlin	13	my $row = $self->{cache}->{$id};
364	dpavlin	2
365	dpavlin	13	if (! $row) {
366	dpavlin	2
367	dpavlin	13	print STDERR "fetch_result_by_id($id) not in cache, hitting disk\n" if ($debug == 2);
368	dpavlin	2
369	dpavlin	17	my $message = $self->fetch_message($id) \|\| return;
370	dpavlin	13
371			$row->{'id'} = $id;
372	dpavlin	19
373	dpavlin	42	foreach my $p (@counters) {
374	dpavlin	22	foreach my $v ($self->unroll($message,$p,'phrase')) {
375	dpavlin	19	push @{$row->{$p}},$v;
376			$self->add_counter($p,$v);
377			}
378			}
379	dpavlin	42
380			$self->add_counter('folder', $1) if ($id =~ m/^(\S+)\s/);
381
382	dpavlin	16	$row->{'subject'} = $self->decode_qp($message->subject);
383	dpavlin	13	$row->{'body'} = $self->plain_text_body($message);
384	dpavlin	20	my $utime = str2time($message->date);
385	dpavlin	13
386	dpavlin	20	$row->{'date_utime'} = $utime;
387
388			$row->{'date'} = strftime("%Y-%m-%d %H:%M:%S", localtime($utime));
389			$self->add_counter_calendar($utime);
390
391	dpavlin	13	# XXX store in cache?
392			$self->{cache}->{$id} = $row;
393			print STDERR "$id stored in cache\n" if ($debug == 2);
394			} else {
395			print STDERR "fetch_result_by_id($id) in cache\n" if ($debug == 2);
396	dpavlin	42	foreach my $p (@counters) {
397	dpavlin	23	foreach my $v (@{$row->{$p}}) {
398			$self->add_counter($p,$v);
399			}
400			}
401
402	dpavlin	42	$self->add_counter('folder', $1) if ($id =~ m/^(\S+)\s/);
403
404	dpavlin	23	$self->add_counter_calendar($row->{date_utime});
405	dpavlin	13	}
406
407	dpavlin	2	return $row;
408
409			}