WebPAC/Output/Sorted.pm

package WebPAC::Output::Sorted;

use warnings;
use strict;

use base qw/WebPAC::Common WebPAC::Output Class::Accessor/;
__PACKAGE__->mk_accessors(qw(
        path
        database

        sortex
));

use Sort::External;
use File::Path;
use Data::Dump qw/dump/;

=head1 NAME

WebPAC::Output::Sorted - create sorted lists

=head1 VERSION

Version 0.01

=cut

our $VERSION = '0.01';

=head1 SYNOPSIS

Create sorted with from data with type C<sorted>.

=head1 FUNCTIONS

=head2 new

 my $output = new WebPAC::Output::Sorted({
        path => '/path/to/sorted_dir',
        database => 'demo',
 });

=head2 init

 $output->init;

=cut

sub init {
        my $self = shift;

        my $log = $self->_get_logger;

        foreach my $p (qw/path database/) {
                $log->logdie("need $p") unless ($self->$p);
        }

        if ( ! -e $self->path ) {
                mkpath $self->path || $log->logdie("can't create ", $self->path,": $!");
                $log->info("created ", $self->path);
        }

}


=head2 add

Adds one entry

  $est->add( 42, $ds );

=cut

sub add {
        my $self = shift;

        my ( $id, $ds ) = @_;

        my $log = $self->_get_logger;
        $log->logdie("need id") unless defined $id;
        $log->logdie("need ds") unless $ds;

        $log->debug("id: $id ds = ",dump($ds));

        my $hash = $self->ds_to_hash( $ds, 'sorted' ) || return;

        warn "add( $id, ",dump($ds)," ) => ", dump( $hash );

        foreach my $f ( keys %$hash ) {

                my $sortex = $self->{sortex}->{$f};

                if ( ! $sortex ) {

                        my $sortscheme = sub { $Sort::External::b <=> $Sort::External::a };
                        $sortex = Sort::External->new(
                                -mem_threshold   => 2**24,            # default: 2**20 (1Mb)
                                -cache_size      => 100_000,          # default: undef (disabled) 
#                               -sortsub         => $sortscheme,      # default sort: standard lexical
#                               -working_dir     => $tmp,
                        );

                        $log->logdie("can't create sorted list for $f: $!") unless $sortex;

                        $log->info("created sorted list for $f");

                        $self->{sortex}->{$f} = $sortex;

                };

                my @v;

                if ( ref( $hash->{$f} ) eq 'ARRAY' ) {
                        @v = @{ $hash->{$f} };
                } else {
                        @v =    $hash->{$f}  ;
                }

                # we want LF in output file :-)
                @v = map { "$_\n" } @v;

                $self->{sortex}->{$f}->feed( @v );

        }

        return 1;
}

=head2 finish

Close index

 $index->finish;

=cut

sub finish {
        my $self = shift;

        my $log = $self->_get_logger();

        $log->info("finish sorted lists");

        foreach my $list ( keys %{ $self->{sortex} } ) {

                my $path = $self->path . '/' . $list . '.txt';
                $log->info("saving $list to $path");

                use Fcntl;
                $self->{sortex}->{$list}->finish( 
                        -outfile => $path,
                        -flags => (O_CREAT | O_WRONLY),
                );
        
        }

        $log->info("over with sorted lists");
}


=head1 AUTHOR

Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>

=head1 COPYRIGHT & LICENSE

Copyright 2007 Dobrica Pavlinusic, All Rights Reserved.

This program is free software; you can redistribute it and/or modify it
under the same terms as Perl itself.

=cut

1;
1	package WebPAC::Output::Sorted;
2
3	use warnings;
4	use strict;
5
6	use base qw/WebPAC::Common WebPAC::Output Class::Accessor/;
7	__PACKAGE__->mk_accessors(qw(
8	path
9	database
10
11	sortex
12	));
13
14	use Sort::External;
15	use File::Path;
16	use Data::Dump qw/dump/;
17
18	=head1 NAME
19
20	WebPAC::Output::Sorted - create sorted lists
21
22	=head1 VERSION
23
24	Version 0.01
25
26	=cut
27
28	our $VERSION = '0.01';
29
30	=head1 SYNOPSIS
31
32	Create sorted with from data with type C<sorted>.
33
34	=head1 FUNCTIONS
35
36	=head2 new
37
38	my $output = new WebPAC::Output::Sorted({
39	path => '/path/to/sorted_dir',
40	database => 'demo',
41	});
42
43	=head2 init
44
45	$output->init;
46
47	=cut
48
49	sub init {
50	my $self = shift;
51
52	my $log = $self->_get_logger;
53
54	foreach my $p (qw/path database/) {
55	$log->logdie("need $p") unless ($self->$p);
56	}
57
58	if ( ! -e $self->path ) {
59	mkpath $self->path \|\| $log->logdie("can't create ", $self->path,": $!");
60	$log->info("created ", $self->path);
61	}
62
63	}
64
65
66	=head2 add
67
68	Adds one entry
69
70	$est->add( 42, $ds );
71
72	=cut
73
74	sub add {
75	my $self = shift;
76
77	my ( $id, $ds ) = @_;
78
79	my $log = $self->_get_logger;
80	$log->logdie("need id") unless defined $id;
81	$log->logdie("need ds") unless $ds;
82
83	$log->debug("id: $id ds = ",dump($ds));
84
85	my $hash = $self->ds_to_hash( $ds, 'sorted' ) \|\| return;
86
87	warn "add( $id, ",dump($ds)," ) => ", dump( $hash );
88
89	foreach my $f ( keys %$hash ) {
90
91	my $sortex = $self->{sortex}->{$f};
92
93	if ( ! $sortex ) {
94
95	my $sortscheme = sub { $Sort::External::b <=> $Sort::External::a };
96	$sortex = Sort::External->new(
97	-mem_threshold => 224, # default: 220 (1Mb)
98	-cache_size => 100_000, # default: undef (disabled)
99	# -sortsub => $sortscheme, # default sort: standard lexical
100	# -working_dir => $tmp,
101	);
102
103	$log->logdie("can't create sorted list for $f: $!") unless $sortex;
104
105	$log->info("created sorted list for $f");
106
107	$self->{sortex}->{$f} = $sortex;
108
109	};
110
111	my @v;
112
113	if ( ref( $hash->{$f} ) eq 'ARRAY' ) {
114	@v = @{ $hash->{$f} };
115	} else {
116	@v = $hash->{$f} ;
117	}
118
119	# we want LF in output file :-)
120	@v = map { "$_\n" } @v;
121
122	$self->{sortex}->{$f}->feed( @v );
123
124	}
125
126	return 1;
127	}
128
129	=head2 finish
130
131	Close index
132
133	$index->finish;
134
135	=cut
136
137	sub finish {
138	my $self = shift;
139
140	my $log = $self->_get_logger();
141
142	$log->info("finish sorted lists");
143
144	foreach my $list ( keys %{ $self->{sortex} } ) {
145
146	my $path = $self->path . '/' . $list . '.txt';
147	$log->info("saving $list to $path");
148
149	use Fcntl;
150	$self->{sortex}->{$list}->finish(
151	-outfile => $path,
152	-flags => (O_CREAT \| O_WRONLY),
153	);
154
155	}
156
157	$log->info("over with sorted lists");
158	}
159
160
161	=head1 AUTHOR
162
163	Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
164
165	=head1 COPYRIGHT & LICENSE
166
167	Copyright 2007 Dobrica Pavlinusic, All Rights Reserved.
168
169	This program is free software; you can redistribute it and/or modify it
170	under the same terms as Perl itself.
171
172	=cut
173
174	1;