Grep/Import/ScrapBook.pm

#!/usr/bin/perl

use warnings;
use strict;

package Grep::Import::ScrapBook;

=head1 NAME

Grep::Import::ScrapBook - importer for local ScrapBook pages

=head1 CONFIGURATION

You can symlink your ScrapBook directory

  ~/Grep/share/web/static$ ln -sf /home/dpavlin/private/ScrapBook scrapbook

or modify L<ScrapBookDir> path (relative to Grep installation static root).

=cut

use XML::Simple;
use File::Slurp;
use HTML::ResolveLink;
use HTML::TreeBuilder;
use Data::Dump qw/dump/;

sub import {
        my $self = shift;

        my $config = Jifty->config->app('Import')->{'ScrapBook'};

        if (! $config ) {
                Jifty->log->warn("skipping ScrapBook importer, no application->Import->ScrapBook config");
                return;
        }

        # required parametars in config.yml
        foreach my $param ( qw/Dir OwnerEmail/ ) {
                Jifty->log->die("can't find $param in Scrapbook config") unless defined ( $config->{$param} );
        };

        my $dir =
                Jifty::Util->app_root . '/' .
                Jifty->config->framework('Web')->{'StaticRoot'} . '/' . $config->{'Dir'};

        my $path = $dir . '/scrapbook.rdf';
        $path =~ s!//+!/!g;

        if ( ! -e $dir  || ! -e $path ) {
                Jifty->log->warn("Skipping ScrapBook import $path: $!");
                return 1;
        }

        my $rdf = XMLin(
                $path,
#               KeyAttr => [ qw/RDF:about/ ],
        ) || die "can't open $path: $!";

#       warn "## original rdf -> ", dump( $rdf );

        my $owner = Grep::Model::User->new();
        $owner->load_by_cols( email => $config->{OwnerEmail} );
        die "can't find ScrapBookOwner ", $config->{OwnerEmail} unless ( $owner->id );

        Jifty->log->info( "Using user ", $owner->id, " from ", $owner->email, " for import" );

        my $feed = Grep::Model::Feed->new( current_user => $owner );
        $feed->load_or_create(
                uri => 'file://' . $path,
                title => 'ScrapBook',
                #source => 'Grep::Source',
                owner => $owner,
        );

        my $search = Grep::Search->new;

        my $stats;

        foreach my $item ( @{ $rdf->{'RDF:Description'} } ) {

                $stats->{total}++;

                #warn "## item = ",dump( $item );

                my $hash;
                foreach my $k ( keys %$item ) {
                        next if $k =~ m/^RDF:/;
                        next if ( $item->{$k} eq '' ); 
                        my $n = $k;
                        $n =~ s/^\w+://;        # strip namespace
                        $hash->{$n} = $item->{$k};
                }
        
                #warn "## hash = ", dump( $hash );


                # fetch full-text content and import it

                my $rel_path = '/data/' . $hash->{id} . '/index.html';

                my $content_path = $dir . $rel_path;
                if ( ! -r $content_path ) {
                        Jifty->log->warn("can't import $content_path: $!");
                        $stats->{failure}++;
                        next;
                }
                my $content = read_file( $content_path ) or
                        die "can't read $content_path: $!";

                my $tree = HTML::TreeBuilder->new or die "can't create html tree";
                $tree->parse( $content ) or die "can't parse fetched content";

                my $body = $tree->look_down( '_tag', 'body' );

                my $resolver = HTML::ResolveLink->new( base => '/static/' . $config->{Dir} . $rel_path );
                $content = $resolver->resolve( $body->as_HTML );

                # create date from id

                my $dt;
                if ( $hash->{id} =~ m/^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/ ) {
                        $dt = DateTime->new(
                                year    => $1,
                                month   => $2,
                                day             => $3,
                                hour    => $4,
                                minute  => $5,
                                second  => $6,
                                #time_zone => 'UTC',
                        );
                } else {
                        warn "can't parse date from ", $hash->{id};
                }

                my $i = Grep::Model::Item->new( current_user => $owner );
                my ($ok,$msg) = $i->load_or_create(
                        in_feed => $feed,
                        title => $hash->{title},
                        link => $hash->{source},
                        content => $content,
                        issued => $hash->{id},
                );

                if ( ! $ok ) {
                        Jifty->log->error( $msg );
                        $stats->{failure}++;
                        next;
                }

                if ( $msg && $msg =~ m/^Found/ ) {
                        $stats->{old}++;
                } else {
                        $stats->{new}++;
                        Jifty->log->info("created ", $i->id ," ", $i->link, " ", length( $content ), " bytes");
                        $search->add( $i, $owner->id );
                }

        }

        $search->finish;

        return $stats;
}

=head1 SEE ALSO

L<http://amb.vis.ne.jp/mozilla/scrapbook/> - ScrapBook FireFox extension

=cut

1;
1	dpavlin	154	#!/usr/bin/perl
2
3			use warnings;
4			use strict;
5
6			package Grep::Import::ScrapBook;
7
8			=head1 NAME
9
10			Grep::Import::ScrapBook - importer for local ScrapBook pages
11
12			=head1 CONFIGURATION
13
14			You can symlink your ScrapBook directory
15
16			~/Grep/share/web/static$ ln -sf /home/dpavlin/private/ScrapBook scrapbook
17
18			or modify L<ScrapBookDir> path (relative to Grep installation static root).
19
20			=cut
21
22			use XML::Simple;
23			use File::Slurp;
24	dpavlin	158	use HTML::ResolveLink;
25	dpavlin	159	use HTML::TreeBuilder;
26	dpavlin	154	use Data::Dump qw/dump/;
27
28			sub import {
29			my $self = shift;
30
31	dpavlin	162	my $config = Jifty->config->app('Import')->{'ScrapBook'};
32
33			if (! $config ) {
34			Jifty->log->warn("skipping ScrapBook importer, no application->Import->ScrapBook config");
35			return;
36			}
37
38			# required parametars in config.yml
39			foreach my $param ( qw/Dir OwnerEmail/ ) {
40			Jifty->log->die("can't find $param in Scrapbook config") unless defined ( $config->{$param} );
41			};
42
43	dpavlin	154	my $dir =
44			Jifty::Util->app_root . '/' .
45	dpavlin	162	Jifty->config->framework('Web')->{'StaticRoot'} . '/' . $config->{'Dir'};
46	dpavlin	154
47			my $path = $dir . '/scrapbook.rdf';
48			$path =~ s!//+!/!g;
49
50			if ( ! -e $dir \|\| ! -e $path ) {
51			Jifty->log->warn("Skipping ScrapBook import $path: $!");
52			return 1;
53			}
54
55			my $rdf = XMLin(
56			$path,
57			# KeyAttr => [ qw/RDF:about/ ],
58			) \|\| die "can't open $path: $!";
59
60			# warn "## original rdf -> ", dump( $rdf );
61
62	dpavlin	157	my $owner = Grep::Model::User->new();
63	dpavlin	162	$owner->load_by_cols( email => $config->{OwnerEmail} );
64			die "can't find ScrapBookOwner ", $config->{OwnerEmail} unless ( $owner->id );
65	dpavlin	157
66			Jifty->log->info( "Using user ", $owner->id, " from ", $owner->email, " for import" );
67
68			my $feed = Grep::Model::Feed->new( current_user => $owner );
69	dpavlin	154	$feed->load_or_create(
70			uri => 'file://' . $path,
71			title => 'ScrapBook',
72			#source => 'Grep::Source',
73	dpavlin	157	owner => $owner,
74	dpavlin	154	);
75
76	dpavlin	158	my $search = Grep::Search->new;
77
78	dpavlin	155	my $stats;
79
80	dpavlin	154	foreach my $item ( @{ $rdf->{'RDF:Description'} } ) {
81
82	dpavlin	155	$stats->{total}++;
83	dpavlin	154
84	dpavlin	155	#warn "## item = ",dump( $item );
85
86	dpavlin	154	my $hash;
87			foreach my $k ( keys %$item ) {
88			next if $k =~ m/^RDF:/;
89			next if ( $item->{$k} eq '' );
90			my $n = $k;
91			$n =~ s/^\w+://; # strip namespace
92			$hash->{$n} = $item->{$k};
93			}
94
95	dpavlin	155	#warn "## hash = ", dump( $hash );
96	dpavlin	154
97
98			# fetch full-text content and import it
99
100	dpavlin	159	my $rel_path = '/data/' . $hash->{id} . '/index.html';
101
102			my $content_path = $dir . $rel_path;
103	dpavlin	154	if ( ! -r $content_path ) {
104			Jifty->log->warn("can't import $content_path: $!");
105	dpavlin	155	$stats->{failure}++;
106	dpavlin	154	next;
107			}
108			my $content = read_file( $content_path ) or
109			die "can't read $content_path: $!";
110
111	dpavlin	159	my $tree = HTML::TreeBuilder->new or die "can't create html tree";
112			$tree->parse( $content ) or die "can't parse fetched content";
113	dpavlin	154
114	dpavlin	159	my $body = $tree->look_down( '_tag', 'body' );
115
116	dpavlin	162	my $resolver = HTML::ResolveLink->new( base => '/static/' . $config->{Dir} . $rel_path );
117	dpavlin	159	$content = $resolver->resolve( $body->as_HTML );
118
119	dpavlin	154	# create date from id
120
121			my $dt;
122			if ( $hash->{id} =~ m/^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/ ) {
123			$dt = DateTime->new(
124			year => $1,
125			month => $2,
126			day => $3,
127			hour => $4,
128			minute => $5,
129			second => $6,
130			#time_zone => 'UTC',
131			);
132			} else {
133			warn "can't parse date from ", $hash->{id};
134			}
135
136	dpavlin	157	my $i = Grep::Model::Item->new( current_user => $owner );
137	dpavlin	155	my ($ok,$msg) = $i->load_or_create(
138	dpavlin	154	in_feed => $feed,
139			title => $hash->{title},
140			link => $hash->{source},
141			content => $content,
142			issued => $hash->{id},
143			);
144
145	dpavlin	155	if ( ! $ok ) {
146			Jifty->log->error( $msg );
147			$stats->{failure}++;
148			next;
149			}
150	dpavlin	154
151	dpavlin	155	if ( $msg && $msg =~ m/^Found/ ) {
152			$stats->{old}++;
153			} else {
154			$stats->{new}++;
155	dpavlin	157	Jifty->log->info("created ", $i->id ," ", $i->link, " ", length( $content ), " bytes");
156			$search->add( $i, $owner->id );
157	dpavlin	155	}
158	dpavlin	154
159			}
160
161	dpavlin	158	$search->finish;
162
163	dpavlin	155	return $stats;
164	dpavlin	154	}
165
166	dpavlin	155	=head1 SEE ALSO
167	dpavlin	154
168	dpavlin	155	L<http://amb.vis.ne.jp/mozilla/scrapbook/> - ScrapBook FireFox extension
169
170			=cut
171
172	dpavlin	154	1;