Grep/Import/ScrapBook.pm

#!/usr/bin/perl

use warnings;
use strict;

package Grep::Import::ScrapBook;

=head1 NAME

Grep::Import::ScrapBook - importer for local ScrapBook pages

=head1 CONFIGURATION

You can symlink your ScrapBook directory

  ~/Grep/share/web/static$ ln -sf /home/dpavlin/private/ScrapBook scrapbook

or modify L<ScrapBookDir> path (relative to Grep installation static root).

=cut

use XML::Simple;
use File::Slurp;
use HTML::ResolveLink;
use HTML::TreeBuilder;
use Data::Dump qw/dump/;

sub import {
        my $self = shift;

        my $dir =
                Jifty::Util->app_root . '/' .
                Jifty->config->framework('Web')->{'StaticRoot'} . '/' .
                Jifty->config->app('ScrapBookDir');

        my $path = $dir . '/scrapbook.rdf';
        $path =~ s!//+!/!g;

        if ( ! -e $dir  || ! -e $path ) {
                Jifty->log->warn("Skipping ScrapBook import $path: $!");
                return 1;
        }

        my $rdf = XMLin(
                $path,
#               KeyAttr => [ qw/RDF:about/ ],
        ) || die "can't open $path: $!";

#       warn "## original rdf -> ", dump( $rdf );

        my $owner = Grep::Model::User->new();
        $owner->load_by_cols( email => Jifty->config->app('ScrapBookOwner') );
        die "can't find ScrapBookOwner ", Jifty->config->app('ScrapBookOwner') unless ( $owner->id );

        Jifty->log->info( "Using user ", $owner->id, " from ", $owner->email, " for import" );

        my $feed = Grep::Model::Feed->new( current_user => $owner );
        $feed->load_or_create(
                uri => 'file://' . $path,
                title => 'ScrapBook',
                #source => 'Grep::Source',
                owner => $owner,
        );

        my $search = Grep::Search->new;

        my $stats;

        foreach my $item ( @{ $rdf->{'RDF:Description'} } ) {

                $stats->{total}++;

                #warn "## item = ",dump( $item );

                my $hash;
                foreach my $k ( keys %$item ) {
                        next if $k =~ m/^RDF:/;
                        next if ( $item->{$k} eq '' ); 
                        my $n = $k;
                        $n =~ s/^\w+://;        # strip namespace
                        $hash->{$n} = $item->{$k};
                }
        
                #warn "## hash = ", dump( $hash );


                # fetch full-text content and import it

                my $rel_path = '/data/' . $hash->{id} . '/index.html';

                my $content_path = $dir . $rel_path;
                if ( ! -r $content_path ) {
                        Jifty->log->warn("can't import $content_path: $!");
                        $stats->{failure}++;
                        next;
                }
                my $content = read_file( $content_path ) or
                        die "can't read $content_path: $!";

                my $tree = HTML::TreeBuilder->new or die "can't create html tree";
                $tree->parse( $content ) or die "can't parse fetched content";

                my $body = $tree->look_down( '_tag', 'body' );

                my $resolver = HTML::ResolveLink->new( base => '/static/' . Jifty->config->app('ScrapBookDir') . $rel_path );
                $content = $resolver->resolve( $body->as_HTML );

                # create date from id

                my $dt;
                if ( $hash->{id} =~ m/^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/ ) {
                        $dt = DateTime->new(
                                year    => $1,
                                month   => $2,
                                day             => $3,
                                hour    => $4,
                                minute  => $5,
                                second  => $6,
                                #time_zone => 'UTC',
                        );
                } else {
                        warn "can't parse date from ", $hash->{id};
                }

                my $i = Grep::Model::Item->new( current_user => $owner );
                my ($ok,$msg) = $i->load_or_create(
                        in_feed => $feed,
                        title => $hash->{title},
                        link => $hash->{source},
                        content => $content,
                        issued => $hash->{id},
                );

                if ( ! $ok ) {
                        Jifty->log->error( $msg );
                        $stats->{failure}++;
                        next;
                }

                if ( $msg && $msg =~ m/^Found/ ) {
                        $stats->{old}++;
                } else {
                        $stats->{new}++;
                        Jifty->log->info("created ", $i->id ," ", $i->link, " ", length( $content ), " bytes");
                        $search->add( $i, $owner->id );
                }

        }

        $search->finish;

        return $stats;
}

=head1 SEE ALSO

L<http://amb.vis.ne.jp/mozilla/scrapbook/> - ScrapBook FireFox extension

=cut

1;
1	dpavlin	154	#!/usr/bin/perl
2
3			use warnings;
4			use strict;
5
6			package Grep::Import::ScrapBook;
7
8			=head1 NAME
9
10			Grep::Import::ScrapBook - importer for local ScrapBook pages
11
12			=head1 CONFIGURATION
13
14			You can symlink your ScrapBook directory
15
16			~/Grep/share/web/static$ ln -sf /home/dpavlin/private/ScrapBook scrapbook
17
18			or modify L<ScrapBookDir> path (relative to Grep installation static root).
19
20			=cut
21
22			use XML::Simple;
23			use File::Slurp;
24	dpavlin	158	use HTML::ResolveLink;
25	dpavlin	159	use HTML::TreeBuilder;
26	dpavlin	154	use Data::Dump qw/dump/;
27
28			sub import {
29			my $self = shift;
30
31			my $dir =
32			Jifty::Util->app_root . '/' .
33			Jifty->config->framework('Web')->{'StaticRoot'} . '/' .
34			Jifty->config->app('ScrapBookDir');
35
36			my $path = $dir . '/scrapbook.rdf';
37			$path =~ s!//+!/!g;
38
39			if ( ! -e $dir \|\| ! -e $path ) {
40			Jifty->log->warn("Skipping ScrapBook import $path: $!");
41			return 1;
42			}
43
44			my $rdf = XMLin(
45			$path,
46			# KeyAttr => [ qw/RDF:about/ ],
47			) \|\| die "can't open $path: $!";
48
49			# warn "## original rdf -> ", dump( $rdf );
50
51	dpavlin	157	my $owner = Grep::Model::User->new();
52			$owner->load_by_cols( email => Jifty->config->app('ScrapBookOwner') );
53			die "can't find ScrapBookOwner ", Jifty->config->app('ScrapBookOwner') unless ( $owner->id );
54
55			Jifty->log->info( "Using user ", $owner->id, " from ", $owner->email, " for import" );
56
57			my $feed = Grep::Model::Feed->new( current_user => $owner );
58	dpavlin	154	$feed->load_or_create(
59			uri => 'file://' . $path,
60			title => 'ScrapBook',
61			#source => 'Grep::Source',
62	dpavlin	157	owner => $owner,
63	dpavlin	154	);
64
65	dpavlin	158	my $search = Grep::Search->new;
66
67	dpavlin	155	my $stats;
68
69	dpavlin	154	foreach my $item ( @{ $rdf->{'RDF:Description'} } ) {
70
71	dpavlin	155	$stats->{total}++;
72	dpavlin	154
73	dpavlin	155	#warn "## item = ",dump( $item );
74
75	dpavlin	154	my $hash;
76			foreach my $k ( keys %$item ) {
77			next if $k =~ m/^RDF:/;
78			next if ( $item->{$k} eq '' );
79			my $n = $k;
80			$n =~ s/^\w+://; # strip namespace
81			$hash->{$n} = $item->{$k};
82			}
83
84	dpavlin	155	#warn "## hash = ", dump( $hash );
85	dpavlin	154
86
87			# fetch full-text content and import it
88
89	dpavlin	159	my $rel_path = '/data/' . $hash->{id} . '/index.html';
90
91			my $content_path = $dir . $rel_path;
92	dpavlin	154	if ( ! -r $content_path ) {
93			Jifty->log->warn("can't import $content_path: $!");
94	dpavlin	155	$stats->{failure}++;
95	dpavlin	154	next;
96			}
97			my $content = read_file( $content_path ) or
98			die "can't read $content_path: $!";
99
100	dpavlin	159	my $tree = HTML::TreeBuilder->new or die "can't create html tree";
101			$tree->parse( $content ) or die "can't parse fetched content";
102	dpavlin	154
103	dpavlin	159	my $body = $tree->look_down( '_tag', 'body' );
104
105			my $resolver = HTML::ResolveLink->new( base => '/static/' . Jifty->config->app('ScrapBookDir') . $rel_path );
106			$content = $resolver->resolve( $body->as_HTML );
107
108	dpavlin	154	# create date from id
109
110			my $dt;
111			if ( $hash->{id} =~ m/^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/ ) {
112			$dt = DateTime->new(
113			year => $1,
114			month => $2,
115			day => $3,
116			hour => $4,
117			minute => $5,
118			second => $6,
119			#time_zone => 'UTC',
120			);
121			} else {
122			warn "can't parse date from ", $hash->{id};
123			}
124
125	dpavlin	157	my $i = Grep::Model::Item->new( current_user => $owner );
126	dpavlin	155	my ($ok,$msg) = $i->load_or_create(
127	dpavlin	154	in_feed => $feed,
128			title => $hash->{title},
129			link => $hash->{source},
130			content => $content,
131			issued => $hash->{id},
132			);
133
134	dpavlin	155	if ( ! $ok ) {
135			Jifty->log->error( $msg );
136			$stats->{failure}++;
137			next;
138			}
139	dpavlin	154
140	dpavlin	155	if ( $msg && $msg =~ m/^Found/ ) {
141			$stats->{old}++;
142			} else {
143			$stats->{new}++;
144	dpavlin	157	Jifty->log->info("created ", $i->id ," ", $i->link, " ", length( $content ), " bytes");
145			$search->add( $i, $owner->id );
146	dpavlin	155	}
147	dpavlin	154
148			}
149
150	dpavlin	158	$search->finish;
151
152	dpavlin	155	return $stats;
153	dpavlin	154	}
154
155	dpavlin	155	=head1 SEE ALSO
156	dpavlin	154
157	dpavlin	155	L<http://amb.vis.ne.jp/mozilla/scrapbook/> - ScrapBook FireFox extension
158
159			=cut
160
161	dpavlin	154	1;