/[Grep]/lib/Grep/Import/ScrapBook.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /lib/Grep/Import/ScrapBook.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 159 - (show annotations)
Sun Jun 10 21:20:28 2007 UTC (16 years, 8 months ago) by dpavlin
File size: 3547 byte(s)
extract just html inside body to kill creazy backgrounds, and use correct
base when resolving ScrapBook pages
1 #!/usr/bin/perl
2
3 use warnings;
4 use strict;
5
6 package Grep::Import::ScrapBook;
7
8 =head1 NAME
9
10 Grep::Import::ScrapBook - importer for local ScrapBook pages
11
12 =head1 CONFIGURATION
13
14 You can symlink your ScrapBook directory
15
16 ~/Grep/share/web/static$ ln -sf /home/dpavlin/private/ScrapBook scrapbook
17
18 or modify L<ScrapBookDir> path (relative to Grep installation static root).
19
20 =cut
21
22 use XML::Simple;
23 use File::Slurp;
24 use HTML::ResolveLink;
25 use HTML::TreeBuilder;
26 use Data::Dump qw/dump/;
27
28 sub import {
29 my $self = shift;
30
31 my $dir =
32 Jifty::Util->app_root . '/' .
33 Jifty->config->framework('Web')->{'StaticRoot'} . '/' .
34 Jifty->config->app('ScrapBookDir');
35
36 my $path = $dir . '/scrapbook.rdf';
37 $path =~ s!//+!/!g;
38
39 if ( ! -e $dir || ! -e $path ) {
40 Jifty->log->warn("Skipping ScrapBook import $path: $!");
41 return 1;
42 }
43
44 my $rdf = XMLin(
45 $path,
46 # KeyAttr => [ qw/RDF:about/ ],
47 ) || die "can't open $path: $!";
48
49 # warn "## original rdf -> ", dump( $rdf );
50
51 my $owner = Grep::Model::User->new();
52 $owner->load_by_cols( email => Jifty->config->app('ScrapBookOwner') );
53 die "can't find ScrapBookOwner ", Jifty->config->app('ScrapBookOwner') unless ( $owner->id );
54
55 Jifty->log->info( "Using user ", $owner->id, " from ", $owner->email, " for import" );
56
57 my $feed = Grep::Model::Feed->new( current_user => $owner );
58 $feed->load_or_create(
59 uri => 'file://' . $path,
60 title => 'ScrapBook',
61 #source => 'Grep::Source',
62 owner => $owner,
63 );
64
65 my $search = Grep::Search->new;
66
67 my $stats;
68
69 foreach my $item ( @{ $rdf->{'RDF:Description'} } ) {
70
71 $stats->{total}++;
72
73 #warn "## item = ",dump( $item );
74
75 my $hash;
76 foreach my $k ( keys %$item ) {
77 next if $k =~ m/^RDF:/;
78 next if ( $item->{$k} eq '' );
79 my $n = $k;
80 $n =~ s/^\w+://; # strip namespace
81 $hash->{$n} = $item->{$k};
82 }
83
84 #warn "## hash = ", dump( $hash );
85
86
87 # fetch full-text content and import it
88
89 my $rel_path = '/data/' . $hash->{id} . '/index.html';
90
91 my $content_path = $dir . $rel_path;
92 if ( ! -r $content_path ) {
93 Jifty->log->warn("can't import $content_path: $!");
94 $stats->{failure}++;
95 next;
96 }
97 my $content = read_file( $content_path ) or
98 die "can't read $content_path: $!";
99
100 my $tree = HTML::TreeBuilder->new or die "can't create html tree";
101 $tree->parse( $content ) or die "can't parse fetched content";
102
103 my $body = $tree->look_down( '_tag', 'body' );
104
105 my $resolver = HTML::ResolveLink->new( base => '/static/' . Jifty->config->app('ScrapBookDir') . $rel_path );
106 $content = $resolver->resolve( $body->as_HTML );
107
108 # create date from id
109
110 my $dt;
111 if ( $hash->{id} =~ m/^(\d{4})(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)$/ ) {
112 $dt = DateTime->new(
113 year => $1,
114 month => $2,
115 day => $3,
116 hour => $4,
117 minute => $5,
118 second => $6,
119 #time_zone => 'UTC',
120 );
121 } else {
122 warn "can't parse date from ", $hash->{id};
123 }
124
125 my $i = Grep::Model::Item->new( current_user => $owner );
126 my ($ok,$msg) = $i->load_or_create(
127 in_feed => $feed,
128 title => $hash->{title},
129 link => $hash->{source},
130 content => $content,
131 issued => $hash->{id},
132 );
133
134 if ( ! $ok ) {
135 Jifty->log->error( $msg );
136 $stats->{failure}++;
137 next;
138 }
139
140 if ( $msg && $msg =~ m/^Found/ ) {
141 $stats->{old}++;
142 } else {
143 $stats->{new}++;
144 Jifty->log->info("created ", $i->id ," ", $i->link, " ", length( $content ), " bytes");
145 $search->add( $i, $owner->id );
146 }
147
148 }
149
150 $search->finish;
151
152 return $stats;
153 }
154
155 =head1 SEE ALSO
156
157 L<http://amb.vis.ne.jp/mozilla/scrapbook/> - ScrapBook FireFox extension
158
159 =cut
160
161 1;

  ViewVC Help
Powered by ViewVC 1.1.26