24 |
my $self = shift; |
my $self = shift; |
25 |
my $value = shift; |
my $value = shift; |
26 |
warn "uri: $value"; |
warn "uri: $value"; |
27 |
if ($value =~ s/=grep/=%s/) { |
if ($value =~ s/\bgrep\b/%s/) { |
28 |
$self->canonicalization_note( uri => 'Replaced grep with %s' ); |
$self->canonicalization_note( uri => 'Replaced grep with %s' ); |
29 |
} |
} |
30 |
return $value; |
return $value; |
31 |
} |
} |
32 |
|
|
33 |
|
=head2 canonicalize_cookie |
34 |
|
|
35 |
|
Remove C<Cookie:> header from beginning and replace EOL with space. |
36 |
|
|
37 |
|
=cut |
38 |
|
|
39 |
|
# disabled for now |
40 |
|
sub xx_canonicalize_cookie { |
41 |
|
my $self = shift; |
42 |
|
my $value = shift; |
43 |
|
|
44 |
|
warn "cookie: $value"; |
45 |
|
|
46 |
|
$self->canonicalization_note( uri => 'Removed Cookie: header' ) |
47 |
|
if ($value =~ s/^Cookie:\s+//); |
48 |
|
|
49 |
|
$self->canonicalization_note( uri => 'Converted EOL to space' ) |
50 |
|
if ($value =~ s/[\n\r]/ /gs); |
51 |
|
|
52 |
|
return $value; |
53 |
|
} |
54 |
=head2 take_action |
=head2 take_action |
55 |
|
|
56 |
=cut |
=cut |
79 |
|
|
80 |
Jifty->log->debug("trying to find feed on $uri"); |
Jifty->log->debug("trying to find feed on $uri"); |
81 |
|
|
82 |
my @feeds = Feed::Find->find( sprintf( $uri, $search_moniker ), $ua ); |
my $r = $ua->get( sprintf( $uri, $search_moniker ) ); |
83 |
|
|
84 |
|
return $self->result->error( $r->status_line . " from $uri" ) unless ( $r->is_success ); |
85 |
|
|
86 |
|
my $ct = $r->header('Content-type') or warn "can't get Content-type"; |
87 |
|
my $content = $r->content; |
88 |
|
|
89 |
|
Jifty->log->debug("got ", length( $content ), " bytes $ct"); |
90 |
|
|
91 |
|
if ( $ct =~ /xml/ ) { |
92 |
|
$self->result->message( "Assuming $uri is feed from $ct" ); |
93 |
|
return $self->SUPER::take_action( @ARGS ); |
94 |
|
} |
95 |
|
|
96 |
|
my @feeds = Feed::Find->find_in_html( \$content ); |
97 |
|
|
98 |
if (@feeds) { |
if (@feeds) { |
99 |
|
|
100 |
Jifty->log->info("found possible feeds: ", dump( @feeds )); |
Jifty->log->info("found possible feeds: ", dump( @feeds )); |
101 |
|
|
102 |
@feeds = map { |
my @search_feeds = map { |
103 |
my $t = $_; |
my $t = $_; |
104 |
$t =~ s/=$search_moniker/=%s/; |
$t =~ s/\b$search_moniker\b/%s/; |
105 |
$t |
$t |
106 |
} grep(/=$search_moniker/,@feeds); |
} grep(/\b$search_moniker\b/,@feeds); |
107 |
|
|
|
$self->result->message('Found ' . @feeds . ' feeds, using first one. ' . join(" ", @feeds) ); |
|
108 |
|
|
109 |
$uri = shift @feeds; |
if ( my $feed_uri = shift @search_feeds ) { |
110 |
|
|
111 |
$self->argument_value('uri', $uri); |
$self->result->message('Found ' . @feeds . " feeds, using first: $feed_uri" ); |
112 |
|
$self->argument_value('uri', $feed_uri); |
113 |
|
|
114 |
Jifty->log->debug("calling parent take_action with new uri $uri"); |
Jifty->log->debug("calling parent take_action with new uri $feed_uri"); |
115 |
|
|
116 |
$self->SUPER::take_action( @ARGS ); |
return $self->SUPER::take_action( @ARGS ); |
117 |
|
|
118 |
return 1; |
} else { |
119 |
|
Jifty->log->debug("found feeds ", dump( @feeds ), " but none of them has search moniker!"); |
120 |
|
} |
121 |
|
} |
122 |
|
|
123 |
} else { |
Jifty->log->debug("no feeds found, trying content_class detection"); |
124 |
|
|
125 |
my $r = $ua->get( sprintf( $uri, $search_moniker ) ); |
my $source = Grep::Source->new(); |
126 |
warn "no feeds in ", $r->content; |
|
127 |
|
if ( my $class = $source->content_class( $content ) ) { |
128 |
|
|
129 |
|
Jifty->log->debug("$class registred for feed $uri"); |
130 |
|
|
131 |
|
$self->argument_value('source', $source); |
132 |
|
$self->result->message("Found $class scraper for $uri" ); |
133 |
|
return $self->SUPER::take_action( @ARGS ); |
134 |
|
|
135 |
|
} else { |
136 |
|
|
137 |
$self->result->error('No feeds found on supplied URI'); |
$self->result->error('No feeds found on supplied URI'); |
138 |
return 0; |
return 0; |
139 |
|
|
140 |
} |
} |
141 |
|
|
142 |
} |
} |