220 |
%{ $input }, |
%{ $input }, |
221 |
); |
); |
222 |
|
|
223 |
my $rules; |
my @norm_array = ref($input->{normalize}) eq 'ARRAY' ? |
224 |
my $normalize_path = $input->{normalize}->{path}; |
@{ $input->{normalize} } : ( $input->{normalize} ); |
225 |
|
|
226 |
$log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i ); |
foreach my $normalize (@norm_array) { |
227 |
|
|
228 |
my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!"; |
my $rules; |
229 |
|
my $normalize_path = $normalize->{path} || $log->logdie("can't find normalize path in config"); |
230 |
|
|
231 |
foreach my $pos ( 0 ... $input_db->size ) { |
$log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i ); |
232 |
|
|
233 |
my $row = $input_db->fetch || next; |
my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!"; |
234 |
|
|
235 |
my $mfn = $row->{'000'}->[0]; |
$log->info("Using $normalize_path for normalization..."); |
236 |
|
|
237 |
if (! $mfn || $mfn !~ m#^\d+$#) { |
# reset position in database |
238 |
$log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); |
$input_db->seek(1); |
|
$mfn = $pos; |
|
|
push @{ $row->{'000'} }, $pos; |
|
|
} |
|
239 |
|
|
240 |
|
foreach my $pos ( 0 ... $input_db->size ) { |
241 |
|
|
242 |
|
my $row = $input_db->fetch || next; |
243 |
|
|
244 |
|
my $mfn = $row->{'000'}->[0]; |
245 |
|
|
246 |
|
if (! $mfn || $mfn !~ m#^\d+$#) { |
247 |
|
$log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); |
248 |
|
$mfn = $pos; |
249 |
|
push @{ $row->{'000'} }, $pos; |
250 |
|
} |
251 |
|
|
|
if ($validate) { |
|
|
my @errors = $validate->validate_errors( $row ); |
|
|
$log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors); |
|
|
} |
|
252 |
|
|
253 |
|
if ($validate) { |
254 |
my $ds = WebPAC::Normalize::data_structure( |
my @errors = $validate->validate_errors( $row ); |
255 |
row => $row, |
$log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors); |
256 |
rules => $rules, |
} |
257 |
lookup => $lookup ? $lookup->lookup_hash : undef, |
|
258 |
); |
|
259 |
|
my $ds = WebPAC::Normalize::data_structure( |
260 |
$db->save_ds( |
row => $row, |
261 |
id => $mfn, |
rules => $rules, |
262 |
ds => $ds, |
lookup => $lookup ? $lookup->lookup_hash : undef, |
263 |
prefix => $input->{name}, |
); |
264 |
) if ($ds && !$stats); |
|
265 |
|
$db->save_ds( |
266 |
$indexer->add( |
id => $mfn, |
267 |
id => $input->{name} . "/" . $mfn, |
ds => $ds, |
268 |
ds => $ds, |
prefix => $input->{name}, |
269 |
type => $config->{$use_indexer}->{type}, |
) if ($ds && !$stats); |
270 |
) if ($indexer); |
|
271 |
|
$indexer->add( |
272 |
|
id => $input->{name} . "/" . $mfn, |
273 |
|
ds => $ds, |
274 |
|
type => $config->{$use_indexer}->{type}, |
275 |
|
) if ($indexer); |
276 |
|
|
277 |
|
$total_rows++; |
278 |
|
} |
279 |
|
|
280 |
$total_rows++; |
$log->info("statistics of fields usage:\n", $input_db->stats) if ($stats); |
|
} |
|
281 |
|
|
282 |
$log->info("statistics of fields usage:\n", $input_db->stats) if ($stats); |
}; |
283 |
|
|
284 |
}; |
} |
285 |
|
|
286 |
eval { $indexer->finish } if ($indexer && $indexer->can('finish')); |
eval { $indexer->finish } if ($indexer && $indexer->can('finish')); |
287 |
|
|