112 |
|
|
113 |
$self->{'encoding'} ||= 'ISO-8859-2'; |
$self->{'encoding'} ||= 'ISO-8859-2'; |
114 |
|
|
|
# running with low_mem flag? well, use DBM::Deep then. |
|
|
if ($self->{'low_mem'}) { |
|
|
$log->info("running with low_mem which impacts performance (<32 Mb memory usage)"); |
|
|
|
|
|
my $db_file = "data.db"; |
|
|
|
|
|
if (-e $db_file) { |
|
|
unlink $db_file or $log->logdie("can't remove '$db_file' from last run"); |
|
|
$log->debug("removed '$db_file' from last run"); |
|
|
} |
|
|
|
|
|
require DBM::Deep; |
|
|
|
|
|
my $db = new DBM::Deep $db_file; |
|
|
|
|
|
$log->logdie("DBM::Deep error: $!") unless ($db); |
|
|
|
|
|
if ($db->error()) { |
|
|
$log->logdie("can't open '$db_file' under low_mem: ",$db->error()); |
|
|
} else { |
|
|
$log->debug("using file '$db_file' for DBM::Deep"); |
|
|
} |
|
|
|
|
|
$self->{'db'} = $db; |
|
|
} |
|
|
|
|
115 |
$self ? return $self : return undef; |
$self ? return $self : return undef; |
116 |
} |
} |
117 |
|
|
259 |
|
|
260 |
$log->info("processing $self->{size}/$size records [$from_rec-$to_rec] convert $code_page -> $self->{encoding}", $self->{stats} ? ' [stats]' : ''); |
$log->info("processing $self->{size}/$size records [$from_rec-$to_rec] convert $code_page -> $self->{encoding}", $self->{stats} ? ' [stats]' : ''); |
261 |
|
|
262 |
|
# turn on low_mem for databases with more than 100000 records! |
263 |
|
if (! $self->{low_mem} && $size > 100000) { |
264 |
|
$log->warn("Using on-disk storage instead of memory for input data. This will affect performance."); |
265 |
|
$self->{low_mem}++; |
266 |
|
} |
267 |
|
|
268 |
|
# running with low_mem flag? well, use DBM::Deep then. |
269 |
|
if ($self->{'low_mem'}) { |
270 |
|
$log->info("running with low_mem which impacts performance (<32 Mb memory usage)"); |
271 |
|
|
272 |
|
my $db_file = "data.db"; |
273 |
|
|
274 |
|
if (-e $db_file) { |
275 |
|
unlink $db_file or $log->logdie("can't remove '$db_file' from last run"); |
276 |
|
$log->debug("removed '$db_file' from last run"); |
277 |
|
} |
278 |
|
|
279 |
|
require DBM::Deep; |
280 |
|
|
281 |
|
my $db = new DBM::Deep $db_file; |
282 |
|
|
283 |
|
$log->logdie("DBM::Deep error: $!") unless ($db); |
284 |
|
|
285 |
|
if ($db->error()) { |
286 |
|
$log->logdie("can't open '$db_file' under low_mem: ",$db->error()); |
287 |
|
} else { |
288 |
|
$log->debug("using file '$db_file' for DBM::Deep"); |
289 |
|
} |
290 |
|
|
291 |
|
$self->{'db'} = $db; |
292 |
|
} |
293 |
|
|
294 |
# read database |
# read database |
295 |
for (my $pos = $from_rec; $pos <= $to_rec; $pos++) { |
for (my $pos = $from_rec; $pos <= $to_rec; $pos++) { |
296 |
|
|