/[webpac2]/trunk/run.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/run.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 307 by dpavlin, Tue Dec 20 00:03:04 2005 UTC revision 423 by dpavlin, Wed Mar 22 00:18:56 2006 UTC
# Line 13  use WebPAC::Input 0.03; Line 13  use WebPAC::Input 0.03;
13  use WebPAC::Store 0.03;  use WebPAC::Store 0.03;
14  use WebPAC::Normalize::XML;  use WebPAC::Normalize::XML;
15  use WebPAC::Output::TT;  use WebPAC::Output::TT;
16  use WebPAC::Output::Estraier 0.08;  use WebPAC::Output::Estraier '0.10';
17  use YAML qw/LoadFile/;  use YAML qw/LoadFile/;
18  use Getopt::Long;  use Getopt::Long;
19  use File::Path;  use File::Path;
20    use Time::HiRes qw/time/;
21    
22  =head1 NAME  =head1 NAME
23    
# Line 40  limit loading to 100 records Line 41  limit loading to 100 records
41    
42  remove database and Hyper Estraier index before indexing  remove database and Hyper Estraier index before indexing
43    
44    =item --only=database_name
45    
46    reindex just single database (legacy name is --one)
47    
48  =item --config conf/config.yml  =item --config conf/config.yml
49    
50  path to YAML configuration file  path to YAML configuration file
# Line 54  my $limit; Line 59  my $limit;
59  my $clean = 0;  my $clean = 0;
60  my $config = 'conf/config.yml';  my $config = 'conf/config.yml';
61  my $debug = 0;  my $debug = 0;
62    my $only_db_name;
63    
64  GetOptions(  GetOptions(
65          "limit=i" => \$limit,          "limit=i" => \$limit,
66          "offset=i" => \$offset,          "offset=i" => \$offset,
67          "clean" => \$clean,          "clean" => \$clean,
68            "one=s" => \$only_db_name,
69            "only=s" => \$only_db_name,
70          "config" => \$config,          "config" => \$config,
71          "debug" => \$debug,          "debug" => \$debug,
72  );  );
# Line 70  print "config = ",Dumper($config) if ($d Line 78  print "config = ",Dumper($config) if ($d
78  die "no databases in config file!\n" unless ($config->{databases});  die "no databases in config file!\n" unless ($config->{databases});
79    
80  my $total_rows = 0;  my $total_rows = 0;
81    my $start_t = time();
82    
83  while (my ($database, $db_config) = each %{ $config->{databases} }) {  while (my ($database, $db_config) = each %{ $config->{databases} }) {
84    
85            next if ($only_db_name && $database !~ m/$only_db_name/i);
86    
87          my $log = _new WebPAC::Common()->_get_logger();          my $log = _new WebPAC::Common()->_get_logger();
88    
89          #          #
# Line 82  while (my ($database, $db_config) = each Line 93  while (my ($database, $db_config) = each
93          my $est_config = $config->{hyperestraier} || $log->logdie("can't find 'hyperestraier' part in confguration");          my $est_config = $config->{hyperestraier} || $log->logdie("can't find 'hyperestraier' part in confguration");
94          $est_config->{database} = $database;          $est_config->{database} = $database;
95          $est_config->{clean} = $clean;          $est_config->{clean} = $clean;
96            $est_config->{label} = $db_config->{name};
97    
98          my $est = new WebPAC::Output::Estraier( %{ $est_config } );          my $est = new WebPAC::Output::Estraier( %{ $est_config } );
99    
# Line 97  while (my ($database, $db_config) = each Line 109  while (my ($database, $db_config) = each
109                  $log->info("creating new database $database in $db_path");                  $log->info("creating new database $database in $db_path");
110                  rmtree( $db_path ) || $log->warn("can't remove $db_path: $!");                  rmtree( $db_path ) || $log->warn("can't remove $db_path: $!");
111          } else {          } else {
112                  $log->info("working on $database in $db_path");                  $log->debug("working on $database in $db_path");
113          }          }
114    
115          my $db = new WebPAC::Store(          my $db = new WebPAC::Store(
# Line 134  while (my ($database, $db_config) = each Line 146  while (my ($database, $db_config) = each
146    
147                  my $input_module = $config->{webpac}->{inputs}->{$type};                  my $input_module = $config->{webpac}->{inputs}->{$type};
148    
149                  $log->info("working on input $input->{path} [$input->{type}] using $input_module");                  $log->info("working on input '$input->{path}' [$input->{type}] using $input_module lookup '$input->{lookup}'");
150    
151                  my $input_db = new WebPAC::Input(                  my $input_db = new WebPAC::Input(
152                          module => $input_module,                          module => $input_module,
# Line 142  while (my ($database, $db_config) = each Line 154  while (my ($database, $db_config) = each
154                          limit => $limit || $input->{limit},                          limit => $limit || $input->{limit},
155                          offset => $offset,                          offset => $offset,
156                          lookup => $lookup,                          lookup => $lookup,
157                            recode => $input->{recode},
158                  );                  );
159                  $log->logdie("can't create input using $input_module") unless ($input);                  $log->logdie("can't create input using $input_module") unless ($input);
160    
# Line 197  while (my ($database, $db_config) = each Line 210  while (my ($database, $db_config) = each
210    
211          };          };
212    
213          $log->info("$total_rows records indexed");          my $dt = time() - $start_t;
214            $log->info("$total_rows records indexed in " .
215                    sprintf("%.2f sec [%.2f rec/sec]",
216                            $dt, ($total_rows / $dt)
217                    )
218            );
219    
220          #          #
221          # add Hyper Estraier links to other databases          # add Hyper Estraier links to other databases
Legend:
Removed from v.307  
changed lines
  Added in v.423
  ViewVC Help
Powered by ViewVC 1.1.26