upstream/0.5.3/estcmd.c

/*************************************************************************************************
 * The command line interface for the core API
 *                                                      Copyright (C) 2004-2005 Mikio Hirabayashi
 * This file is part of Hyper Estraier.
 * Hyper Estraier is free software; you can redistribute it and/or modify it under the terms of
 * the GNU Lesser General Public License as published by the Free Software Foundation; either
 * version 2.1 of the License or any later version.  Hyper Estraier is distributed in the hope
 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 * You should have received a copy of the GNU Lesser General Public License along with Hyper
 * Estraier; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307 USA.
 *************************************************************************************************/


#include "estraier.h"
#include "myconf.h"

#define NUMBUFSIZ      32                /* size of a buffer for a number */
#define URIBUFSIZ      8192              /* size of a buffer for an URI */
#define MINIBNUM       31                /* bucket number of a small map */
#define SEARCHMAX      10                /* maximum number of shown documents */
#define SNIPWWIDTH     480               /* whole width of the snippet */
#define SNIPHWIDTH     96                /* width of beginning of the text */
#define SNIPAWIDTH     96                /* width around each highlighted word */
#define CACHEMAX       (512*1024*1024)   /* max chache size by mega bytes */
#define DATTRLPATH     "_lpath"          /* name of the attribute of the local path */
#define DATTRLFILE     "_lfile"          /* name of the attribute of the local file name */
#define DATTRSCORE     "#score"          /* name of the pseudo-attribute of score */
#define DATTRKWORDS    "#kwords"         /* name of the pseudo-attribute of keywords */
#define KWDBNAME       "kwords"          /* name of the database for keywords */
#define KWDBBNUM       122869            /* bucket number of the keyword database */
#define KWDBDNUM       3                 /* division number of the keyword database */
#define KWORDNUM       32                /* number of shown keywords */
#define RDOCSNUM       6                 /* number of sections of a raondom document */
#define RDOCCNUM       256               /* number of characters for int a section */

enum {                                   /* enumeration for viewing modes */
  VM_ID,                                 /* ID only */
  VM_URI,                                /* ID and URI */
  VM_ATTR,                               /* all attributes */
  VM_FULL,                               /* all attributes and body text */
  VM_SNIP,                               /* all attributes and snippet */
  VM_HMRD,                               /* human readable */
  VM_XML,                                /* XML */
  VM_DUMP                                /* dump draft files */
};

enum {                                   /* enumeration for file formats */
  FF_AUTO,                               /* automatic detection */
  FF_DRAFT,                              /* draft */
  FF_TEXT,                               /* plain text */
  FF_HTML,                               /* HTML */
  FF_MIME,                               /* MIME */
  FF_NONE                                /* ignored */
};

enum {                                   /* enumeration for test documents */
  RD_ENG,                                /* English */
  RD_LAT,                                /* Latin */
  RD_EURO,                               /* European mix */
  RD_ORI,                                /* Oriental */
  RD_JPN,                                /* Japanese */
  RD_CHAO,                               /* chaos */
  RD_RAND                                /* selected at random */
};


/* global variables */
const char *g_progname;                  /* program name */
int g_sigterm = FALSE;                   /* flag for termination signal */
int g_putopts = 0;                       /* options of registration */
int g_outopts = 0;                       /* options of deletion */
int g_optopts = 0;                       /* options of optimization */
const char *g_inputcode = "UTF-8";       /* input encoding */
int g_inputlang = ESTLANGEN;             /* prefered language */
const char *g_pathcode = NULL;           /* path encoding */
int g_pathfull = FALSE;                  /* whether to record full paths */
CBLIST *g_pathattrs = NULL;              /* names of elements in path extension */
int g_oextmodes = 0;                     /* extra open modes */
int g_viewmode = VM_ID;                  /* viewing mode */
int g_filefmt = FF_AUTO;                 /* file format */
CBMAP *g_xcmdmap = NULL;                 /* map of suffixes and filter commands */
int g_filtorig = FALSE;                  /* whether to use filter for original files */
int g_stdate = FALSE;                    /* whether to adopt date by stat */
int g_chkmdate = FALSE;                  /* whether to check modification date */
double g_cachesize = -1;                 /* size of the cache */
int g_doforce = FALSE;                   /* whether to force purging or extracting */
int g_kwordnum = KWORDNUM;               /* number of keywords */
int g_rdmode = RD_RAND;                  /* mode of random documents */


/* function prototypes */
int main(int argc, char **argv);
static void printferror(const char *format, ...);
static void printfinfo(const char *format, ...);
static void dbinform(const char *msg);
static void setsignals(void);
static void sigtermhandler(int num);
static void usage(void);
static int runput(int argc, char **argv);
static int runout(int argc, char **argv);
static int runget(int argc, char **argv);
static int runlist(int argc, char **argv);
static int runuriid(int argc, char **argv);
static int runmeta(int argc, char **argv);
static int runinform(int argc, char **argv);
static int runoptimize(int argc, char **argv);
static int runsearch(int argc, char **argv);
static int rungather(int argc, char **argv);
static int runpurge(int argc, char **argv);
static int runextkeys(int argc, char **argv);
static int rundraft(int argc, char **argv);
static int runbreak(int argc, char **argv);
static int runrandput(int argc, char **argv);
static int runwicked(int argc, char **argv);
static int runregression(int argc, char **argv);
static int procput(const char *dbname, const char *filename);
static int procout(const char *dbname, int id, const char *expr);
static int procget(const char *dbname, int id, const char *expr, const char *attr);
static int proclist(const char *dbname);
static int procuriid(const char *dbname, const char *uri);
static int procmeta(const char *dbname, const char *mname, const char *mvalue);
static int procinform(const char *dbname);
static int procoptimize(const char *dbname);
static int procsearch(const char *dbname, const char *phrase,
                      const CBLIST *attrs, const char *ord, int max, int opts, int sim);
static int procgather(const char *dbname, const char *filename);
static int procpurge(const char *dbname, const char *prefix);
static int procextkeys(const char *dbname, const char *prefix, int ni);
static int procdraft(const char *filename);
static int procbreak(const char *filename, int wt);
static int procrandput(const char *dbname, int dnum);
static int procwicked(const char *dbname, int dnum);
static int procregression(const char *dbname);
static void xmlprintf(const char *format, ...);
static int strtolang(const char *str);
static char *fgetl(FILE *ifp);
static int doputdoc(ESTDB *db, const char *path, const CBLIST *attrs);
static const char *pathtourl(const char *path);
static const char *urltofile(const char *uri);
static char *urltopath(const char *uri);
static CBMAP *vectorizer(void *db, int id, void *kwdb);
static ESTDOC *est_doc_new_with_xcmd(const char *buf, int size, const char *path,
                                     const char *xcmd, const char *tmpdir,
                                     const char *penc, int plang);
static ESTDOC *est_doc_new_from_draft_enc(const char *buf, int size, const char *enc);
static ESTDOC *est_doc_new_from_text(const char *buf, int size, const char *penc, int plang);
static ESTDOC *est_doc_new_from_html(const char *buf, int size, const char *penc, int plang);
static char *est_html_enc(const char *str);
static char *est_html_raw_text(const char *html);
static ESTDOC *est_doc_new_from_mime(const char *buf, int size, const char *penc, int plang);
static void est_doc_add_attr_mime(ESTDOC *doc, const char *name, const char *value);
static ESTDOC *est_doc_new_from_chaos(int cnum, int snum, int mode);
static char *est_random_str(int cnum, int mode);


/* main routine */
int main(int argc, char **argv){
  const char *tmp;
  int rv;
  if((tmp = getenv("ESTDBGFD")) != NULL) dpdbgfd = atoi(tmp);
  cbstdiobin();
  g_progname = argv[0];
  g_sigterm = FALSE;
  if(argc < 2) usage();
  rv = 0;
  if(!strcmp(argv[1], "put")){
    setsignals();
    rv = runput(argc, argv);
  } else if(!strcmp(argv[1], "out")){
    setsignals();
    rv = runout(argc, argv);
  } else if(!strcmp(argv[1], "get")){
    rv = runget(argc, argv);
  } else if(!strcmp(argv[1], "list")){
    rv = runlist(argc, argv);
  } else if(!strcmp(argv[1], "uriid")){
    rv = runuriid(argc, argv);
  } else if(!strcmp(argv[1], "meta")){
    setsignals();
    rv = runmeta(argc, argv);
  } else if(!strcmp(argv[1], "inform")){
    rv = runinform(argc, argv);
  } else if(!strcmp(argv[1], "optimize")){
    setsignals();
    rv = runoptimize(argc, argv);
  } else if(!strcmp(argv[1], "search")){
    rv = runsearch(argc, argv);
  } else if(!strcmp(argv[1], "gather")){
    setsignals();
    rv = rungather(argc, argv);
  } else if(!strcmp(argv[1], "purge")){
    setsignals();
    rv = runpurge(argc, argv);
  } else if(!strcmp(argv[1], "extkeys")){
    setsignals();
    rv = runextkeys(argc, argv);
  } else if(!strcmp(argv[1], "draft")){
    rv = rundraft(argc, argv);
  } else if(!strcmp(argv[1], "break")){
    rv = runbreak(argc, argv);
  } else if(!strcmp(argv[1], "randput")){
    setsignals();
    rv = runrandput(argc, argv);
  } else if(!strcmp(argv[1], "wicked")){
    setsignals();
    rv = runwicked(argc, argv);
  } else if(!strcmp(argv[1], "regression")){
    setsignals();
    rv = runregression(argc, argv);
  } else if(!strcmp(argv[1], "version") || !strcmp(argv[1], "--version")){
    printf("Hyper Estraier %s on %s\n", est_version, ESTSYSNAME);
    printf("Copyright (C) 2004-2005 Mikio Hirabayashi.\n");
    rv = 0;
  } else {
    usage();
  }
  return rv;
}


/* print formatted error string and flush the buffer */
static void printferror(const char *format, ...){
  va_list ap;
  va_start(ap, format);
  fprintf(stderr, "%s: ERROR: ", g_progname);
  vfprintf(stderr, format, ap);
  fputc('\n', stderr);
  fflush(stderr);
  va_end(ap);
}


/* print formatted information string and flush the buffer */
static void printfinfo(const char *format, ...){
  va_list ap;
  va_start(ap, format);
  printf("%s: INFO: ", g_progname);
  vprintf(format, ap);
  putchar('\n');
  fflush(stdout);
  va_end(ap);
}


/* callback function for database events */
static void dbinform(const char *msg){
  printfinfo("%s", msg);
}


/* set signal handlers */
static void setsignals(void){
  signal(1, sigtermhandler);
  signal(2, sigtermhandler);
  signal(3, sigtermhandler);
  signal(13, sigtermhandler);
  signal(15, sigtermhandler);
}


/* handler of termination signal */
static void sigtermhandler(int num){
  static int tries = 0;
  if(tries++ <= 4){
    signal(num, sigtermhandler);
  } else {
    signal(num, SIG_DFL);
  }
  g_sigterm = TRUE;
  printfinfo("the termination signal %d catched", num);
}


/* print the usage and exit */
static void usage(void){
  fprintf(stderr, "%s: command line utility for the core API of Hyper Estraier\n", g_progname);
  fprintf(stderr, "\n");
  fprintf(stderr, "usage:\n");
  fprintf(stderr, "  %s put [-cl] db [file]\n", g_progname);
  fprintf(stderr, "  %s out [-cl] db expr\n", g_progname);
  fprintf(stderr, "  %s get db expr\n", g_progname);
  fprintf(stderr, "  %s list db\n", g_progname);
  fprintf(stderr, "  %s uriid db uri\n", g_progname);
  fprintf(stderr, "  %s meta db [name [value]]\n", g_progname);
  fprintf(stderr, "  %s inform db\n", g_progname);
  fprintf(stderr, "  %s optimize [-onp] [-ond] db\n", g_progname);
  fprintf(stderr, "  %s search [-ic enc] [-vu|-va|-vf|-vs|-vh|-vx|-dd] [-gs|-gf|-ga]"
          " [-ni] [-sf] [-hs] [-attr expr] [-ord expr] [-max num] [-sim id] db [phrase]\n",
          g_progname);
  fprintf(stderr, "  %s gather [-cl] [-fe|-ft|-fh|-fm] [-fx sufs cmd] [-fz] [-fo]"
          " [-ic enc] [-il lang] [-pc enc] [-pf] [-px name] [-apn] [-sd] [-cm] [-cs num]"
          " db [file|dir]\n", g_progname);
  fprintf(stderr, "  %s purge [-cl] [-fc] db [prefix]\n", g_progname);
  fprintf(stderr, "  %s extkeys [-fc] [-ni] [-kn num] db [prefix]\n", g_progname);
  fprintf(stderr, "  %s draft [-ft|-fh|-fm] [-ic enc] [-il lang] [file]\n", g_progname);
  fprintf(stderr, "  %s break [-ic enc] [-il lang] [-apn] [-wt] [file]\n", g_progname);
  fprintf(stderr, "  %s randput [-ren|-rla|-reu|-ror|-rjp|-rch] [-cs num] db dnum\n",
          g_progname);
  fprintf(stderr, "  %s wicked db dnum\n", g_progname);
  fprintf(stderr, "  %s regression db\n", g_progname);
  fprintf(stderr, "  %s version\n", g_progname);
  fprintf(stderr, "\n");
  exit(1);
}


/* parse arguments of the put command */
static int runput(int argc, char **argv){
  char *dbname, *filename;
  int i, rv;
  dbname = NULL;
  filename = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-cl")){
        g_putopts |= ESTPDCLEAN;
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else if(!filename){
      filename = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname) usage();
  rv = procput(dbname, filename);
  return rv;
}


/* parse arguments of the out command */
static int runout(int argc, char **argv){
  char *dbname, *expr;
  int i, id, rv;
  dbname = NULL;
  expr = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-cl")){
        g_outopts |= ESTODCLEAN;
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else if(!expr){
      expr = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname || !expr) usage();
  if((id = atoi(expr)) > 0) expr = NULL;
  rv = procout(dbname, id, expr);
  return rv;
}


/* parse arguments of the get command */
static int runget(int argc, char **argv){
  char *dbname, *expr, *attr;
  int i, id, rv;
  dbname = NULL;
  expr = NULL;
  attr = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      usage();
    } else if(!dbname){
      dbname = argv[i];
    } else if(!expr){
      expr = argv[i];
    } else if(!attr){
      attr = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname || !expr) usage();
  if((id = atoi(expr)) > 0) expr = NULL;
  rv = procget(dbname, id, expr, attr);
  return rv;
}


/* parse arguments of the list command */
static int runlist(int argc, char **argv){
  char *dbname;
  int i, rv;
  dbname = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      usage();
    } else if(!dbname){
      dbname = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname) usage();
  rv = proclist(dbname);
  return rv;
}


/* parse arguments of the uriid command */
static int runuriid(int argc, char **argv){
  char *dbname, *uri;
  int i, rv;
  dbname = NULL;
  uri = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      usage();
    } else if(!dbname){
      dbname = argv[i];
    } else if(!uri){
      uri = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname || !uri) usage();
  rv = procuriid(dbname, uri);
  return rv;
}


/* parse arguments of the meta command */
static int runmeta(int argc, char **argv){
  char *dbname, *mname, *mvalue;
  int i, del, rv;
  dbname = NULL;
  mname = NULL;
  mvalue = NULL;
  del = FALSE;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      usage();
    } else if(!dbname){
      dbname = argv[i];
    } else if(!mname){
      mname = argv[i];
    } else if(!mvalue){
      mvalue = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname) usage();
  rv = procmeta(dbname, mname, mvalue);
  return rv;
}


/* parse arguments of the inform command */
static int runinform(int argc, char **argv){
  char *dbname;
  int i, rv;
  dbname = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      usage();
    } else if(!dbname){
      dbname = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname) usage();
  rv = procinform(dbname);
  return rv;
}


/* parse arguments of the optimize command */
static int runoptimize(int argc, char **argv){
  char *dbname;
  int i, rv;
  dbname = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-onp")){
        g_optopts |= ESTOPTNOPURGE;
      } else if(!strcmp(argv[i], "-ond")){
        g_optopts |= ESTOPTNODBOPT;
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname) usage();
  rv = procoptimize(dbname);
  return rv;
}


/* parse arguments of the search command */
static int runsearch(int argc, char **argv){
  CBDATUM *pbuf;
  CBLIST *attrs;
  char *dbname, *ord, *phrase, *tmp;
  int i, max, opts, sim, rv;
  dbname = NULL;
  ord = NULL;
  max = SEARCHMAX;
  opts = 0;
  sim = -1;
  pbuf = cbdatumopen("", 0);
  cbglobalgc(pbuf, (void (*)(void *))cbdatumclose);
  attrs = cblistopen();
  cbglobalgc(attrs, (void (*)(void *))cblistclose);
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-ic")){
        if(++i >= argc) usage();
        g_inputcode = argv[i];
      } else if(!strcmp(argv[i], "-gs")){
        opts |= ESTCONDSURE;
      } else if(!strcmp(argv[i], "-gf")){
        opts |= ESTCONDFAST;
      } else if(!strcmp(argv[i], "-ga")){
        opts |= ESTCONDAGIT;
      } else if(!strcmp(argv[i], "-ni")){
        opts |= ESTCONDNOIDF;
      } else if(!strcmp(argv[i], "-sf")){
        opts |= ESTCONDSIMPLE;
      } else if(!strcmp(argv[i], "-hs")){
        opts |= ESTCONDSCFB;
      } else if(!strcmp(argv[i], "-vu")){
        g_viewmode = VM_URI;
      } else if(!strcmp(argv[i], "-va")){
        g_viewmode = VM_ATTR;
      } else if(!strcmp(argv[i], "-vf")){
        g_viewmode = VM_FULL;
      } else if(!strcmp(argv[i], "-vs")){
        g_viewmode = VM_SNIP;
      } else if(!strcmp(argv[i], "-vh")){
        g_viewmode = VM_HMRD;
      } else if(!strcmp(argv[i], "-vx")){
        g_viewmode = VM_XML;
      } else if(!strcmp(argv[i], "-dd")){
        g_viewmode = VM_DUMP;
      } else if(!strcmp(argv[i], "-attr")){
        if(++i >= argc) usage();
        cblistpush(attrs, argv[i], -1);
      } else if(!strcmp(argv[i], "-ord")){
        if(++i >= argc) usage();
        ord = argv[i];
      } else if(!strcmp(argv[i], "-max")){
        if(++i >= argc) usage();
        max = atoi(argv[i]);
      } else if(!strcmp(argv[i], "-sim")){
        if(++i >= argc) usage();
        sim = atoi(argv[i]);
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else {
      if(cbdatumsize(pbuf) > 0) cbdatumcat(pbuf, " ", 1);
      cbdatumcat(pbuf, argv[i], -1);
    }
  }
  if(!dbname) usage();
  if(!(phrase = est_iconv(cbdatumptr(pbuf), -1, g_inputcode, "UTF-8", NULL, NULL))){
    printferror("%s: unsupported encoding\n", g_inputcode);
    return 1;
  }
  cbstrtrim(phrase);
  for(i = 0; i < cblistnum(attrs); i++){
    if((tmp = est_iconv(cblistval(attrs, i, NULL), -1, g_inputcode, "UTF-8", NULL, NULL)) != NULL){
      cblistover(attrs, i, tmp, -1);
      free(tmp);
    }
  }
  rv = procsearch(dbname, phrase, attrs, ord, max, opts, sim);
  free(phrase);
  return rv;
}


/* parse arguments of the gather command */
static int rungather(int argc, char **argv){
  CBLIST *list;
  const char *elem;
  char *dbname, *filename;
  int i, j, rv;
  g_pathattrs = cblistopen();
  cbglobalgc(g_pathattrs, (void (*)(void *))cblistclose);
  g_xcmdmap = cbmapopenex(MINIBNUM);
  cbglobalgc(g_xcmdmap, (void (*)(void *))cbmapclose);
  dbname = NULL;
  filename = NULL;
  g_inputcode = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-cl")){
        g_putopts |= ESTPDCLEAN;
      } else if(!strcmp(argv[i], "-fe")){
        g_filefmt = FF_DRAFT;
      } else if(!strcmp(argv[i], "-ft")){
        g_filefmt = FF_TEXT;
      } else if(!strcmp(argv[i], "-fh")){
        g_filefmt = FF_HTML;
      } else if(!strcmp(argv[i], "-fm")){
        g_filefmt = FF_MIME;
      } else if(!strcmp(argv[i], "-fx")){
        if((i += 2) >= argc) usage();
        list = cbsplit(argv[i-1], -1, ",");
        for(j = 0; j < cblistnum(list); j++){
          elem = cblistval(list, j, NULL);
          if(elem[0] != '\0') cbmapput(g_xcmdmap, elem, -1, argv[i], -1, FALSE);
        }
        cblistclose(list);
      } else if(!strcmp(argv[i], "-fz")){
        g_filefmt = FF_NONE;
      } else if(!strcmp(argv[i], "-fo")){
        g_filtorig = TRUE;
      } else if(!strcmp(argv[i], "-ic")){
        if(++i >= argc) usage();
        g_inputcode = argv[i];
      } else if(!strcmp(argv[i], "-il")){
        if(++i >= argc) usage();
        g_inputlang = strtolang(argv[i]);
      } else if(!strcmp(argv[i], "-pc")){
        if(++i >= argc) usage();
        g_pathcode = argv[i];
      } else if(!strcmp(argv[i], "-pf")){
        g_pathfull = TRUE;
      } else if(!strcmp(argv[i], "-px")){
        if(++i >= argc) usage();
        cblistpush(g_pathattrs, argv[i], -1);
      } else if(!strcmp(argv[i], "-apn")){
        g_oextmodes |= ESTDBPERFNG;
      } else if(!strcmp(argv[i], "-sd")){
        g_stdate = TRUE;
      } else if(!strcmp(argv[i], "-cm")){
        g_chkmdate = TRUE;
      } else if(!strcmp(argv[i], "-cs")){
        if(++i >= argc) usage();
        g_cachesize = strtod(argv[i], NULL) * 1024 * 1024;
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else if(!filename){
      filename = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname || !filename) usage();
  rv = procgather(dbname, filename);
  return rv;
}


/* parse arguments of the purge command */
static int runpurge(int argc, char **argv){
  char *dbname, *prefix;
  int i, rv;
  dbname = NULL;
  prefix = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-cl")){
        g_outopts |= ESTODCLEAN;
      } else if(!strcmp(argv[i], "-fc")){
        g_doforce = TRUE;
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else if(!prefix){
      prefix = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname) usage();
  rv = procpurge(dbname, prefix);
  return rv;
}


/* parse arguments of the extkeys command */
static int runextkeys(int argc, char **argv){
  char *dbname, *prefix;
  int i, ni, rv;
  dbname = NULL;
  prefix = NULL;
  ni = FALSE;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-fc")){
        g_doforce = TRUE;
      } else if(!strcmp(argv[i], "-ni")){
        ni = TRUE;
      } else if(!strcmp(argv[i], "-kn")){
        if(++i >= argc) usage();
        g_kwordnum = atoi(argv[i]);
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else if(!prefix){
      prefix = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname || g_kwordnum < 1) usage();
  rv = procextkeys(dbname, prefix, ni);
  return rv;
}


/* parse arguments of the draft command */
static int rundraft(int argc, char **argv){
  char *filename;
  int i, rv;
  filename = NULL;
  g_filefmt = FF_DRAFT;
  g_inputcode = NULL;
  for(i = 2; i < argc; i++){
    if(!filename && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-ft")){
        g_filefmt = FF_TEXT;
      } else if(!strcmp(argv[i], "-fh")){
        g_filefmt = FF_HTML;
      } else if(!strcmp(argv[i], "-fm")){
        g_filefmt = FF_MIME;
      } else if(!strcmp(argv[i], "-ic")){
        if(++i >= argc) usage();
        g_inputcode = argv[i];
      } else if(!strcmp(argv[i], "-il")){
        if(++i >= argc) usage();
        g_inputlang = strtolang(argv[i]);
      } else {
        usage();
      }
    } else if(!filename){
      filename = argv[i];
    } else {
      usage();
    }
  }
  rv = procdraft(filename);
  return rv;
}


/* parse arguments of the break command */
static int runbreak(int argc, char **argv){
  char *filename;
  int i, wt, rv;
  filename = NULL;
  wt = FALSE;
  for(i = 2; i < argc; i++){
    if(!filename && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-ic")){
        if(++i >= argc) usage();
        g_inputcode = argv[i];
      } else if(!strcmp(argv[i], "-il")){
        if(++i >= argc) usage();
        g_inputlang = strtolang(argv[i]);
      } else if(!strcmp(argv[i], "-apn")){
        g_oextmodes |= ESTDBPERFNG;
      } else if(!strcmp(argv[i], "-wt")){
        wt = TRUE;
      } else {
        usage();
      }
    } else if(!filename){
      filename = argv[i];
    } else {
      usage();
    }
  }
  rv = procbreak(filename, wt);
  return rv;
}


/* parse arguments of the randput command */
static int runrandput(int argc, char **argv){
  char *dbname, *dnstr;
  int i, dnum, rv;
  dbname = NULL;
  dnstr = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-ren")){
        g_rdmode = RD_ENG;
      } else if(!strcmp(argv[i], "-rla")){
        g_rdmode = RD_LAT;
      } else if(!strcmp(argv[i], "-reu")){
        g_rdmode = RD_EURO;
      } else if(!strcmp(argv[i], "-ror")){
        g_rdmode = RD_ORI;
      } else if(!strcmp(argv[i], "-rjp")){
        g_rdmode = RD_JPN;
      } else if(!strcmp(argv[i], "-rch")){
        g_rdmode = RD_CHAO;
      } else if(!strcmp(argv[i], "-cs")){
        if(++i >= argc) usage();
        g_cachesize = strtod(argv[i], NULL) * 1024 * 1024;
      } else {
        usage();
      }
    } else if(!dbname){
      dbname = argv[i];
    } else if(!dnstr){
      dnstr = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname || !dnstr) usage();
  if((dnum = atoi(dnstr)) < 1) usage();
  rv = procrandput(dbname, dnum);
  return rv;
}


/* parse arguments of the wicked command */
static int runwicked(int argc, char **argv){
  char *dbname, *dnstr;
  int i, dnum, rv;
  dbname = NULL;
  dnstr = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      usage();
    } else if(!dbname){
      dbname = argv[i];
    } else if(!dnstr){
      dnstr = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname || !dnstr) usage();
  if((dnum = atoi(dnstr)) < 1) usage();
  rv = procwicked(dbname, dnum);
  return rv;
}


/* parse arguments of the regression command */
static int runregression(int argc, char **argv){
  char *dbname;
  int i, rv;
  dbname = NULL;
  for(i = 2; i < argc; i++){
    if(!dbname && argv[i][0] == '-'){
      usage();
    } else if(!dbname){
      dbname = argv[i];
    } else {
      usage();
    }
  }
  if(!dbname) usage();
  rv = procregression(dbname);
  return rv;
}


/* perform the put command */
static int procput(const char *dbname, const char *filename){
  ESTDB *db;
  ESTDOC *doc;
  const char *uri;
  char *draft;
  int ecode;
  if(!(draft = cbreadfile(filename, NULL))){
    printferror("%s: could not open", filename ? filename : "(stdin)");
    return 1;
  }
  if(!(db = est_db_open(dbname, ESTDBWRITER | ESTDBCREAT, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    free(draft);
    return 1;
  }
  est_db_set_informer(db, dbinform);
  doc = est_doc_new_from_draft(draft);
  if(!est_db_put_doc(db, doc, g_putopts)){
    printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
    est_doc_delete(doc);
    est_db_close(db, &ecode);
    free(draft);
    return 1;
  }
  if(!(uri = est_doc_attr(doc, ESTDATTRURI))) uri = "";
  printfinfo("%d (%s): registered", est_doc_id(doc), uri);
  est_doc_delete(doc);
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    free(draft);
    return 1;
  }
  free(draft);
  return 0;
}


/* perform the out command */
static int procout(const char *dbname, int id, const char *expr){
  ESTDB *db;
  int ecode;
  if(!(db = est_db_open(dbname, ESTDBWRITER, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  est_db_set_informer(db, dbinform);
  if(expr && (id = est_db_uri_to_id(db, expr)) < 1){
    printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
    est_db_close(db, &ecode);
    return 1;
  }
  if(!est_db_out_doc(db, id, g_outopts)){
    printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
    est_db_close(db, &ecode);
    return 1;
  }
  printfinfo("%d: deleted", id);
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  return 0;
}


/* perform the get command */
static int procget(const char *dbname, int id, const char *expr, const char *attr){
  ESTDB *db;
  ESTDOC *doc;
  char *draft;
  int ecode;
  if(!(db = est_db_open(dbname, ESTDBREADER | ESTDBLCKNB, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  if(expr && (id = est_db_uri_to_id(db, expr)) < 1){
    printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
    est_db_close(db, &ecode);
    return 1;
  }
  if(attr){
    if(!(draft = est_db_get_doc_attr(db, id, attr))){
      printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
      est_db_close(db, &ecode);
      return 1;
    }
    printf("%s\n", draft);
    free(draft);
  } else {
    if(!(doc = est_db_get_doc(db, id, 0))){
      printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
      est_db_close(db, &ecode);
      return 1;
    }
    draft = est_doc_dump_draft(doc);
    printf("%s", draft);
    free(draft);
    est_doc_delete(doc);
  }
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  return 0;
}


/* perform the list command */
static int proclist(const char *dbname){
  ESTDB *db;
  ESTDOC *doc;
  const char *vbuf;
  int ecode, id;
  if(!(db = est_db_open(dbname, ESTDBREADER | ESTDBLCKNB, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  if(!est_db_iter_init(db)){
    printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
    est_db_close(db, &ecode);
    return 1;
  }
  while((id = est_db_iter_next(db)) > 0){
    if((doc = est_db_get_doc(db, id, ESTGDNOTEXT)) != NULL){
      if(!(vbuf = est_doc_attr(doc, ESTDATTRURI))) vbuf = "";
      printf("%d\t%s\n", id, vbuf);
      est_doc_delete(doc);
    }
  }
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  return 0;
}


/* perform the uriid command */
static int procuriid(const char *dbname, const char *uri){
  ESTDB *db;
  int ecode, id;
  if(!(db = est_db_open(dbname, ESTDBREADER | ESTDBLCKNB, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  if((id = est_db_uri_to_id(db, uri)) == -1){
    printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
    est_db_close(db, &ecode);
    return 1;
  }
  printf("%d\n", id);
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  return 0;
}


/* perform the meta command */
static int procmeta(const char *dbname, const char *mname, const char *mvalue){
  ESTDB *db;
  CBLIST *names;
  char *vbuf;
  int i, ecode;
  if(!(db = est_db_open(dbname, mvalue ? (ESTDBWRITER | ESTDBCREAT) : (ESTDBREADER | ESTDBLCKNB),
                        &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  if(mname){
    if(mvalue){
      est_db_add_meta(db, mname, mvalue[0] != '\0' ? mvalue : NULL);
    } else {
      if((vbuf = est_db_meta(db, mname)) != NULL){
        printf("%s\n", vbuf);
        free(vbuf);
      }
    }
  } else {
    names = est_db_meta_names(db);
    for(i = 0; i < cblistnum(names); i++){
      printf("%s\n", cblistval(names, i, NULL));
    }
    cblistclose(names);
  }
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  return 0;
}


/* perform the inform command */
static int procinform(const char *dbname){
  ESTDB *db;
  int ecode;
  if(!(db = est_db_open(dbname, ESTDBREADER | ESTDBLCKNB, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  printf("number of documents: %d\n", est_db_doc_num(db));
  printf("number of words: %d\n", est_db_word_num(db));
  printf("file size: %.0f\n", est_db_size(db));
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  return 0;
}


/* perform the optimize command */
static int procoptimize(const char *dbname){
  ESTDB *db;
  char path[URIBUFSIZ];
  int ecode;
  time_t curtime;
  curtime = time(NULL);
  if(!(db = est_db_open(dbname, ESTDBWRITER, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  est_db_set_informer(db, dbinform);
  sprintf(path, "%s%c%s", dbname, ESTPATHCHR, KWDBNAME);
  unlink(path);
  if(!est_db_optimize(db, g_optopts)){
    printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
    est_db_close(db, &ecode);
    return 1;
  }
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  curtime = time(NULL) - curtime;
  printfinfo("finished successfully: elapsed time: %dh %dm %ds",
             (int)(curtime / 3600), (int)((curtime / 60) % 60), (int)(curtime % 60));
  return 0;
}


/* perform the search command */
static int procsearch(const char *dbname, const char *phrase,
                      const CBLIST *attrs, const char *ord, int max, int opts, int sim){
  ESTDB *db;
  ESTCOND *cond;
  ESTDOC *doc;
  CURIA *kwdb;
  CBDATUM *pbuf;
  CBMAP *svmap, *hints, *kwords;
  CBLIST *names, *words, *lines;
  const char *kbuf, *vbuf, *line;
  char *draft, path[URIBUFSIZ], numbuf[NUMBUFSIZ], *word, *pv;
  int i, j, ecode, ksiz, vsiz, *res, rnum, id, sc, fin, cnt;
  double curtime;
  if(!(db = est_db_open(dbname, ESTDBREADER | ESTDBLCKNB, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  sprintf(path, "%s%c%s", dbname, ESTPATHCHR, KWDBNAME);
  if((kwdb = cropen(path, CR_OREADER, -1, -1)) != NULL)
    est_db_set_vectorizer(db, vectorizer, kwdb);
  cond = est_cond_new();
  if(sim > 0){
    svmap = kwdb ? vectorizer(db, sim, kwdb) : NULL;
    if(!svmap && (doc = est_db_get_doc(db, sim, 0)) != NULL){
      svmap = est_db_etch_doc((opts & ESTCONDNOIDF) ? NULL : db, doc, KWORDNUM);
      est_doc_delete(doc);
    }
    if(svmap){
      pbuf = cbdatumopen(ESTOPSIMILAR, -1);
      cbmapiterinit(svmap);
      while((kbuf = cbmapiternext(svmap, &ksiz)) != NULL){
        vbuf = cbmapget(svmap, kbuf, ksiz, &vsiz);
        cbdatumcat(pbuf, " WITH ", -1);
        cbdatumcat(pbuf, vbuf, vsiz);
        cbdatumcat(pbuf, " ", 1);
        cbdatumcat(pbuf, kbuf, ksiz);
      }
      est_cond_set_phrase(cond, cbdatumptr(pbuf));
      cbdatumclose(pbuf);
      cbmapclose(svmap);
    }
  } else {
    while(*phrase > '\0' && *phrase <= ' '){
      phrase++;
    }
    if(phrase[0] != '\0' || cblistnum(attrs) < 1) est_cond_set_phrase(cond, phrase);
  }
  for(i = 0; i < cblistnum(attrs); i++){
    est_cond_add_attr(cond, cblistval(attrs, i, NULL));
  }
  if(ord) est_cond_set_order(cond, ord);
  if(max >= 0) est_cond_set_max(cond, max);
  est_cond_set_options(cond, opts);
  hints = cbmapopenex(MINIBNUM);
  curtime = est_gettimeofday();
  res = est_db_search(db, cond, &rnum, hints);
  curtime = est_gettimeofday() - curtime;
  if(g_viewmode == VM_XML){
    xmlprintf("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
    xmlprintf("<estresult version=\"%@\">\n", est_version);
    xmlprintf("<meta>\n");
    xmlprintf("<hit number=\"%@\"/>\n", cbmapget(hints, "", 0, NULL));
    cbmapiterinit(hints);
    while((kbuf = cbmapiternext(hints, NULL)) != NULL){
      if(kbuf[0] == '\0') continue;
      vbuf = cbmapget(hints, kbuf, -1, NULL);
      xmlprintf("<hit key=\"%@\" number=\"%@\"/>\n", kbuf, vbuf);
    }
    xmlprintf("<time time=\"%.3f\"/>\n", curtime / 1000.0);
    xmlprintf("<total documents=\"%d\" words=\"%d\"/>\n",
              est_db_doc_num(db), est_db_word_num(db));
    xmlprintf("</meta>\n");
  } else {
    printf("%s\n", est_border_str());
    printf("VERSION\t%s\n", _EST_PROTVER);
    printf("NODE\tlocal\n");
    printf("HIT\t%s\n", cbmapget(hints, "", 0, NULL));
    cbmapiterinit(hints);
    cnt = 1;
    while((kbuf = cbmapiternext(hints, NULL)) != NULL){
      if(kbuf[0] == '\0') continue;
      vbuf = cbmapget(hints, kbuf, -1, NULL);
      printf("HINT#%d\t%s\t%s\n", cnt, kbuf, vbuf);
      cnt++;
    }
    printf("TIME\t%.3f\n", curtime / 1000.0);
    printf("DOCNUM\t%d\n", est_db_doc_num(db));
    printf("WORDNUM\t%d\n", est_db_word_num(db));
    switch(g_viewmode){
    case VM_ID:
      printf("VIEW\tID\n");
      break;
    case VM_URI:
      printf("VIEW\tURI\n");
      break;
    case VM_ATTR:
      printf("VIEW\tATTRIBUTE\n");
      break;
    case VM_FULL:
      printf("VIEW\tFULL\n");
      break;
    case VM_SNIP:
      printf("VIEW\tSNIPPET\n");
      break;
    case VM_HMRD:
      printf("VIEW\tHUMAN\n");
      break;
    }
    printf("\n");
    if(g_viewmode == VM_ID || g_viewmode == VM_URI ||
       g_viewmode == VM_HMRD || g_viewmode == VM_DUMP) printf("%s\n", est_border_str());
  }
  for(i = 0; i < rnum ; i++){
    id = res[i];
    sc = est_cond_score(cond, i);
    switch(g_viewmode){
    case VM_URI:
      if((doc = est_db_get_doc(db, id, ESTGDNOTEXT)) != NULL){
        if(!(vbuf = est_doc_attr(doc, ESTDATTRURI))) vbuf = "";
        printf("%d\t%s\n", id, vbuf);
        est_doc_delete(doc);
      }
      break;
    case VM_ATTR:
      if((doc = est_db_get_doc(db, id, ESTGDNOTEXT)) != NULL){
        if(sc >= 0){
          sprintf(numbuf, "%d", sc);
          est_doc_add_attr(doc, DATTRSCORE, numbuf);
        }
        printf("%s\n", est_border_str());
        names = est_doc_attr_names(doc);
        for(j = 0; j < cblistnum(names); j++){
          kbuf = cblistval(names, j, NULL);
          vbuf = est_doc_attr(doc, kbuf);
          printf("%s=%s\n", kbuf, vbuf);
        }
        cblistclose(names);
        est_doc_delete(doc);
      }
      printf("\n");
      break;
    case VM_FULL:
      if((doc = est_db_get_doc(db, id, 0)) != NULL){
        if(sc >= 0){
          sprintf(numbuf, "%d", sc);
          est_doc_add_attr(doc, DATTRSCORE, numbuf);
        }
        printf("%s\n", est_border_str());
        draft = est_doc_dump_draft(doc);
        printf("%s", draft);
        free(draft);
        est_doc_delete(doc);
      }
      break;
    case VM_SNIP:
      if((doc = est_db_get_doc(db, id, 0)) != NULL){
        if(sc >= 0){
          sprintf(numbuf, "%d", sc);
          est_doc_add_attr(doc, DATTRSCORE, numbuf);
        }
        printf("%s\n", est_border_str());
        names = est_doc_attr_names(doc);
        for(j = 0; j < cblistnum(names); j++){
          kbuf = cblistval(names, j, NULL);
          vbuf = est_doc_attr(doc, kbuf);
          printf("%s=%s\n", kbuf, vbuf);
        }
        cblistclose(names);
        kwords = kwdb ? vectorizer(db, id, kwdb) : NULL;
        if(!kwords) kwords = est_db_etch_doc(db, doc, KWORDNUM);
        if(cbmaprnum(kwords) > 0){
          printf("%s=", DATTRKWORDS);
          cbmapiterinit(kwords);
          for(j = 0; (kbuf = cbmapiternext(kwords, NULL)) != NULL; j++){
            if(j > 0) printf(" ");
            printf("%s %s", kbuf, cbmapget(kwords, kbuf, -1, NULL));
          }
          printf("\n");
        }
        cbmapclose(kwords);
        printf("\n");
        words = cbmapkeys(hints);
        draft = est_doc_make_snippet(doc, words, SNIPWWIDTH, SNIPHWIDTH, SNIPAWIDTH);
        printf("%s", draft);
        free(draft);
        cblistclose(words);
        est_doc_delete(doc);
      }
      break;
    case VM_HMRD:
      if((doc = est_db_get_doc(db, id, 0)) != NULL){
        if(sc >= 0){
          sprintf(numbuf, "%d", sc);
          est_doc_add_attr(doc, DATTRSCORE, numbuf);
        }
        printf("\n");
        if((vbuf = est_doc_attr(doc, ESTDATTRURI)) != NULL) printf("URI: %s\n", vbuf);
        if((vbuf = est_doc_attr(doc, ESTDATTRTITLE)) != NULL) printf("Title: %s\n", vbuf);
        printf("  ");
        words = cbmapkeys(hints);
        draft = est_doc_make_snippet(doc, words, SNIPWWIDTH, SNIPHWIDTH, SNIPAWIDTH);
        lines = cbsplit(draft, -1, "\n");
        fin = TRUE;
        for(j = 0; j < cblistnum(lines); j++){
          line = cblistval(lines, j, NULL);
          if(line[0] != '\0'){
            word = cbmemdup(line, -1);
            if((pv = strchr(word, '\t')) != NULL) *pv = '\0';
            printf("%s", word);
            free(word);
            fin = TRUE;
          } else if(fin){
            printf(" ... ");
            fin = FALSE;
          }
        }
        cblistclose(lines);
        free(draft);
        cblistclose(words);
        printf("\n\n");
        est_doc_delete(doc);
      }
      break;
    case VM_XML:
      if((doc = est_db_get_doc(db, id, 0)) != NULL){
        if(sc >= 0){
          sprintf(numbuf, "%d", sc);
          est_doc_add_attr(doc, DATTRSCORE, numbuf);
        }
        if(!(vbuf = est_doc_attr(doc, ESTDATTRURI))) vbuf = "";
        xmlprintf("<document id=\"%d\" uri=\"%@\">\n", id, vbuf);
        names = est_doc_attr_names(doc);
        for(j = 0; j < cblistnum(names); j++){
          kbuf = cblistval(names, j, NULL);
          if(!strcmp(kbuf, ESTDATTRID) || !strcmp(kbuf, ESTDATTRURI)) continue;
          vbuf = est_doc_attr(doc, kbuf);
          xmlprintf("<attribute name=\"%@\" value=\"%@\"/>\n", kbuf, vbuf);
        }
        cblistclose(names);
        kwords = kwdb ? vectorizer(db, id, kwdb) : NULL;
        if(!kwords) kwords = est_db_etch_doc(db, doc, KWORDNUM);
        if(cbmaprnum(kwords) > 0){
          xmlprintf("<vector>");
          cbmapiterinit(kwords);
          for(j = 0; (kbuf = cbmapiternext(kwords, NULL)) != NULL; j++){
            xmlprintf("<element key=\"%@\" number=\"%@\"/>",
                      kbuf, cbmapget(kwords, kbuf, -1, NULL));
          }
          xmlprintf("</vector>\n");
        }
        cbmapclose(kwords);
        words = cbmapkeys(hints);
        draft = est_doc_make_snippet(doc, words, SNIPWWIDTH, SNIPHWIDTH, SNIPAWIDTH);
        lines = cbsplit(draft, -1, "\n");
        fin = TRUE;
        xmlprintf("<snippet>");
        for(j = 0; j < cblistnum(lines); j++){
          line = cblistval(lines, j, NULL);
          if(line[0] != '\0'){
            word = cbmemdup(line, -1);
            if((pv = strchr(word, '\t')) != NULL){
              *pv = '\0';
              pv++;
              xmlprintf("<key normal=\"%@\">%@</key>", pv, word);
            } else {
              xmlprintf("%@", word);
            }
            free(word);
            fin = TRUE;
          } else if(fin){
            xmlprintf("<delimiter/>");
            fin = FALSE;
          }
        }
        xmlprintf("</snippet>\n");
        cblistclose(lines);
        free(draft);
        cblistclose(words);
        xmlprintf("</document>\n");
        est_doc_delete(doc);
      }
      break;
    case VM_DUMP:
      if((doc = est_db_get_doc(db, id, 0)) != NULL){
        if(sc >= 0){
          sprintf(numbuf, "%d", sc);
          est_doc_add_attr(doc, DATTRSCORE, numbuf);
        }
        if(!(vbuf = est_doc_attr(doc, ESTDATTRURI))) vbuf = "";
        sprintf(path, "%08d%cest", id, ESTEXTCHR);
        printf("%s\t%s\n", path, vbuf);
        draft = est_doc_dump_draft(doc);
        if(!(cbwritefile(path, draft, -1))) printferror("%s: could not open", path);
        free(draft);
        est_doc_delete(doc);
      }
      break;
    default:
      printf("%d\n", id);
      break;
    }
  }
  if(g_viewmode == VM_XML){
    xmlprintf("</estresult>\n");
  } else {
    printf("%s:END\n", est_border_str());
  }
  free(res);
  cbmapclose(hints);
  est_cond_delete(cond);
  if(kwdb) crclose(kwdb);
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  return 0;
}


/* perform the gather command */
static int procgather(const char *dbname, const char *filename){
  ESTDB *db;
  CBLIST *list, *clist, *attrs;
  FILE *ifp;
  const char *tmp;
  char *line, *path;
  int i, err, ecode;
  time_t curtime;
  struct stat sbuf;
  curtime = time(NULL);
  err = FALSE;
  if(stat(filename, &sbuf) != -1 && S_ISDIR(sbuf.st_mode)){
    printfinfo("reading list from the directory: %s", filename);
    if((db = est_db_open(dbname, ESTDBWRITER | ESTDBCREAT | g_oextmodes, &ecode)) != NULL){
      est_db_set_informer(db, dbinform);
      if(g_cachesize > 0){
        if(g_cachesize > CACHEMAX) g_cachesize = CACHEMAX;
        est_db_set_cache_size(db, g_cachesize, -1, -1);
      }
      list = cblistopen();
      cblistunshift(list, filename, -1);
      while((line = cblistshift(list, NULL)) != NULL){
        if(stat(line, &sbuf) != -1 && S_ISDIR(sbuf.st_mode) && (clist = cbdirlist(line)) != NULL){
          cblistsort(clist);
          for(i = cblistnum(clist) - 1; i >= 0; i--){
            tmp = cblistval(clist, i, NULL);
            if(!strcmp(tmp, ESTCDIRSTR) || !strcmp(tmp, ESTPDIRSTR)) continue;
            path = cbsprintf("%s%c%s", line, ESTPATHCHR, tmp);
            cblistunshift(list, path, -1);
            free(path);
          }
          cblistclose(clist);
        } else {
          if(!doputdoc(db, line, NULL)){
            printferror("%s: %s", line, est_err_msg(est_db_error(db)));
            err = TRUE;
          }
        }
        free(line);
        if(err || g_sigterm) break;
      }
      cblistclose(list);
      if(!est_db_close(db, &ecode)){
        printferror("%s: %s", dbname, est_err_msg(ecode));
        err = TRUE;
      }
    } else {
      printferror("%s: %s", dbname, est_err_msg(ecode));
      err = TRUE;
    }
  } else {
    if(!strcmp(filename, "-")){
      ifp = stdin;
      printfinfo("reading list from the standard input", filename);
    } else if((ifp = fopen(filename, "rb")) != NULL){
      printfinfo("reading list from the file: %s", filename);
    } else {
      printferror("%s: could not open", filename);
      return 1;
    }
    if((db = est_db_open(dbname, ESTDBWRITER | ESTDBCREAT | g_oextmodes, &ecode)) != NULL){
      est_db_set_informer(db, dbinform);
      if(g_cachesize > 0){
        if(g_cachesize > CACHEMAX) g_cachesize = CACHEMAX;
        est_db_set_cache_size(db, g_cachesize, -1, -1);
      }
      while((line = fgetl(ifp)) != NULL){
        if(line[0] == '\0'){
          free(line);
          continue;
        }
        if(cblistnum(g_pathattrs) > 0){
          attrs = cbsplit(line, -1, "\t");
          path = cblistshift(attrs, NULL);
          if(!doputdoc(db, path, attrs)){
            printferror("%s: %s", path, est_err_msg(est_db_error(db)));
            err = TRUE;
          }
          free(path);
          cblistclose(attrs);
        } else {
          if(!doputdoc(db, line, NULL)){
            printferror("%s: %s", line, est_err_msg(est_db_error(db)));
            err = TRUE;
          }
        }
        free(line);
        if(err || g_sigterm) break;
      }
      if(!est_db_close(db, &ecode)){
        printferror("%s: %s", dbname, est_err_msg(ecode));
        err = TRUE;
      }
    } else {
      printferror("%s: %s", dbname, est_err_msg(ecode));
      err = TRUE;
    }
    if(ifp != stdin) fclose(ifp);
  }
  curtime = time(NULL) - curtime;
  if(!err) printfinfo("finished successfully: elapsed time: %dh %dm %ds",
                      (int)(curtime / 3600), (int)((curtime / 60) % 60), (int)(curtime % 60));
  return err ? 1 : 0;
}


/* perform the purge command */
static int procpurge(const char *dbname, const char *prefix){
  ESTDB *db;
  ESTCOND *cond;
  ESTDOC *doc;
  const char *luri;
  char *attr, *path;
  int i, ecode, err, *res, rnum;
  time_t curtime;
  struct stat sbuf;
  curtime = time(NULL);
  if(!(db = est_db_open(dbname, ESTDBWRITER, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  est_db_set_informer(db, dbinform);
  cond = est_cond_new();
  attr = cbsprintf("%s STRBW %s", DATTRLPATH, prefix ? prefix : "");
  est_cond_add_attr(cond, attr);
  res = est_db_search(db, cond, &rnum, NULL);
  err = FALSE;
  for(i = 0; i < rnum; i++){
    if(!(doc = est_db_get_doc(db, res[i], ESTGDNOTEXT))) continue;
    if((luri = est_doc_attr(doc, DATTRLPATH)) != NULL){
      if(g_doforce){
        if(est_db_out_doc(db, res[i], g_outopts)){
          printfinfo("%d (%s): deleted", res[i], luri);
        } else {
          printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
          err = TRUE;
        }
      } else if((path = urltopath(luri)) != NULL){
        if(stat(path, &sbuf) != -1){
          printfinfo("%s: passed", luri);
        } else {
          if(est_db_out_doc(db, res[i], g_outopts)){
            printfinfo("%d (%s): deleted", res[i], luri);
          } else {
            printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
            err = TRUE;
          }
        }
      } else {
        printfinfo("%s: ignored", luri);
      }
    } else {
      printfinfo("(%d): ignored", res[i]);
    }
    est_doc_delete(doc);
    if(err || g_sigterm) break;
  }
  free(res);
  est_cond_delete(cond);
  free(attr);
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  curtime = time(NULL) - curtime;
  if(!err) printfinfo("finished successfully: elapsed time: %dh %dm %ds",
                      (int)(curtime / 3600), (int)((curtime / 60) % 60), (int)(curtime % 60));
  return err ? 1 : 0;
}


/* perform the extkeys command */
static int procextkeys(const char *dbname, const char *prefix, int ni){
  ESTDB *db;
  ESTCOND *cond;
  ESTDOC *doc;
  CURIA *kwdb;
  CBMAP *kwords;
  const char *uri;
  char path[URIBUFSIZ], *attr, *mbuf;
  int i, ecode, err, *res, rnum, msiz;
  time_t curtime;
  curtime = time(NULL);
  if(!(db = est_db_open(dbname, ESTDBWRITER, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  est_db_set_informer(db, dbinform);
  if(!ni && (!prefix || prefix[0] == '\0')) est_db_fill_key_cache(db);
  sprintf(path, "%s%c%s", dbname, ESTPATHCHR, KWDBNAME);
  if(!(kwdb = cropen(path, CR_OWRITER | CR_OCREAT, KWDBBNUM, KWDBDNUM))){
    printferror("%s: the keyword database has some errors", dbname);
    est_db_close(db, &ecode);
    return 1;
  }
  crsetalign(kwdb, -4);
  cond = est_cond_new();
  attr = cbsprintf("%s STRBW %s", DATTRLPATH, prefix ? prefix : "");
  est_cond_add_attr(cond, attr);
  res = est_db_search(db, cond, &rnum, NULL);
  err = FALSE;
  for(i = 0; i < rnum; i++){
    if(!g_doforce && crvsiz(kwdb, (char *)&(res[i]), sizeof(int)) > 0){
      printfinfo("%d: passed", res[i]);
      continue;
    }
    if(!(doc = est_db_get_doc(db, res[i], 0))) continue;
    if(!(uri = est_doc_attr(doc, ESTDATTRURI))) uri = "";
    kwords = est_db_etch_doc(ni ? NULL : db, doc, g_kwordnum);
    mbuf = cbmapdump(kwords, &msiz);
    fflush(stdout);
    if(crput(kwdb, (char *)&(res[i]), sizeof(int), mbuf, msiz, CR_DOVER)){
      printfinfo("%d (%s): extracted", res[i], uri);
    } else {
      printferror("%s: the keyword database has some errors", dbname);
      err = TRUE;
    }
    free(mbuf);
    cbmapclose(kwords);
    est_doc_delete(doc);
    if(err || g_sigterm) break;
  }
  free(res);
  est_cond_delete(cond);
  free(attr);
  if(!crclose(kwdb)){
    printferror("%s: the keyword database has some errors", dbname);
    err = TRUE;
  }
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  curtime = time(NULL) - curtime;
  if(!err) printfinfo("finished successfully: elapsed time: %dh %dm %ds",
                      (int)(curtime / 3600), (int)((curtime / 60) % 60), (int)(curtime % 60));
  return err ? 1 : 0;
}


/* perform the draft command */
static int procdraft(const char *filename){
  ESTDOC *doc;
  char *buf, *draft;
  int size;
  if(!(buf = cbreadfile(filename, &size))){
    printferror("%s: could not open", filename ? filename : "(stdin)");
    return 1;
  }
  switch(g_filefmt){
  case FF_TEXT:
    doc = est_doc_new_from_text(buf, size, g_inputcode, g_inputlang);
    break;
  case FF_HTML:
    doc = est_doc_new_from_html(buf, size, g_inputcode, g_inputlang);
    break;
  case FF_MIME:
    doc = est_doc_new_from_mime(buf, size, g_inputcode, g_inputlang);
    break;
  default:
    doc = est_doc_new_from_draft_enc(buf, size, g_inputcode);
    break;
  }
  draft = est_doc_dump_draft(doc);
  printf("%s", draft);
  free(draft);
  est_doc_delete(doc);
  free(buf);
  return 0;
}


/* perform the break command */
static int procbreak(const char *filename, int wt){
  CBLIST *words;
  char *str, *phrase;
  int i;
  if(filename && filename[0] == '@'){
    str = cbmemdup(filename + 1, -1);
  } else if(!(str = cbreadfile(filename, NULL))){
    printferror("%s: could not open", filename ? filename : "(stdin)");
    return 1;
  }
  if(!(phrase = est_iconv(str, -1, g_inputcode, "UTF-8", NULL, NULL))){
    printferror("%s: unsupported encoding\n", g_inputcode);
    free(str);
    return 1;
  }
  g_inputcode = NULL;
  words = cblistopen();
  if(g_oextmodes & ESTDBPERFNG){
    est_break_text_perfng(phrase, words, TRUE, wt);
  } else {
    est_break_text(phrase, words, TRUE, wt);
  }
  for(i = 0; i < cblistnum(words); i++){
    printf("%s\n", cblistval(words, i, NULL));
  }
  cblistclose(words);
  free(phrase);
  free(str);
  return 0;
}


/* perform the randput command */
static int procrandput(const char *dbname, int dnum){
  ESTDB *db;
  ESTDOC *doc;
  const char *mode;
  char uri[URIBUFSIZ];
  int i, ecode, err;
  time_t curtime;
  curtime = time(NULL);
  if(!(db = est_db_open(dbname, ESTDBWRITER | ESTDBCREAT | ESTDBTRUNC, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  est_db_set_informer(db, dbinform);
  if(g_cachesize > 0){
    if(g_cachesize > CACHEMAX) g_cachesize = CACHEMAX;
    est_db_set_cache_size(db, g_cachesize, -1, -1);
  }
  err = FALSE;
  for(i = 0; i < dnum; i++){
    doc = est_doc_new_from_chaos(RDOCCNUM, RDOCSNUM, g_rdmode);
    sprintf(uri, "file:///tmp/randput-%08d-%05d.est", i + 1, getpid());
    est_doc_add_attr(doc, ESTDATTRURI, uri);
    if(est_db_put_doc(db, doc, 0)){
      if(!(mode = est_doc_attr(doc, "mode"))) mode = "unknown";
      printfinfo("%d (%s) (%s): registered", est_doc_id(doc), uri, mode);
    } else {
      printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
      err = TRUE;
    }
    est_doc_delete(doc);
    if(err || g_sigterm) break;
  }
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  curtime = time(NULL) - curtime;
  if(!err) printfinfo("finished successfully: elapsed time: %dh %dm %ds",
                      (int)(curtime / 3600), (int)((curtime / 60) % 60), (int)(curtime % 60));
  return err ? 1 : 0;
}


/* perform the wicked command */
static int procwicked(const char *dbname, int dnum){
  ESTDB *db;
  ESTDOC *doc;
  ESTCOND *cond;
  CBLIST *words;
  char uri[URIBUFSIZ], *oper, *value, *first, *second, *phrase;
  int i, j, ecode, err, *res, rnum;
  double rnd;
  time_t curtime;
  curtime = time(NULL);
  if(!(db = est_db_open(dbname, ESTDBWRITER | ESTDBCREAT | ESTDBTRUNC, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  est_db_set_informer(db, dbinform);
  est_db_set_cache_size(db, 1024 * 1024 * 128, 1024, 256);
  est_db_set_special_cache(db, ESTDATTRURI, 128);
  err = FALSE;
  for(i = 0; i < dnum; i++){
    rnd = est_random();
    if((int)(rnd * INT_MAX) % dnum < 5){
      rnd = est_random();
      if(rnd < 0.3){
        if(!est_db_close(db, &ecode)){
          printferror("%s: %s", dbname, est_err_msg(ecode));
          return 1;
        }
        if(!(db = est_db_open(dbname, ESTDBWRITER, &ecode))){
          printferror("%s: %s", dbname, est_err_msg(ecode));
          return 1;
        }
        est_db_set_informer(db, dbinform);
        est_db_set_cache_size(db, 1024 * 1024 * 128, 1024, 256);
        est_db_set_special_cache(db, ESTDATTRURI, i / 10 + 1);
      } else if(rnd < 0.5){
        if(!est_db_optimize(db, (int)(est_random() * INT_MAX) % 2 == 0) ? ESTOPTNOPURGE : 0)
          err = TRUE;
      } else if(rnd < 0.8){
        if(!est_db_flush(db, 1024)) err = TRUE;
      } else {
        if(!est_db_sync(db)) err = TRUE;
      }
    } else if(rnd < 0.05){
      if(est_db_out_doc(db, (int)(est_random() * INT_MAX) % (i + 1) + 1,
                        ((int)(est_random() * INT_MAX) % 2 == 0) ? ESTODCLEAN : 0)){
        printfinfo("[%d:%d]: out", i + 1, est_db_doc_num(db));
      } else if(est_db_error(db) != ESTENOITEM){
        err = TRUE;
      }
    } else if(rnd < 0.1){
      if((value = est_db_get_doc_attr(db, (int)(est_random() * INT_MAX) % (i + 1) + 1,
                                      ESTDATTRURI)) != NULL){
        printfinfo("[%d:%d]: attr: %s", i + 1, est_db_doc_num(db), value);
        free(value);
      }
    } else if(rnd < 0.25){
      rnd = est_random();
      if(rnd < 0.5){
        oper = " OR ";
      } else if(rnd < 0.7){
        oper = " AND ";
      } else if(rnd < 0.8){
        oper = " NOTAND ";
      } else if(rnd < 0.9){
        oper = " ";
      } else {
        oper = "";
      }
      first = est_random_str(5, (int)(est_random() * INT_MAX) % RD_RAND);
      second = est_random_str(2, (int)(est_random() * INT_MAX) % RD_RAND);
      phrase = cbsprintf("%s%s%s", first, oper, second);
      cond = est_cond_new();
      est_cond_set_phrase(cond, phrase);
      if(est_random() < 0.25) est_cond_add_attr(cond, "@uri STREW 0.est");
      if(est_random() < 0.25) est_cond_set_order(cond, "@uri STRD");
      if(est_random() < 0.05) est_cond_set_options(cond, ESTCONDSURE | ESTCONDSCFB);
      if(est_random() < 0.05) est_cond_set_options(cond, ESTCONDAGIT | ESTCONDNOIDF);
      res = est_db_search(db, cond, &rnum, NULL);
      printfinfo("[%d:%d]: search: %d hits", i + 1, est_db_doc_num(db), rnum);
      if(est_random() < 0.05){
        for(j = 0; j < rnum && j < 100; j++){
          if((doc = est_db_get_doc(db, res[j], 0)) != NULL){
            if(i % 10 == 0){
              free(est_doc_cat_texts(doc));
              free(est_doc_dump_draft(doc));
              words = cblistopen();
              cblistpush(words, "vw", -1);
              cblistpush(words, "xy", -1);
              cblistpush(words, "z", -1);
              free(est_doc_make_snippet(doc, words, 100, 10, 10));
              cblistclose(words);
            }
            est_doc_delete(doc);
          } else if(est_db_error(db) != ESTENOITEM){
            err = TRUE;
          }
        }
      }
      free(res);
      est_cond_delete(cond);
      free(phrase);
      free(first);
      free(second);
    } else {
      doc = est_doc_new_from_chaos(100, 3, est_random() < 0.5 ? RD_EURO : RD_RAND);
      if(est_random() < 0.2){
        sprintf(uri, "file:///tmp/wicked-%08d-%05d.est",
                (int)(est_random() * INT_MAX) % (i + 1) + 1, getpid());
      } else {
        sprintf(uri, "file:///tmp/wicked-%08d-%05d.est", i + 1, getpid());
      }
      est_doc_add_attr(doc, ESTDATTRURI, uri);
      if(!est_db_put_doc(db, doc, est_random() < 0.5 ? ESTPDCLEAN : 0)) err = TRUE;
      est_doc_delete(doc);
    }
    if(err || g_sigterm) break;
  }
  if(err) printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  curtime = time(NULL) - curtime;
  if(!err) printfinfo("finished successfully: elapsed time: %dh %dm %ds",
                      (int)(curtime / 3600), (int)((curtime / 60) % 60), (int)(curtime % 60));
  return err ? 1 : 0;
}


/* perform the regression command */
static int procregression(const char *dbname){
  ESTDB *db;
  ESTDOC *doc;
  ESTCOND *cond;
  int i, ecode, err, *res, rnum;
  time_t curtime;
  curtime = time(NULL);
  printfinfo("# opening the database");
  if(!(db = est_db_open(dbname, ESTDBWRITER | ESTDBCREAT | ESTDBTRUNC, &ecode))){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  est_db_set_informer(db, dbinform);
  err = FALSE;
  if(!err){
    printfinfo("# checking registration of small documents");
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///small/one");
    est_doc_add_text(doc, "One!");
    est_doc_add_hidden_text(doc, "(Check it out, come on!)");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///small/two");
    est_doc_add_text(doc, "Two!!");
    est_doc_add_hidden_text(doc, "(Check it out, come on!)");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///small/three");
    est_doc_add_text(doc, "Three!!!");
    est_doc_add_hidden_text(doc, "(Check it out, come on!)");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///empty");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
  }
  if(!err){
    printfinfo("# checking registration of an english document");
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///english");
    est_doc_add_attr(doc, ESTDATTRTITLE, "Hyper Estraier");
    est_doc_add_text(doc, "% This is a displayed sentence. ;-)");
    est_doc_add_text(doc, "Hyper Estraier is a full-text search system for communities.");
    est_doc_add_text(doc, "A little suffering is good for the soul.");
    est_doc_add_text(doc, "They have been at a great feast of languages, and stolen the scraps.");
    est_doc_add_hidden_text(doc, "(Give it up, Yo!  Give it up, Yo!)");
    est_doc_add_hidden_text(doc, "% This is a hidden sentence. :-<");
    est_doc_add_hidden_text(doc, "(Check it out, come on!)");
    est_doc_add_hidden_text(doc, "");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
  }
  if(!err){
    printfinfo("# checking registration of a japanese document");
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///japanese");
    est_doc_add_attr(doc, ESTDATTRTITLE, "\xe5\xb9\xb3\xe6\x9e\x97\xe5\xb9\xb9\xe9\x9b\x84");
    est_doc_add_text(doc, "\xe6\x9c\xac\xe6\x97\xa5\xe3\x81\xaf\xe6\x99\xb4\xe5\xa4\xa9\xe3"
                     "\x81\xaa\xe3\x82\x8a\xe3\x80\x82");
    est_doc_add_text(doc, "\xe6\x9c\x95\xe3\x81\xaf\xe5\x9b\xbd\xe5\xae\xb6\xe7\xac\xac\xe4"
                     "\xb8\x80\xe3\x81\xae\xe4\xb8\x8b\xe5\x83\x95\xe3\x81\xa7\xe3\x81"
                     "\x82\xe3\x82\x8b\xe3\x80\x82");
    est_doc_add_hidden_text(doc, "(Check it out, come on!)");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
  }
  if(!err){
    printfinfo("# checking duplication of documents");
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///duplication");
    est_doc_add_text(doc, "Gamble, you gatta chance to make a Rumble!");
    est_doc_add_hidden_text(doc, "(Check it out, come on!)");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
    doc = est_doc_new();
    est_doc_add_attr(doc, ESTDATTRURI, "file:///duplication");
    est_doc_add_text(doc, "bring back hey, one more time!");
    est_doc_add_hidden_text(doc, "(Check it out, come on!)");
    if(!est_db_put_doc(db, doc, ESTPDCLEAN)) err = TRUE;
    est_doc_delete(doc);
    if(est_db_doc_num(db) != 7){
      printferror("%s: the number of documents is invalid", dbname);
      err = TRUE;
    }
  }
  if(!err){
    printfinfo("# checking search for unfixed documents");
    cond = est_cond_new();
    est_cond_set_phrase(cond, "check");
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != 6){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_delete(cond);
  }
  if(!err){
    printfinfo("# checking partial flushing of the index");
    if(!est_db_flush(db, 32)) err = TRUE;
  }
  if(!err){
    printfinfo("# checking deletion with cleaning of a document");
    if(!est_db_out_doc(db, 1, ESTODCLEAN)) err = TRUE;
  }
  if(!err){
    printfinfo("# checking synchronization");
    if(!est_db_sync(db)) err = TRUE;
  }
  if(!err){
    printfinfo("# checking deletion without cleaning of a document");
    if(!est_db_out_doc(db, 2, 0)) err = TRUE;
  }
  if(!err){
    printfinfo("# checking word search");
    cond = est_cond_new();
    est_cond_set_phrase(cond, "check it AND on");
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != 5){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_set_phrase(cond, "RUMBLE OR \xe3\x80\x82");
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != 1){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_delete(cond);
  }
  if(!err){
    printfinfo("# checking attribute search");
    cond = est_cond_new();
    est_cond_add_attr(cond, "@uri !ISTRINC SMaLl");
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != est_db_doc_num(db) - 1){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_delete(cond);
    cond = est_cond_new();
    est_cond_add_attr(cond, "@uri STRBW file://");
    est_cond_add_attr(cond, "@title STRINC \xe5\xb9\xb3");
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != 1){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_delete(cond);
  }
  if(!err){
    printfinfo("# checking combined search");
    cond = est_cond_new();
    est_cond_set_phrase(cond, "\xe5\x9b\xbd\xe5\xae\xb6\xe7\xac\xac\xe4\xb8\x80");
    est_cond_add_attr(cond, "@uri");
    est_cond_set_order(cond, "@title");
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != 1){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_delete(cond);
    cond = est_cond_new();
    est_cond_set_phrase(cond, "one | \xe3\x80\x82 | check & check it ! hogehoge");
    est_cond_add_attr(cond, "@uri STRBW file://");
    est_cond_set_order(cond, "@title STRD");
    est_cond_set_options(cond, ESTCONDSURE | ESTCONDNOIDF | ESTCONDSIMPLE);
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != 4){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_delete(cond);
  }
  if(!err){
    printfinfo("# checking optimization");
    if(!est_db_optimize(db, 0)) err = TRUE;
    cond = est_cond_new();
    est_cond_set_phrase(cond, "check");
    res = est_db_search(db, cond, &rnum, NULL);
    if(rnum != 4){
      printferror("%s: the number of result is invalid", dbname);
      err = TRUE;
    }
    free(res);
    est_cond_delete(cond);
  }
  if(!err){
    printfinfo("# checking traversal access");
    cond = est_cond_new();
    est_cond_set_phrase(cond, "[UVSET]");
    res = est_db_search(db, cond, &rnum, NULL);
    for(i = 0; i < rnum; i++){
      if(!(doc = est_db_get_doc(db, res[i], 0))){
        printferror("%s: a document cannot be retrieved", dbname);
        err = TRUE;
        break;
      }
      est_doc_delete(doc);
    }
    free(res);
    est_cond_delete(cond);
  }
  if(err) printferror("%s: %s", dbname, est_err_msg(est_db_error(db)));
  printfinfo("# closing the database");
  if(!est_db_close(db, &ecode)){
    printferror("%s: %s", dbname, est_err_msg(ecode));
    return 1;
  }
  curtime = time(NULL) - curtime;
  if(!err) printfinfo("# finished successfully: elapsed time: %dh %dm %ds",
                      (int)(curtime / 3600), (int)((curtime / 60) % 60), (int)(curtime % 60));
  return err ? 1 : 0;
}


/* output escaped string */
static void xmlprintf(const char *format, ...){
  va_list ap;
  char *tmp, cbuf[32];
  unsigned char c;
  int cblen;
  va_start(ap, format);
  while(*format != '\0'){
    if(*format == '%'){
      cbuf[0] = '%';
      cblen = 1;
      format++;
      while(strchr("0123456789 .+-", *format) && *format != '\0' && cblen < 31){
        cbuf[cblen++] = *format;
        format++;
      }
      cbuf[cblen++] = *format;
      cbuf[cblen] = '\0';
      switch(*format){
      case 's':
        tmp = va_arg(ap, char *);
        if(!tmp) tmp = "(null)";
        printf(cbuf, tmp);
        break;
      case 'd':
        printf(cbuf, va_arg(ap, int));
        break;
      case 'o': case 'u': case 'x': case 'X': case 'c':
        printf(cbuf, va_arg(ap, unsigned int));
        break;
      case 'e': case 'E': case 'f': case 'g': case 'G':
        printf(cbuf, va_arg(ap, double));
        break;
      case '@':
        tmp = va_arg(ap, char *);
        if(!tmp) tmp = "(null)";
        while(*tmp){
          switch(*tmp){
          case '&': printf("&amp;"); break;
          case '<': printf("&lt;"); break;
          case '>': printf("&gt;"); break;
          case '"': printf("&quot;"); break;
          default:
            if(!((*tmp >= 0 && *tmp <= 0x8) || (*tmp >= 0x0e && *tmp <= 0x1f))) putchar(*tmp);
            break;
          }
          tmp++;
        }
        break;
      case '?':
        tmp = va_arg(ap, char *);
        if(!tmp) tmp = "(null)";
        while(*tmp){
          c = *(unsigned char *)tmp;
          if((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
             (c >= '0' && c <= '9') || (c != '\0' && strchr("_-.", c))){
            putchar(c);
          } else {
            printf("%%%02X", c);
          }
          tmp++;
        }
        break;
      case '%':
        putchar('%');
        break;
      }
    } else {
      putchar(*format);
    }
    format++;
  }
  va_end(ap);
}


/* get the language value */
static int strtolang(const char *str){
  if(!cbstricmp(str, "en")) return ESTLANGEN;
  if(!cbstricmp(str, "ja")) return ESTLANGJA;
  if(!cbstricmp(str, "zh")) return ESTLANGZH;
  if(!cbstricmp(str, "ko")) return ESTLANGKO;
  return ESTLANGMISC;
}


/* read a line */
static char *fgetl(FILE *ifp){
  char *buf;
  int c, len, blen;
  buf = NULL;
  len = 0;
  blen = 1024;
  while((c = fgetc(ifp)) != EOF){
    if(blen <= len) blen *= 2;
    buf = cbrealloc(buf, blen + 1);
    if(c == '\n') c = '\0';
    if(c != '\r') buf[len++] = c;
    if(c == '\0') break;
  }
  if(!buf) return NULL;
  buf[len] = '\0';
  return buf;
}


/* register a document */
static int doputdoc(ESTDB *db, const char *path, const CBLIST *attrs){
  ESTDOC *doc, *edoc;
  const char *uri, *vbuf, *xcmd;
  char *dbuf, *tbuf;
  int i, err, fmt, id, dsiz;
  time_t emdate, fmdate;
  struct stat sbuf;
  xcmd = NULL;
  if(cbmaprnum(g_xcmdmap) > 0){
    cbmapiterinit(g_xcmdmap);
    while((vbuf = cbmapiternext(g_xcmdmap, NULL)) != NULL){
      if(cbstrbwimatch(path, vbuf)){
        xcmd = cbmapget(g_xcmdmap, vbuf, -1, NULL);
        break;
      }
    }
  }
  fmt = g_filefmt;
  if(g_filefmt == FF_NONE && !xcmd) return TRUE;
  if(g_filefmt == FF_AUTO){
    if(cbstrbwimatch(path, ESTEXTSTR "est")){
      fmt = FF_DRAFT;
    } else if(cbstrbwimatch(path, ESTEXTSTR "txt") || cbstrbwimatch(path, ESTEXTSTR "text") ||
              cbstrbwimatch(path, ESTEXTSTR "asc")){
      fmt = FF_TEXT;
    } else if(cbstrbwimatch(path, ESTEXTSTR "html") || cbstrbwimatch(path, ESTEXTSTR "htm") ||
              cbstrbwimatch(path, ESTEXTSTR "xhtml") || cbstrbwimatch(path, ESTEXTSTR "xht")){
      fmt = FF_HTML;
    } else if(cbstrbwimatch(path, ESTEXTSTR "eml") || cbstrbwimatch(path, ESTEXTSTR "mime") ||
              cbstrbwimatch(path, ESTEXTSTR "mht") || cbstrbwimatch(path, ESTEXTSTR "mhtml")){
      fmt = FF_MIME;
    } else if(!xcmd){
      return TRUE;
    }
  }
  if(stat(path, &sbuf) == -1 || !S_ISREG(sbuf.st_mode) || !(uri = pathtourl(path))){
    printferror("%s: could not open", path);
    return TRUE;
  }
  emdate = -1;
  if(g_chkmdate && (id = est_db_uri_to_id(db, uri)) > 0 &&
     (edoc = est_db_get_doc(db, id, ESTGDNOTEXT)) != NULL){
    if((vbuf = est_doc_attr(edoc, ESTDATTRMDATE)) != NULL) emdate = cbstrmktime(vbuf);
    est_doc_delete(edoc);
  }
  if(g_stdate && emdate >= 0 && emdate >= sbuf.st_mtime){
    printfinfo("%s: passed", path);
    return TRUE;
  }
  if(g_filtorig){
    dbuf = cbmemdup("", 0);
    dsiz = 0;
  } else {
    if(!(dbuf = cbreadfile(path, &dsiz))){
      printferror("%s: could not open", path);
      return TRUE;
    }
  }
  if(xcmd){
    doc = est_doc_new_with_xcmd(dbuf, dsiz, path, xcmd, est_db_name(db),
                                g_inputcode, g_inputlang);
  } else {
    switch(fmt){
    case FF_TEXT:
      doc = est_doc_new_from_text(dbuf, dsiz, g_inputcode, g_inputlang);
      break;
    case FF_HTML:
      doc = est_doc_new_from_html(dbuf, dsiz, g_inputcode, g_inputlang);
      break;
    case FF_MIME:
      doc = est_doc_new_from_mime(dbuf, dsiz, g_inputcode, g_inputlang);
      break;
    default:
      doc = est_doc_new_from_draft_enc(dbuf, dsiz, g_inputcode);
      break;
    }
  }
  if(attrs){
    for(i = 0; i < cblistnum(g_pathattrs) && i < cblistnum(attrs); i++){
      est_doc_add_attr(doc, cblistval(g_pathattrs, i, NULL), cblistval(attrs, i, NULL));
    }
  }
  if(!est_doc_attr(doc, ESTDATTRURI)) est_doc_add_attr(doc, ESTDATTRURI, uri);
  est_doc_add_attr(doc, DATTRLPATH, uri);
  est_doc_add_attr(doc, DATTRLFILE, urltofile(uri));
  uri = est_doc_attr(doc, ESTDATTRURI);
  if(g_stdate){
    tbuf = cbdatestrwww(sbuf.st_ctime, 0);
    est_doc_add_attr(doc, ESTDATTRCDATE, tbuf);
    free(tbuf);
    tbuf = cbdatestrwww(sbuf.st_mtime, 0);
    est_doc_add_attr(doc, ESTDATTRMDATE, tbuf);
    free(tbuf);
  }
  if(g_chkmdate && emdate == -1 && (id = est_db_uri_to_id(db, uri)) > 0 &&
     (edoc = est_db_get_doc(db, id, ESTGDNOTEXT)) != NULL){
    if((vbuf = est_doc_attr(edoc, ESTDATTRMDATE)) != NULL) emdate = cbstrmktime(vbuf);
    est_doc_delete(edoc);
  }
  fmdate = -1;
  if(g_chkmdate && (vbuf = est_doc_attr(doc, ESTDATTRMDATE)) != NULL) fmdate = cbstrmktime(vbuf);
  err = FALSE;
  if(emdate >= 0 && emdate >= fmdate){
    printfinfo("%s: passed", path);
  } else if(est_db_put_doc(db, doc, g_putopts)){
    printfinfo("%d (%s): registered", est_doc_id(doc), uri);
  } else {
    printferror("%s: %s", est_db_name(db), est_err_msg(est_db_error(db)));
    err = TRUE;
  }
  est_doc_delete(doc);
  free(dbuf);
  return err ? FALSE : TRUE;
}


/* get the URL of a path */
static const char *pathtourl(const char *path){
  static char pbuf[URIBUFSIZ];
  const char *elem;
  char *wp, *ebuf;
  CBLIST *list;
  int i, esiz;
  if(strlen(path) >= URIBUFSIZ / 4) return NULL;
  if(g_pathcode){
    wp = est_realpath(path);
    if(!(ebuf = est_iconv(wp, -1, g_pathcode, "UTF-8", &esiz, NULL))){
      esiz = strlen(wp);
      ebuf = cbmemdup(wp, esiz);
    }
    list = cbsplit(ebuf, esiz, ESTPATHSTR);
    free(ebuf);
    free(wp);
    for(i = 0; i < cblistnum(list); i++){
      elem = cblistval(list, i, &esiz);
      if((ebuf = est_iconv(elem, esiz, "UTF-8", g_pathcode, &esiz, NULL)) != NULL){
        cblistover(list, i, ebuf, esiz);
        free(ebuf);
      }
    }
  } else {
    wp = est_realpath(path);
    list = cbsplit(wp, -1, ESTPATHSTR);
    free(wp);
  }
  wp = pbuf;
  wp += sprintf(wp, "file://");
  for(i = 0; i < cblistnum(list); i++){
    elem = cblistval(list, i, NULL);
    if(elem[0] == '\0') continue;
    if(i < 1 && ((elem[0] >= 'A' && elem[0] <= 'Z') || (elem[0] >= 'a' && elem[0] <= 'z')) &&
       elem[1] == ':'){
      wp += sprintf(wp, "%c|", elem[0]);
      continue;
    }
    ebuf = cburlencode(elem, -1);
    wp += sprintf(wp, "/%s", ebuf);
    free(ebuf);
  }
  *wp = '\0';
  cblistclose(list);
  return pbuf;
}


/* get the file name of a URL */
static const char *urltofile(const char *uri){
  static char pbuf[URIBUFSIZ];
  const char *rp;
  char *dbuf, *ebuf;
  int dsiz;
  if(g_pathfull){
    if((rp = strstr(uri, "//")) != NULL){
      rp += 2;
      if(((rp[0] >= 'A' && rp[0] <= 'Z') || (rp[0] >= 'a' && rp[0] <= 'z')) &&
         rp[1] == '|' && rp[2] == '/') rp += 2;
    } else {
      rp = uri;
    }
  } else if((rp = strrchr(uri, '/')) != NULL){
    rp++;
  } else {
    rp = uri;
  }
  dbuf = cburldecode(rp, &dsiz);
  if((ebuf = est_iconv(dbuf, dsiz, g_pathcode ? g_pathcode : "ISO-8859-1", "UTF-8", NULL, NULL))
     != NULL){
    sprintf(pbuf, "%s", ebuf);
    free(ebuf);
  } else {
    sprintf(pbuf, "%s", rp);
  }
  free(dbuf);
  return pbuf;
}


/* geth the local path of a URL */
static char *urltopath(const char *uri){
  static char pbuf[URIBUFSIZ];
  const char *elem;
  char *wp, *dbuf;
  CBLIST *list;
  int i;
  if(!cbstrfwimatch(uri, "file://")) return NULL;
  if(!(uri = strchr(uri + 7, '/'))) return NULL;
  list = cbsplit(uri, -1, "/");
  wp = pbuf;
  for(i = 0; i < cblistnum(list); i++){
    elem = cblistval(list, i, NULL);
    if(elem[0] == '\0') continue;
    if(i < 1 && ((elem[0] >= 'A' && elem[0] <= 'Z') || (elem[0] >= 'a' && elem[0] <= 'z')) &&
       elem[1] == '|'){
      wp += sprintf(wp, "%c:", elem[0]);
      continue;
    }
    dbuf = cburldecode(elem, NULL);
    wp += sprintf(wp, "%c%s", ESTPATHCHR, dbuf);
    free(dbuf);
  }
  *wp = '\0';
  cblistclose(list);
  return pbuf;
}


/* create a vector of keywords */
static CBMAP *vectorizer(void *db, int id, void *kwdb){
  CBMAP *kwords;
  char *mbuf;
  int msiz;
  if(!(mbuf = crget((CURIA *)kwdb, (char *)&id, sizeof(int), 0, -1, &msiz))) return NULL;
  kwords = cbmapload(mbuf, msiz);
  free(mbuf);
  return kwords;
}


/* create a document object with an outer command */
static ESTDOC *est_doc_new_with_xcmd(const char *buf, int size, const char *path,
                                     const char *xcmd, const char *tmpdir,
                                     const char *penc, int plang){
  ESTDOC *doc;
  const char *pv, *ext;
  char iname[URIBUFSIZ], oname[URIBUFSIZ], ebuf[URIBUFSIZ], cmd[URIBUFSIZ];
  char *rbuf, numbuf[NUMBUFSIZ];
  int fmt, rsiz;
  assert(buf && size >= 0 && path && xcmd && tmpdir);
  sprintf(ebuf, "ESTORIGFILE=%s", path);
  ext = NULL;
  if((pv = strrchr(path, ESTPATHCHR)) != NULL) path = pv;
  if((pv = strrchr(path, ESTEXTCHR)) != NULL) ext = pv;
  if(!ext) ext = "";
  sprintf(iname, "%s%cxcmd-in-%08d%s", tmpdir, ESTPATHCHR, getpid(), ext);
  sprintf(oname, "%s%cxcmd-out-%08d%cest", tmpdir, ESTPATHCHR, getpid(), ESTEXTCHR);
  fmt = FF_DRAFT;
  if(cbstrfwmatch(xcmd, "T@")){
    fmt = FF_TEXT;
    xcmd += 2;
  } else if(cbstrfwmatch(xcmd, "H@")){
    fmt = FF_HTML;
    xcmd += 2;
  } else if(cbstrfwmatch(xcmd, "M@")){
    fmt = FF_MIME;
    xcmd += 2;
  }
  sprintf(cmd, "%s %s %s", xcmd, iname, oname);
  if(!g_filtorig) cbwritefile(iname, buf, size);
  putenv(ebuf);
  system(cmd);
  if((rbuf = cbreadfile(oname, &rsiz)) != NULL){
    switch(fmt){
    case FF_TEXT:
      doc = est_doc_new_from_text(rbuf, rsiz, penc, plang);
      break;
    case FF_HTML:
      doc = est_doc_new_from_html(rbuf, rsiz, penc, plang);
      break;
    case FF_MIME:
      doc = est_doc_new_from_mime(rbuf, rsiz, penc, plang);
      break;
    default:
      doc = est_doc_new_from_draft_enc(rbuf, rsiz, penc);
      break;
    }
    free(rbuf);
  } else {
    doc = est_doc_new();
  }
  if(fmt != FF_DRAFT){
    sprintf(numbuf, "%d", size);
    est_doc_add_attr(doc, ESTDATTRSIZE, numbuf);
    est_doc_add_attr(doc, ESTDATTRTYPE, est_ext_type(ext));
  }
  unlink(oname);
  unlink(iname);
  return doc;
}


/* create a document object from draft data in another encoding */
static ESTDOC *est_doc_new_from_draft_enc(const char *buf, int size, const char *enc){
  ESTDOC *doc;
  char *rbuf;
  assert(buf);
  if(enc && (rbuf = est_iconv(buf, size, enc, "UTF-8", NULL, NULL)) != NULL){
    doc = est_doc_new_from_draft(rbuf);
    free(rbuf);
  } else {
    doc = est_doc_new_from_draft(buf);
  }
  return doc;
}


/* create a document object from plain text */
static ESTDOC *est_doc_new_from_text(const char *buf, int size, const char *penc, int plang){
  ESTDOC *doc;
  CBLIST *lines;
  CBDATUM *datum;
  const char *enc, *text, *line;
  char *nbuf, numbuf[NUMBUFSIZ];
  int i;
  assert(buf);
  doc = est_doc_new();
  enc = penc ? penc : est_enc_name(buf, size, plang);
  if(!strcmp(enc, "UTF-8")){
    nbuf = NULL;
    text = buf;
  } else {
    text = buf;
    nbuf = est_iconv(buf, size, enc, "UTF-8", NULL, NULL);
    if(nbuf) text = nbuf;
  }
  lines = cbsplit(text, -1, "\n");
  datum = cbdatumopen("", 0);
  for(i = 0; i < CB_LISTNUM(lines); i++){
    line = CB_LISTVAL(lines, i, NULL);
    while(*line == ' ' || *line == '\t' || *line == '\r'){
      line++;
    }
    if(line[0] == '\0'){
      est_doc_add_text(doc, CB_DATUMPTR(datum));
      cbdatumsetsize(datum, 0);
    } else {
      cbdatumcat(datum, " ", 1);
      cbdatumcat(datum, line, -1);
    }
  }
  est_doc_add_text(doc, CB_DATUMPTR(datum));
  cbdatumclose(datum);
  cblistclose(lines);
  est_doc_add_attr(doc, ESTDATTRTYPE, "text/plain");
  sprintf(numbuf, "%d", size);
  est_doc_add_attr(doc, ESTDATTRSIZE, numbuf);
  if(nbuf) free(nbuf);
  return doc;
}


/* create a document object from HTML */
static ESTDOC *est_doc_new_from_html(const char *buf, int size, const char *penc, int plang){
  ESTDOC *doc;
  CBLIST *elems;
  CBMAP *attrs;
  CBDATUM *datum;
  const char *enc, *html, *elem, *next, *name, *content;
  char *nbuf, *nenc, *rbuf, *lbuf, numbuf[NUMBUFSIZ];
  int i, esiz;
  assert(buf);
  doc = est_doc_new();
  enc = est_enc_name(buf, size, plang);
  html = NULL;
  nbuf = NULL;
  if(!strcmp(enc, "UTF-16") || !strcmp(enc, "UTF-16BE") || !strcmp(enc, "UTF-16LE")){
    nbuf = est_iconv(buf, size, enc, "UTF-8", NULL, NULL);
  } else if(!strcmp(enc, "US-ASCII")){
    nbuf = NULL;
  } else {
    if((nenc = penc ? cbmemdup(penc, -1) : est_html_enc(buf)) != NULL){
      if(cbstricmp(nenc, "UTF-8")){
        nbuf = est_iconv(buf, size, nenc, "UTF-8", NULL, NULL);
        if(!nbuf) nbuf = est_iconv(buf, size, enc, "UTF-8", NULL, NULL);
      }
      free(nenc);
    } else {
      nbuf = est_iconv(buf, size, enc, "UTF-8", NULL, NULL);
    }
  }
  if(nbuf) html = nbuf;
  if(!html) html = buf;
  datum = cbdatumopen("", 0);
  elems = cbxmlbreak(html, TRUE);
  for(i = 0; i < CB_LISTNUM(elems); i++){
    elem = CB_LISTVAL2(elems, i, &esiz);
    if(!(next = cblistval(elems, i + 1, NULL))) next = "";
    if(elem[0] == '<'){
      if(cbstrfwimatch(elem, "<meta")){
        attrs = cbxmlattrs(elem);
        name = cbmapget(attrs, "name", -1, NULL);
        if(!name) name = cbmapget(attrs, "Name", -1, NULL);
        if(!name) name = cbmapget(attrs, "NAME", -1, NULL);
        if(!name) name = cbmapget(attrs, "http-equiv", -1, NULL);
        if(!name) name = cbmapget(attrs, "Http-equiv", -1, NULL);
        if(!name) name = cbmapget(attrs, "Http-Equiv", -1, NULL);
        if(!name) name = cbmapget(attrs, "HTTP-EQUIV", -1, NULL);
        content = cbmapget(attrs, "content", -1, NULL);
        if(!content) content = cbmapget(attrs, "Content", -1, NULL);
        if(!content) content = cbmapget(attrs, "CONTENT", -1, NULL);
        if(name && content){
          lbuf = cbmemdup(name, -1);
          cbstrtolower(lbuf);
          cbstrsqzspc(lbuf);
          if(!strcmp(lbuf, "author")){
            if(strchr(content, '&')){
              rbuf = est_html_raw_text(content);
              est_doc_add_attr(doc, ESTDATTRAUTHOR, rbuf);
              free(rbuf);
            } else {
              est_doc_add_attr(doc, ESTDATTRAUTHOR, content);
            }
          }
          if(name[0] != '@'){
            if(strchr(content, '&')){
              rbuf = est_html_raw_text(content);
              est_doc_add_attr(doc, lbuf, rbuf);
              free(rbuf);
            } else {
              est_doc_add_attr(doc, lbuf, content);
            }
          }
          free(lbuf);
        }
        cbmapclose(attrs);
      } else if(cbstrfwimatch(elem, "<title") && next[0] != '\0' && next[0] != '<'){
        if(strchr(next, '&')){
          rbuf = est_html_raw_text(next);
          est_doc_add_attr(doc, ESTDATTRTITLE, rbuf);
          est_doc_add_hidden_text(doc, rbuf);
          free(rbuf);
        } else {
          est_doc_add_attr(doc, ESTDATTRTITLE, next);
          est_doc_add_hidden_text(doc, next);
        }
        i++;
      } else if(cbstrfwimatch(elem, "<style") || cbstrfwimatch(elem, "<script")){
        i++;
      } else if(cbstrfwimatch(elem, "<h1") || cbstrfwimatch(elem, "<h2") ||
                cbstrfwimatch(elem, "<h3") || cbstrfwimatch(elem, "<h4") ||
                cbstrfwimatch(elem, "<h5") || cbstrfwimatch(elem, "<h6") ||
                cbstrfwimatch(elem, "<p>") || cbstrfwimatch(elem, "<p ") ||
                cbstrfwimatch(elem, "<div") || cbstrfwimatch(elem, "<hr") ||
                cbstrfwimatch(elem, "<ul") || cbstrfwimatch(elem, "<ol") ||
                cbstrfwimatch(elem, "<dl") || cbstrfwimatch(elem, "<li") ||
                cbstrfwimatch(elem, "<dt") || cbstrfwimatch(elem, "<dd") ||
                cbstrfwimatch(elem, "<th") || cbstrfwimatch(elem, "<td") ||
                cbstrfwimatch(elem, "<pre")){
        if(strchr(CB_DATUMPTR(datum), '&')){
          rbuf = est_html_raw_text(CB_DATUMPTR(datum));
          est_doc_add_text(doc, rbuf);
          free(rbuf);
        } else {
          est_doc_add_text(doc, CB_DATUMPTR(datum));
        }
        cbdatumsetsize(datum, 0);
      }
    } else {
      cbdatumcat(datum, " ", -1);
      cbdatumcat(datum, elem, esiz);
    }
  }
  cblistclose(elems);
  if(strchr(CB_DATUMPTR(datum), '&')){
    rbuf = est_html_raw_text(CB_DATUMPTR(datum));
    est_doc_add_text(doc, rbuf);
    free(rbuf);
  } else {
    est_doc_add_text(doc, CB_DATUMPTR(datum));
  }
  cbdatumclose(datum);
  if(nbuf) free(nbuf);
  est_doc_add_attr(doc, ESTDATTRTYPE, "text/html");
  sprintf(numbuf, "%d", size);
  est_doc_add_attr(doc, ESTDATTRSIZE, numbuf);
  return doc;
}


/* get the encoding of an HTML string */
static char *est_html_enc(const char *str){
  CBLIST *elems;
  CBMAP *attrs;
  const char *elem, *equiv, *content;
  char *enc, *pv;
  int i;
  assert(str);
  elems = cbxmlbreak(str, TRUE);
  for(i = 0; i < CB_LISTNUM(elems); i++){
    elem = CB_LISTVAL(elems, i, NULL);
    if(elem[0] != '<' || !cbstrfwimatch(elem, "<meta")) continue;
    enc = NULL;
    attrs = cbxmlattrs(elem);
    equiv = cbmapget(attrs, "http-equiv", -1, NULL);
    if(!equiv) equiv = cbmapget(attrs, "HTTP-EQUIV", -1, NULL);
    if(!equiv) equiv = cbmapget(attrs, "Http-Equiv", -1, NULL);
    if(!equiv) equiv = cbmapget(attrs, "Http-equiv", -1, NULL);
    if(equiv && !cbstricmp(equiv, "Content-Type")){
      content = cbmapget(attrs, "content", -1, NULL);
      if(!content) content = cbmapget(attrs, "Content", -1, NULL);
      if(!content) content = cbmapget(attrs, "CONTENT", -1, NULL);
      if(content && ((pv = strstr(content, "charset")) != NULL ||
                     (pv = strstr(content, "Charset")) != NULL ||
                     (pv = strstr(content, "CHARSET")) != NULL)){
        enc = cbmemdup(pv + 8, -1);
        if((pv = strchr(enc, ';')) != NULL || (pv = strchr(enc, '\r')) != NULL ||
           (pv = strchr(enc, '\n')) != NULL || (pv = strchr(enc, ' ')) != NULL) *pv = '\0';
      }
    }
    cbmapclose(attrs);
    if(enc){
      cblistclose(elems);
      return enc;
    }
  }
  cblistclose(elems);
  return NULL;
}


/* unescape entity references of HTML */
static char *est_html_raw_text(const char *html){
  static const char *pairs[] = {
    /* basic symbols */
    "&amp;", "&", "&lt;", "<", "&gt;", ">", "&quot;", "\"", "&apos;", "'",
    /* ISO-8859-1 */
    "&nbsp;", "\xc2\xa0", "&iexcl;", "\xc2\xa1", "&cent;", "\xc2\xa2",
    "&pound;", "\xc2\xa3", "&curren;", "\xc2\xa4", "&yen;", "\xc2\xa5",
    "&brvbar;", "\xc2\xa6", "&sect;", "\xc2\xa7", "&uml;", "\xc2\xa8",
    "&copy;", "\xc2\xa9", "&ordf;", "\xc2\xaa", "&laquo;", "\xc2\xab",
    "&not;", "\xc2\xac", "&shy;", "\xc2\xad", "&reg;", "\xc2\xae",
    "&macr;", "\xc2\xaf", "&deg;", "\xc2\xb0", "&plusmn;", "\xc2\xb1",
    "&sup2;", "\xc2\xb2", "&sup3;", "\xc2\xb3", "&acute;", "\xc2\xb4",
    "&micro;", "\xc2\xb5", "&para;", "\xc2\xb6", "&middot;", "\xc2\xb7",
    "&cedil;", "\xc2\xb8", "&sup1;", "\xc2\xb9", "&ordm;", "\xc2\xba",
    "&raquo;", "\xc2\xbb", "&frac14;", "\xc2\xbc", "&frac12;", "\xc2\xbd",
    "&frac34;", "\xc2\xbe", "&iquest;", "\xc2\xbf", "&Agrave;", "\xc3\x80",
    "&Aacute;", "\xc3\x81", "&Acirc;", "\xc3\x82", "&Atilde;", "\xc3\x83",
    "&Auml;", "\xc3\x84", "&Aring;", "\xc3\x85", "&AElig;", "\xc3\x86",
    "&Ccedil;", "\xc3\x87", "&Egrave;", "\xc3\x88", "&Eacute;", "\xc3\x89",
    "&Ecirc;", "\xc3\x8a", "&Euml;", "\xc3\x8b", "&Igrave;", "\xc3\x8c",
    "&Iacute;", "\xc3\x8d", "&Icirc;", "\xc3\x8e", "&Iuml;", "\xc3\x8f",
    "&ETH;", "\xc3\x90", "&Ntilde;", "\xc3\x91", "&Ograve;", "\xc3\x92",
    "&Oacute;", "\xc3\x93", "&Ocirc;", "\xc3\x94", "&Otilde;", "\xc3\x95",
    "&Ouml;", "\xc3\x96", "&times;", "\xc3\x97", "&Oslash;", "\xc3\x98",
    "&Ugrave;", "\xc3\x99", "&Uacute;", "\xc3\x9a", "&Ucirc;", "\xc3\x9b",
    "&Uuml;", "\xc3\x9c", "&Yacute;", "\xc3\x9d", "&THORN;", "\xc3\x9e",
    "&szlig;", "\xc3\x9f", "&agrave;", "\xc3\xa0", "&aacute;", "\xc3\xa1",
    "&acirc;", "\xc3\xa2", "&atilde;", "\xc3\xa3", "&auml;", "\xc3\xa4",
    "&aring;", "\xc3\xa5", "&aelig;", "\xc3\xa6", "&ccedil;", "\xc3\xa7",
    "&egrave;", "\xc3\xa8", "&eacute;", "\xc3\xa9", "&ecirc;", "\xc3\xaa",
    "&euml;", "\xc3\xab", "&igrave;", "\xc3\xac", "&iacute;", "\xc3\xad",
    "&icirc;", "\xc3\xae", "&iuml;", "\xc3\xaf", "&eth;", "\xc3\xb0",
    "&ntilde;", "\xc3\xb1", "&ograve;", "\xc3\xb2", "&oacute;", "\xc3\xb3",
    "&ocirc;", "\xc3\xb4", "&otilde;", "\xc3\xb5", "&ouml;", "\xc3\xb6",
    "&divide;", "\xc3\xb7", "&oslash;", "\xc3\xb8", "&ugrave;", "\xc3\xb9",
    "&uacute;", "\xc3\xba", "&ucirc;", "\xc3\xbb", "&uuml;", "\xc3\xbc",
    "&yacute;", "\xc3\xbd", "&thorn;", "\xc3\xbe", "&yuml;", "\xc3\xbf",
    /* ISO-10646 */
    "&fnof;", "\xc6\x92", "&Alpha;", "\xce\x91", "&Beta;", "\xce\x92",
    "&Gamma;", "\xce\x93", "&Delta;", "\xce\x94", "&Epsilon;", "\xce\x95",
    "&Zeta;", "\xce\x96", "&Eta;", "\xce\x97", "&Theta;", "\xce\x98",
    "&Iota;", "\xce\x99", "&Kappa;", "\xce\x9a", "&Lambda;", "\xce\x9b",
    "&Mu;", "\xce\x9c", "&Nu;", "\xce\x9d", "&Xi;", "\xce\x9e",
    "&Omicron;", "\xce\x9f", "&Pi;", "\xce\xa0", "&Rho;", "\xce\xa1",
    "&Sigma;", "\xce\xa3", "&Tau;", "\xce\xa4", "&Upsilon;", "\xce\xa5",
    "&Phi;", "\xce\xa6", "&Chi;", "\xce\xa7", "&Psi;", "\xce\xa8",
    "&Omega;", "\xce\xa9", "&alpha;", "\xce\xb1", "&beta;", "\xce\xb2",
    "&gamma;", "\xce\xb3", "&delta;", "\xce\xb4", "&epsilon;", "\xce\xb5",
    "&zeta;", "\xce\xb6", "&eta;", "\xce\xb7", "&theta;", "\xce\xb8",
    "&iota;", "\xce\xb9", "&kappa;", "\xce\xba", "&lambda;", "\xce\xbb",
    "&mu;", "\xce\xbc", "&nu;", "\xce\xbd", "&xi;", "\xce\xbe",
    "&omicron;", "\xce\xbf", "&pi;", "\xcf\x80", "&rho;", "\xcf\x81",
    "&sigmaf;", "\xcf\x82", "&sigma;", "\xcf\x83", "&tau;", "\xcf\x84",
    "&upsilon;", "\xcf\x85", "&phi;", "\xcf\x86", "&chi;", "\xcf\x87",
    "&psi;", "\xcf\x88", "&omega;", "\xcf\x89", "&thetasym;", "\xcf\x91",
    "&upsih;", "\xcf\x92", "&piv;", "\xcf\x96", "&bull;", "\xe2\x80\xa2",
    "&hellip;", "\xe2\x80\xa6", "&prime;", "\xe2\x80\xb2", "&Prime;", "\xe2\x80\xb3",
    "&oline;", "\xe2\x80\xbe", "&frasl;", "\xe2\x81\x84", "&weierp;", "\xe2\x84\x98",
    "&image;", "\xe2\x84\x91", "&real;", "\xe2\x84\x9c", "&trade;", "\xe2\x84\xa2",
    "&alefsym;", "\xe2\x84\xb5", "&larr;", "\xe2\x86\x90", "&uarr;", "\xe2\x86\x91",
    "&rarr;", "\xe2\x86\x92", "&darr;", "\xe2\x86\x93", "&harr;", "\xe2\x86\x94",
    "&crarr;", "\xe2\x86\xb5", "&lArr;", "\xe2\x87\x90", "&uArr;", "\xe2\x87\x91",
    "&rArr;", "\xe2\x87\x92", "&dArr;", "\xe2\x87\x93", "&hArr;", "\xe2\x87\x94",
    "&forall;", "\xe2\x88\x80", "&part;", "\xe2\x88\x82", "&exist;", "\xe2\x88\x83",
    "&empty;", "\xe2\x88\x85", "&nabla;", "\xe2\x88\x87", "&isin;", "\xe2\x88\x88",
    "&notin;", "\xe2\x88\x89", "&ni;", "\xe2\x88\x8b", "&prod;", "\xe2\x88\x8f",
    "&sum;", "\xe2\x88\x91", "&minus;", "\xe2\x88\x92", "&lowast;", "\xe2\x88\x97",
    "&radic;", "\xe2\x88\x9a", "&prop;", "\xe2\x88\x9d", "&infin;", "\xe2\x88\x9e",
    "&ang;", "\xe2\x88\xa0", "&and;", "\xe2\x88\xa7", "&or;", "\xe2\x88\xa8",
    "&cap;", "\xe2\x88\xa9", "&cup;", "\xe2\x88\xaa", "&int;", "\xe2\x88\xab",
    "&there4;", "\xe2\x88\xb4", "&sim;", "\xe2\x88\xbc", "&cong;", "\xe2\x89\x85",
    "&asymp;", "\xe2\x89\x88", "&ne;", "\xe2\x89\xa0", "&equiv;", "\xe2\x89\xa1",
    "&le;", "\xe2\x89\xa4", "&ge;", "\xe2\x89\xa5", "&sub;", "\xe2\x8a\x82",
    "&sup;", "\xe2\x8a\x83", "&nsub;", "\xe2\x8a\x84", "&sube;", "\xe2\x8a\x86",
    "&supe;", "\xe2\x8a\x87", "&oplus;", "\xe2\x8a\x95", "&otimes;", "\xe2\x8a\x97",
    "&perp;", "\xe2\x8a\xa5", "&sdot;", "\xe2\x8b\x85", "&lceil;", "\xe2\x8c\x88",
    "&rceil;", "\xe2\x8c\x89", "&lfloor;", "\xe2\x8c\x8a", "&rfloor;", "\xe2\x8c\x8b",
    "&lang;", "\xe2\x8c\xa9", "&rang;", "\xe2\x8c\xaa", "&loz;", "\xe2\x97\x8a",
    "&spades;", "\xe2\x99\xa0", "&clubs;", "\xe2\x99\xa3", "&hearts;", "\xe2\x99\xa5",
    "&diams;", "\xe2\x99\xa6", "&OElig;", "\xc5\x92", "&oelig;", "\xc5\x93",
    "&Scaron;", "\xc5\xa0", "&scaron;", "\xc5\xa1", "&Yuml;", "\xc5\xb8",
    "&circ;", "\xcb\x86", "&tilde;", "\xcb\x9c", "&ensp;", "\xe2\x80\x82",
    "&emsp;", "\xe2\x80\x83", "&thinsp;", "\xe2\x80\x89", "&zwnj;", "\xe2\x80\x8c",
    "&zwj;", "\xe2\x80\x8d", "&lrm;", "\xe2\x80\x8e", "&rlm;", "\xe2\x80\x8f",
    "&ndash;", "\xe2\x80\x93", "&mdash;", "\xe2\x80\x94", "&lsquo;", "\xe2\x80\x98",
    "&rsquo;", "\xe2\x80\x99", "&sbquo;", "\xe2\x80\x9a", "&ldquo;", "\xe2\x80\x9c",
    "&rdquo;", "\xe2\x80\x9d", "&bdquo;", "\xe2\x80\x9e", "&dagger;", "\xe2\x80\xa0",
    "&Dagger;", "\xe2\x80\xa1", "&permil;", "\xe2\x80\xb0", "&lsaquo;", "\xe2\x80\xb9",
    "&rsaquo;", "\xe2\x80\xba", "&euro;", "\xe2\x82\xac",
    NULL
  };
  char *raw, *wp, buf[2], *tmp;
  int i, j, hit, num, tsiz;
  assert(html);
  CB_MALLOC(raw, strlen(html) * 3 + 1);
  wp = raw;
  while(*html != '\0'){
    if(*html == '&'){
      if(*(html + 1) == '#'){
        if(*(html + 2) == 'x' || *(html + 2) == 'X'){
          num = strtol(html + 3, NULL, 16);
        } else {
          num = atoi(html + 2);
        }
        buf[0] = num / 256;
        buf[1] = num % 256;
        if((tmp = est_uconv_out(buf, 2, &tsiz)) != NULL){
          for(j = 0; j < tsiz; j++){
            *wp = ((unsigned char *)tmp)[j];
            wp++;
          }
          free(tmp);
        }
        while(*html != ';' && *html != ' ' && *html != '\n' && *html != '\0'){
          html++;
        }
        if(*html == ';') html++;
      } else {
        hit = FALSE;
        for(i = 0; pairs[i] != NULL; i += 2){
          if(cbstrfwmatch(html, pairs[i])){
            wp += sprintf(wp, "%s", pairs[i+1]);
            html += strlen(pairs[i]);
            hit = TRUE;
            break;
          }
        }
        if(!hit){
          *wp = *html;
          wp++;
          html++;
        }
      }
    } else {
      *wp = *html;
      wp++;
      html++;
    }
  }
  *wp = '\0';
  return raw;
}


/* create a document object from MIME */
static ESTDOC *est_doc_new_from_mime(const char *buf, int size, const char *penc, int plang){
  ESTDOC *doc, *tdoc;
  CBMAP *attrs;
  const CBLIST *texts;
  CBLIST *parts, *lines;
  CBDATUM *datum;
  const char *key, *val, *bound, *part, *text, *line;
  char *body, *swap, numbuf[NUMBUFSIZ];
  int i, j, bsiz, psiz, ssiz, mht;
  assert(buf);
  doc = est_doc_new();
  attrs = cbmapopenex(MINIBNUM);
  body = cbmimebreak(buf, size, attrs, &bsiz);
  if((val = cbmapget(attrs, "subject", -1, NULL)) != NULL){
    est_doc_add_attr_mime(doc, ESTDATTRTITLE, val);
    if((val = est_doc_attr(doc, ESTDATTRTITLE)) != NULL) est_doc_add_hidden_text(doc, val);
  }
  if((val = cbmapget(attrs, "from", -1, NULL)) != NULL)
    est_doc_add_attr_mime(doc, ESTDATTRAUTHOR, val);
  if((val = cbmapget(attrs, "date", -1, NULL)) != NULL){
    est_doc_add_attr_mime(doc, ESTDATTRCDATE, val);
    est_doc_add_attr_mime(doc, ESTDATTRMDATE, val);
  }
  est_doc_add_attr(doc, ESTDATTRTYPE, "message/rfc822");
  sprintf(numbuf, "%d", size);
  est_doc_add_attr(doc, ESTDATTRSIZE, numbuf);
  cbmapiterinit(attrs);
  while((key = cbmapiternext(attrs, NULL)) != NULL){
    if((key[0] >= 'A' && key[0] <= 'Z') || key[0] == '@') continue;
    val = cbmapget(attrs, key, -1, NULL);
    est_doc_add_attr_mime(doc, key, val);
  }
  if((key = cbmapget(attrs, "TYPE", -1, NULL)) != NULL && cbstrfwimatch(key, "multipart/")){
    mht = cbstrfwimatch(key, "multipart/related");
    if((bound = cbmapget(attrs, "BOUNDARY", -1, NULL)) != NULL){
      parts = cbmimeparts(body, bsiz, bound);
      for(i = 0; i < CB_LISTNUM(parts) && i < 8; i++){
        part = CB_LISTVAL2(parts, i, &psiz);
        tdoc = est_doc_new_from_mime(part, psiz, penc, plang);
        if(mht){
          if((text = est_doc_attr(tdoc, ESTDATTRTITLE)) != NULL)
            est_doc_add_attr(doc, ESTDATTRTITLE, text);
          if((text = est_doc_attr(tdoc, ESTDATTRAUTHOR)) != NULL)
            est_doc_add_attr(doc, ESTDATTRAUTHOR, text);
        }
        texts = est_doc_texts(tdoc);
        for(j = 0; j < CB_LISTNUM(texts); j++){
          text = CB_LISTVAL(texts, j, NULL);
          est_doc_add_text(doc, text);
        }
        est_doc_delete(tdoc);
      }
      cblistclose(parts);
    }
  } else {
    if((key = cbmapget(attrs, "content-transfer-encoding", -1, NULL)) != NULL &&
       cbstrfwimatch(key, "base64")){
      swap = cbbasedecode(body, &ssiz);
      free(body);
      body = swap;
      bsiz = ssiz;
    } else if((key = cbmapget(attrs, "content-transfer-encoding", -1, NULL)) != NULL &&
              cbstrfwimatch(key, "quoted-printable")){
      swap = cbquotedecode(body, &ssiz);
      free(body);
      body = swap;
      bsiz = ssiz;
    }
    if(!(key = cbmapget(attrs, "TYPE", -1, NULL)) || cbstrfwimatch(key, "text/plain")){
      if(penc && (swap = est_iconv(body, bsiz, penc, "UTF-8", &ssiz, NULL)) != NULL){
        free(body);
        body = swap;
        bsiz = ssiz;
      } else if((key = cbmapget(attrs, "CHARSET", -1, NULL)) != NULL &&
                (swap = est_iconv(body, bsiz, key, "UTF-8", &ssiz, NULL)) != NULL){
        free(body);
        body = swap;
        bsiz = ssiz;
      }
      lines = cbsplit(body, bsiz, "\n");
      datum = cbdatumopen("", 0);
      for(i = 0; i < CB_LISTNUM(lines); i++){
        line = CB_LISTVAL(lines, i, NULL);
        while(*line == ' ' || *line == '>' || *line == '|' || *line == '\t' || *line == '\r'){
          line++;
        }
        if(line[0] == '\0'){
          est_doc_add_text(doc, CB_DATUMPTR(datum));
          cbdatumsetsize(datum, 0);
        } else {
          cbdatumcat(datum, " ", 1);
          cbdatumcat(datum, line, -1);
        }
      }
      est_doc_add_text(doc, CB_DATUMPTR(datum));
      cbdatumclose(datum);
      cblistclose(lines);
    } else if(cbstrfwimatch(key, "text/html")){
      tdoc = est_doc_new_from_html(body, bsiz, penc, plang);
      if((text = est_doc_attr(tdoc, ESTDATTRTITLE)) != NULL){
        if(!est_doc_attr(doc, ESTDATTRTITLE)) est_doc_add_attr(doc, ESTDATTRTITLE, text);
        est_doc_add_text(doc, text);
      }
      if((text = est_doc_attr(tdoc, ESTDATTRAUTHOR)) != NULL){
        if(!est_doc_attr(doc, ESTDATTRAUTHOR)) est_doc_add_attr(doc, ESTDATTRAUTHOR, text);
        est_doc_add_text(doc, text);
      }
      texts = est_doc_texts(tdoc);
      for(i = 0; i < CB_LISTNUM(texts); i++){
        text = CB_LISTVAL(texts, i, NULL);
        est_doc_add_text(doc, text);
      }
      est_doc_delete(tdoc);
    } else if(cbstrfwimatch(key, "message/rfc822")){
      tdoc = est_doc_new_from_mime(body, bsiz, penc, plang);
      if((text = est_doc_attr(tdoc, ESTDATTRTITLE)) != NULL){
        if(!est_doc_attr(doc, ESTDATTRTITLE)) est_doc_add_attr(doc, ESTDATTRTITLE, text);
        est_doc_add_text(doc, text);
      }
      if((text = est_doc_attr(tdoc, ESTDATTRAUTHOR)) != NULL){
        if(!est_doc_attr(doc, ESTDATTRAUTHOR)) est_doc_add_attr(doc, ESTDATTRAUTHOR, text);
        est_doc_add_text(doc, text);
      }
      texts = est_doc_texts(tdoc);
      for(i = 0; i < CB_LISTNUM(texts); i++){
        text = CB_LISTVAL(texts, i, NULL);
        est_doc_add_text(doc, text);
      }
      est_doc_delete(tdoc);
    } else if(cbstrfwimatch(key, "text/")){
      tdoc = est_doc_new_from_text(body, bsiz, penc, plang);
      texts = est_doc_texts(tdoc);
      for(i = 0; i < CB_LISTNUM(texts); i++){
        text = CB_LISTVAL(texts, i, NULL);
        est_doc_add_text(doc, text);
      }
      est_doc_delete(tdoc);
    }
  }
  free(body);
  cbmapclose(attrs);
  return doc;
}


/* set mime value as an attribute of a document */
static void est_doc_add_attr_mime(ESTDOC *doc, const char *name, const char *value){
  char enc[64], *ebuf, *rbuf;
  assert(doc && name && value);
  ebuf = cbmimedecode(value, enc);
  if((rbuf = est_iconv(ebuf, -1, enc, "UTF-8", NULL, NULL)) != NULL){
    est_doc_add_attr(doc, name, rbuf);
    free(rbuf);
  }
  free(ebuf);
}


/* generate a document with random text */
static ESTDOC *est_doc_new_from_chaos(int cnum, int snum, int mode){
  ESTDOC *doc;
  char *str;
  int i;
  doc = est_doc_new();
  snum *= pow(est_random_nd() + 0.5, 3.0);
  if(mode == RD_RAND){
    mode = est_random() * 100;
    if(mode < 20){
      mode = RD_ENG;
      est_doc_add_attr(doc, "mode", "english");
    } else if(mode < 40){
      mode = RD_LAT;
      est_doc_add_attr(doc, "mode", "latin");
    } else if(mode < 60){
      mode = RD_EURO;
      est_doc_add_attr(doc, "mode", "euromix");
    } else if(mode < 65){
      mode = RD_ORI;
      est_doc_add_attr(doc, "mode", "oriental");
    } else if(mode < 95){
      mode = RD_JPN;
      est_doc_add_attr(doc, "mode", "japanese");
    } else {
      mode = RD_CHAO;
      est_doc_add_attr(doc, "mode", "chaos");
    }
  }
  switch(mode){
  case RD_ENG: est_doc_add_attr(doc, "mode", "english"); break;
  case RD_LAT: est_doc_add_attr(doc, "mode", "latin"); break;
  case RD_ORI: est_doc_add_attr(doc, "mode", "oriental"); break;
  case RD_JPN: est_doc_add_attr(doc, "mode", "japanese"); break;
  case RD_EURO: est_doc_add_attr(doc, "mode", "euromix"); break;
  case RD_CHAO: est_doc_add_attr(doc, "mode", "chaos"); break;
  }
  for(i = 0; i <= snum; i++){
    str = est_random_str(cnum, mode);
    if(est_random() < 0.05){
      est_doc_add_hidden_text(doc, str);
    } else {
      est_doc_add_text(doc, str);
    }
    free(str);
  }
  return doc;
}


/* generate random string */
static char *est_random_str(int cnum, int mode){
  const char echrs[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
  CBDATUM *buf;
  char wc[2], *str;
  int i, c, wlen, dec, mm, big, n;
  buf = cbdatumopen("", 0);
  cnum *= pow(est_random_nd() + 0.5, 3.0);
  wlen = est_random_nd() * 8 + 4;
  dec = (int)(est_random() * INT_MAX) % 10;
  big = (((int)(est_random() * INT_MAX) % 0x29)) * 0x100;
  for(i = 0; i < cnum; i++){
    switch(mode){
    case RD_ENG: case RD_LAT: case RD_EURO:
      mm = (int)(est_random() * INT_MAX) % 100;
      if((mode == RD_LAT || mode == RD_EURO) && mm < 5){
        c = 0x00a1 + (int)(pow(est_random_nd(), 2.0) * (0x00ff - 0x00a0));
      } else if(mode == RD_EURO && (mm < 30 || dec > 8)){
        if(dec % 2 == 0){
          c = 0x0391 + (int)(pow(est_random_nd(), 2.0) * (0x03d6 - 0x0391));
        } else {
          c = 0x0400 + (int)(pow(est_random_nd(), 2.0) * (0x045f - 0x0400));
        }
      } else if(mm < 95){
        if((n = est_random_nd() * (sizeof(echrs) - 1)) == (sizeof(echrs) - 1)) n = 0;
        c = echrs[n];
      } else {
        c = (int)(est_random() * ('@' - ' ')) + ' ';
      }
      if(--wlen < 1){
        c = ' ';
        wlen = pow(est_random_nd(), 3.0) * 8 + 4;
        dec = (int)(est_random() * INT_MAX) % 10;
      }
      break;
    case RD_ORI:
      c = big + est_random_nd() * 0x100;
      if(--wlen < 1){
        wlen = pow(est_random_nd(), 3.0) * 12 + 6;
        big = (((int)(est_random() * INT_MAX) % 0x29)) * 0x100;
      }
      break;
    case RD_JPN:
      if(dec < 4){
        c = 0x3041 + pow(est_random_nd(), 3.0) * (0x3094 - 0x3041);
      } else if(dec < 7){
        c = 0x30a1 + pow(est_random_nd(), 3.0) * (0x30fe - 0x30a1);
      } else if(dec < 9){
        c = 0x4e00 + pow(est_random_nd(), 3.0) * (0x9faf - 0x4e00);
      } else {
        if(est_random() < 0.7){
          c = 0x00a1 + (int)(pow(est_random_nd(), 2.0) * (0x00ff - 0x00a0));
        } else {
          c = 0x3041 + est_random() * (0xffef - 0x3041);
        }
      }
      if(--wlen < 1){
        wlen = pow(est_random_nd(), 3.0) * 12 + 6;
        dec = (int)(est_random() * INT_MAX) % 10;
      }
      break;
    default:
      if(est_random() < 0.2){
        c = 0x00a1 + (int)est_random() * (0x00ff - 0x00a0);
      } else {
        c = (int)(est_random() * 0x10000);
      }
      break;
    }
    if(c <= 0 || c >= 0x10000) c = 0x0020;
    wc[0] = c / 0x100;
    wc[1] = c % 0x100;
    cbdatumcat(buf, wc, 2);
  }
  str = est_iconv(CB_DATUMPTR(buf), CB_DATUMSIZE(buf), "UTF-16BE", "UTF-8", NULL, NULL);
  cbdatumclose(buf);
  return str;
}


/* END OF FILE */