/** * @file xdict.c (dictionary query) * @author Hightman Mar * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) * $Id$ */ #ifdef HAVE_CONFIG_H # include "config.h" #endif #ifdef WIN32 # include "config_win32.h" #endif #include "xdict.h" #include "xtree.h" #include "xdb.h" #include "crc32.h" #include #include #include #include // for unlink #ifndef WIN32 # include #endif #include #include /* temp file format for TEXT xdb */ #if !defined(PATH_MAX) || (PATH_MAX < 1024) # define XDICT_PATH_MAX 1024 #else # define XDICT_PATH_MAX PATH_MAX #endif #ifdef HAVE_STRTOK_R # define _strtok_r strtok_r #else static char *_strtok_r(char *s, char *delim, char **lasts) { register char *spanp; register int c, sc; char *tok; if (s == NULL && (s = *lasts) == NULL) return NULL; /* * Skip (span) leading delimiters (s += strspn(s, delim), sort of). */ cont: c = *s++; for (spanp = (char *) delim; (sc = *spanp++) != 0;) { if (c == sc) goto cont; } if (c == 0) { /* no non-delimiter characters */ *lasts = NULL; return NULL; } tok = s - 1; /* * Scan token (scan for delimiters: s += strcspn(s, delim), sort of). * Note that delim must have one NUL; we stop if we see that, too. */ for (;;) { c = *s++; spanp = (char *) delim; do { if ((sc = *spanp++) == c) { if (c == 0) s = NULL; else s[-1] = '\0'; *lasts = s; return tok; } } while (sc != 0); } } #endif #ifdef WIN32 # include static void _realpath(const char *src, char *dst) { int len = strlen(src); if (strchr(src, ':') != NULL) memcpy(dst, src, len + 1); else { char *ptr; getcwd(dst, XDICT_PATH_MAX - len - 2); ptr = dst + strlen(dst); *ptr++ = '/'; memcpy(ptr, src, len + 1); } } #else # define _realpath realpath #endif /* open the text dict */ static xdict_t _xdict_open_txt(const char *fpath, int mode, unsigned char *ml) { xdict_t xd; xtree_t xt; char buf[XDICT_PATH_MAX], tmpfile[XDICT_PATH_MAX]; struct stat st1, st2; // check the input filepath _realpath(fpath, buf); if (stat(buf, &st1) < 0) return NULL; // check dest file & orginal file, compare there mtime #ifdef WIN32 { char *tmp_ptr; GetTempPath(sizeof(tmpfile) - 20, tmpfile); tmp_ptr = tmpfile + strlen(tmpfile); if (tmp_ptr[-1] == '\\') tmp_ptr--; sprintf(tmp_ptr, "\\scws-%08x.xdb", scws_crc32(buf)); } #else sprintf(tmpfile, "/tmp/scws-%08x.xdb", scws_crc32(buf)); #endif if (!stat(tmpfile, &st2) && st2.st_mtime > st1.st_mtime) { xdb_t x; if ((x = xdb_open(tmpfile, 'r')) != NULL) { xd = (xdict_t) malloc(sizeof(xdict_st)); memset(xd, 0, sizeof(xdict_st)); xd->ref = 1; if (mode & SCWS_XDICT_MEM) { /* convert the xdb(disk) -> xtree(memory) */ if ((xt = xdb_to_xtree(x, NULL)) != NULL) { xdb_close(x); xd->xdict = (void *) xt; xd->xmode = SCWS_XDICT_MEM; return xd; } } xd->xmode = SCWS_XDICT_XDB; xd->xdict = (void *) x; return xd; } } // create xtree if ((xt = xtree_new(0, 0)) == NULL) return NULL; else { int cl, kl; FILE *fp; word_st word, *w; char *key, *part, *last, *delim = " \t\r\n"; // re-build the xdb file from text file if ((fp = fopen(buf, "r")) == NULL) return NULL; // parse every line word.attr[2] = '\0'; while (fgets(buf, sizeof(buf) - 1, fp) != NULL) { // [\t[\t[\t]]] if (buf[0] == ';' || buf[0] == '#') continue; key = _strtok_r(buf, delim, &last); if (key == NULL) continue; kl = strlen(key); // init the word do { word.tf = word.idf = 1.0; word.flag = SCWS_WORD_FULL; word.attr[0] = '@'; word.attr[1] = '\0'; if (!(part = _strtok_r(NULL, delim, &last))) break; word.tf = (float) atof(part); if (!(part = _strtok_r(NULL, delim, &last))) break; word.idf = (float) atof(part); if (part = _strtok_r(NULL, delim, &last)) { word.attr[0] = part[0]; if (part[1]) word.attr[1] = part[1]; } } while (0); // save into xtree if ((w = xtree_nget(xt, key, kl, NULL)) == NULL) { w = (word_st *) pmalloc(xt->p, sizeof(word_st)); memcpy(w, &word, sizeof(word)); xtree_nput(xt, w, sizeof(word), key, kl); } else { w->tf = word.tf; w->idf = word.idf; w->flag |= word.flag; strcpy(w->attr, word.attr); } // parse the part cl = ml[(unsigned char) (key[0])]; while (1) { cl += ml[(unsigned char) (key[cl])]; if (cl >= kl) break; if ((w = xtree_nget(xt, key, cl, NULL)) != NULL) w->flag |= SCWS_WORD_PART; else { w = (word_st *) pmalloc_z(xt->p, sizeof(word_st)); w->flag = SCWS_WORD_PART; xtree_nput(xt, w, sizeof(word), key, cl); } } } fclose(fp); // optimize the xtree & save to xdb xtree_optimize(xt); unlink(tmpfile); xtree_to_xdb(xt, tmpfile); chmod(tmpfile, 0777); // return xtree xd = (xdict_t) malloc(sizeof(xdict_st)); memset(xd, 0, sizeof(xdict_st)); xd->ref = 1; xd->xdict = (void *) xt; xd->xmode = SCWS_XDICT_MEM; return xd; } } /* setup & open the dict */ xdict_t xdict_open(const char *fpath, int mode) { xdict_t xd; xdb_t x; if (!(x = xdb_open(fpath, 'r'))) return NULL; xd = (xdict_t) malloc(sizeof(xdict_st)); memset(xd, 0, sizeof(xdict_st)); xd->ref = 1; if (mode & SCWS_XDICT_MEM) { xtree_t xt; /* convert the xdb(disk) -> xtree(memory) */ if ((xt = xdb_to_xtree(x, NULL)) != NULL) { xdb_close(x); xd->xdict = (void *) xt; xd->xmode = SCWS_XDICT_MEM; return xd; } } xd->xmode = SCWS_XDICT_XDB; xd->xdict = (void *) x; return xd; } /* add a dict */ xdict_t xdict_add(xdict_t xd, const char *fpath, int mode, unsigned char *ml) { xdict_t xx; xx = (mode & SCWS_XDICT_TXT ? _xdict_open_txt(fpath, mode, ml) : xdict_open(fpath, mode)); if (xx != NULL) { xx->next = xd; return xx; } return xd; } /* fork the dict */ xdict_t xdict_fork(xdict_t xd) { xdict_t xx; for (xx = xd; xx != NULL; xx = xx->next) { xx->ref++; } return xd; } /* close the dict */ void xdict_close(xdict_t xd) { xdict_t xx; while ((xx = xd) != NULL) { xd = xx->next; xx->ref--; if (xx->ref == 0) { if (xx->xmode == SCWS_XDICT_MEM) xtree_free((xtree_t) xx->xdict); else { xdb_close((xdb_t) xx->xdict); } free(xx); } } } /* query the word */ #define _FLAG_BOTH(x) (((x)->flag & (SCWS_WORD_PART|SCWS_WORD_FULL)) == (SCWS_WORD_PART|SCWS_WORD_FULL)) #define _FLAG_FULL(x) ((x)->flag & SCWS_WORD_FULL) #define _FLAG_PART(x) ((x)->flag & SCWS_WORD_PART) #define _FLAG_MALLOC(x) ((x)->flag & SCWS_WORD_MALLOCED) word_t xdict_query(xdict_t xd, const char *key, int len) { word_t value, value2; value = value2 = NULL; while (xd != NULL) { if (xd->xmode == SCWS_XDICT_MEM) { /* this is ThreadSafe, recommend. */ value = (word_t) xtree_nget((xtree_t) xd->xdict, key, len, NULL); } else { /* the value malloced in lib-XDB. free required */ value = (word_t) xdb_nget((xdb_t) xd->xdict, key, len, NULL); if (value != NULL) value->flag |= SCWS_WORD_MALLOCED; } xd = xd->next; // check value2 if (value != NULL) { if (value2 == NULL) { if (_FLAG_BOTH(value)) return value; value2 = value; } else { if (_FLAG_FULL(value2) && _FLAG_PART(value)) { value2->flag |= SCWS_WORD_PART; if (_FLAG_MALLOC(value)) free(value); return value2; } if (_FLAG_FULL(value) && _FLAG_PART(value2)) { value->flag |= SCWS_WORD_PART; if (_FLAG_MALLOC(value2)) free(value2); return value; } if (_FLAG_MALLOC(value)) free(value); } } } return value2; }