1 /* Copyright(C) 2004 Brazil
3 This library is free software; you can redistribute it and/or
4 modify it under the terms of the GNU Lesser General Public
5 License as published by the Free Software Foundation; either
6 version 2.1 of the License, or (at your option) any later version.
8 This library is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 Lesser General Public License for more details.
13 You should have received a copy of the GNU Lesser General Public
14 License along with this library; if not, write to the Free Software
15 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 /* Declare this is the fast version of Senna */
27 /* available bit for sen_index_create flag */
28 #define SEN_INDEX_NORMALIZE 0x0001
29 #define SEN_INDEX_SPLIT_ALPHA 0x0002
30 #define SEN_INDEX_SPLIT_DIGIT 0x0004
31 #define SEN_INDEX_SPLIT_SYMBOL 0x0008
32 #define SEN_INDEX_MORPH_ANALYSE 0x0000
33 #define SEN_INDEX_NGRAM 0x0010
34 #define SEN_INDEX_DELIMITED 0x0020
35 #define SEN_INDEX_ENABLE_SUFFIX_SEARCH 0x0100
36 #define SEN_INDEX_DISABLE_SUFFIX_SEARCH 0x0200
37 #define SEN_INDEX_WITH_VGRAM 0x1000
38 #define SEN_INDEX_SHARED_LEXICON 0x2000
39 #define SEN_INDEX_WITH_VACUUM 0x8000
41 /* 16 tokenizers can be registered */
42 #define SEN_INDEX_TOKENIZER_MASK 0x00f0
44 #define SEN_SYM_MAX_KEY_SIZE 8192
46 #define SEN_SYM_WITH_SIS 0x80000000
48 #define SEN_SNIP_NORMALIZE 0x0001
49 #define SEN_SNIP_COPY_TAG 0x0002
50 #define SEN_SNIP_SKIP_LEADING_SPACES 0x0004
51 #define SEN_QUERY_SCAN_NORMALIZE SEN_SNIP_NORMALIZE
53 #define SEN_LEX_NGRAM_UNIT_SIZE 2
55 #ifndef SEN_STACK_SIZE
56 #define SEN_STACK_SIZE 0x10000000
57 #endif /* SEN_STACK_SIZE */
60 #define SEN_QUERY_AND '+'
61 #endif /* SEN_QUERY_AND */
63 #define SEN_QUERY_BUT '-'
64 #endif /* SEN_QUERY_BUT */
65 #ifndef SEN_QUERY_ADJ_INC
66 #define SEN_QUERY_ADJ_INC '>'
67 #endif /* SEN_QUERY_ADJ_POS2 */
68 #ifndef SEN_QUERY_ADJ_DEC
69 #define SEN_QUERY_ADJ_DEC '<'
70 #endif /* SEN_QUERY_ADJ_POS1 */
71 #ifndef SEN_QUERY_ADJ_NEG
72 #define SEN_QUERY_ADJ_NEG '~'
73 #endif /* SEN_QUERY_ADJ_NEG */
74 #ifndef SEN_QUERY_PREFIX
75 #define SEN_QUERY_PREFIX '*'
76 #endif /* SEN_QUERY_PREFIX */
77 #ifndef SEN_QUERY_PARENL
78 #define SEN_QUERY_PARENL '('
79 #endif /* SEN_QUERY_PARENL */
80 #ifndef SEN_QUERY_PARENR
81 #define SEN_QUERY_PARENR ')'
82 #endif /* SEN_QUERY_PARENR */
83 #ifndef SEN_QUERY_QUOTEL
84 #define SEN_QUERY_QUOTEL '"'
85 #endif /* SEN_QUERY_QUOTEL */
86 #ifndef SEN_QUERY_QUOTER
87 #define SEN_QUERY_QUOTER '"'
88 #endif /* SEN_QUERY_QUOTER */
89 #ifndef SEN_QUERY_ESCAPE
90 #define SEN_QUERY_ESCAPE '\\'
91 #endif /* SEN_QUERY_ESCAPE */
99 sen_file_operation_error,
100 sen_invalid_argument,
118 sen_rec_document = 0,
139 sen_sel_term_extract,
145 sen_sort_descending = 0,
162 typedef struct _sen_db sen_db;
163 typedef struct _sen_ctx sen_ctx;
164 typedef struct _sen_ctx_info sen_ctx_info;
165 typedef struct _sen_set sen_set;
166 typedef struct _sen_sym sen_sym;
167 typedef struct _sen_inv sen_inv;
168 typedef struct _sen_index sen_index;
169 typedef struct _sen_records sen_records;
170 typedef struct _sen_set_cursor sen_set_cursor;
171 typedef struct _sen_set_element *sen_set_eh;
172 typedef struct _sen_value sen_value;
173 typedef struct _sen_values sen_values;
174 typedef struct _sen_select_optarg sen_select_optarg;
175 typedef struct _sen_group_optarg sen_group_optarg;
176 typedef struct _sen_sort_optarg sen_sort_optarg;
177 typedef struct _sen_set_sort_optarg sen_set_sort_optarg;
178 typedef struct _sen_snip sen_snip;
179 typedef struct _sen_query sen_query;
180 typedef struct _sen_logger_info sen_logger_info;
181 typedef struct _sen_snip_mapping sen_snip_mapping;
182 typedef struct _sen_records_heap sen_records_heap;
183 typedef struct _sen_vgram sen_vgram;
184 typedef struct _sen_vgram_buf sen_vgram_buf;
185 typedef struct _sen_sym_scan_hit sen_sym_scan_hit;
186 typedef unsigned sen_id;
188 #define SEN_ID_MAX 0x3fffffff
190 typedef sen_set_eh sen_recordh;
192 struct _sen_ctx_info {
194 unsigned int com_status;
195 unsigned int com_info;
196 struct _sen_rbuf *outbuf;
208 struct _sen_records {
209 sen_rec_unit record_unit;
210 sen_rec_unit subrec_unit;
211 unsigned int max_n_subrecs;
212 unsigned int record_size;
213 unsigned int subrec_size;
216 sen_set_cursor *cursor;
217 sen_recordh *curr_rec;
220 int ignore_deleted_records;
227 unsigned int str_len;
236 struct _sen_select_optarg {
238 int similarity_threshold;
242 int (*func)(sen_records *, const void *, int, void *);
246 struct _sen_group_optarg {
248 int (*func)(sen_records *, const sen_recordh *, void *, void *);
253 struct _sen_sort_optarg {
255 int (*compar)(sen_records *, const sen_recordh *, sen_records *, const sen_recordh *, void *);
259 struct _sen_set_sort_optarg {
261 int (*compar)(sen_set *, sen_set_eh *, sen_set *, sen_set_eh *, void *);
263 sen_set *compar_arg0;
266 struct _sen_snip_mapping {
270 #define SEN_LOG_TIME 1
271 #define SEN_LOG_TITLE 2
272 #define SEN_LOG_MESSAGE 4
273 #define SEN_LOG_LOCATION 8
275 struct _sen_logger_info {
276 sen_log_level max_level;
278 void (*func)(int, const char *, const char *, const char *, const char *, void *);
282 struct _sen_sym_scan_hit {
288 typedef int (*query_term_callback)(const char *, unsigned int, void *);
291 /******** query language API ********/
293 sen_db *sen_db_create(const char *path, int flags, sen_encoding encoding);
294 sen_db *sen_db_open(const char *path);
295 sen_rc sen_db_close(sen_db *s);
297 #define SEN_CTX_MORE 0x01
298 #define SEN_CTX_TAIL 0x02
299 #define SEN_CTX_HEAD 0x04
300 #define SEN_CTX_QUIET 0x08
301 #define SEN_CTX_QUIT 0x10
303 #define SEN_CTX_USEQL 1
304 #define SEN_CTX_BATCHMODE 2
306 sen_ctx *sen_ctx_open(sen_db *db, int flags);
307 sen_ctx *sen_ctx_connect(const char *host, int port, int flags);
308 sen_rc sen_ctx_load(sen_ctx *c, const char *path);
309 sen_rc sen_ctx_send(sen_ctx *c, char *str, unsigned int str_len, int flags);
310 sen_rc sen_ctx_recv(sen_ctx *c, char **str, unsigned int *str_len, int *flags);
311 sen_rc sen_ctx_close(sen_ctx *c);
312 sen_rc sen_ctx_info_get(sen_ctx *c, sen_ctx_info *info);
314 /******** basic API ********/
316 sen_rc sen_init(void);
317 sen_rc sen_fin(void);
318 sen_rc sen_info(char **version,
319 char **configure_options,
321 sen_encoding *default_encoding,
322 unsigned int *initial_n_segments,
323 unsigned int *partial_match_threshold);
325 sen_index *sen_index_create(const char *path, int key_size, int flags,
326 int initial_n_segments, sen_encoding encoding);
327 sen_index *sen_index_open(const char *path);
328 sen_rc sen_index_close(sen_index *i);
329 sen_rc sen_index_remove(const char *path);
330 sen_rc sen_index_rename(const char *old_name, const char *new_name);
331 sen_rc sen_index_upd(sen_index *i, const void *key,
332 const char *oldvalue, unsigned int oldvalue_len,
333 const char *newvalue, unsigned int newvalue_len);
334 sen_records *sen_index_sel(sen_index *i,
335 const char *string, unsigned int string_len);
336 int sen_records_next(sen_records *r, void *keybuf, int buf_size, int *score);
337 sen_rc sen_records_rewind(sen_records *r);
338 int sen_records_curr_score(sen_records *r);
339 int sen_records_curr_key(sen_records *r, void *keybuf, int buf_size);
340 int sen_records_nhits(sen_records *r);
341 int sen_records_find(sen_records *r, const void *key);
342 sen_rc sen_records_close(sen_records *r);
344 /******** advanced API ********/
346 sen_values *sen_values_open(void);
347 sen_rc sen_values_close(sen_values *v);
348 sen_rc sen_values_add(sen_values *v,
349 const char *str, unsigned int str_len,
350 unsigned int weight);
352 sen_records *sen_records_open(sen_rec_unit record_unit,
353 sen_rec_unit subrec_unit,
354 unsigned int max_n_subrecs);
355 sen_records *sen_records_union(sen_records *a, sen_records *b);
356 sen_records *sen_records_subtract(sen_records *a, sen_records *b);
357 sen_records *sen_records_intersect(sen_records *a, sen_records *b);
358 int sen_records_difference(sen_records *a, sen_records *b);
359 sen_rc sen_records_sort(sen_records *r, int limit, sen_sort_optarg *optarg);
360 sen_rc sen_records_group(sen_records *r, int limit, sen_group_optarg *optarg);
361 const sen_recordh *sen_records_curr_rec(sen_records *r);
362 const sen_recordh *sen_records_at(sen_records *r, const void *key,
363 unsigned section, unsigned pos,
364 int *score, int *n_subrecs);
365 sen_rc sen_record_info(sen_records *r, const sen_recordh *rh,
366 void *keybuf, int buf_size, int *key_size,
367 int *section, int *pos, int *score, int *n_subrecs);
368 sen_rc sen_record_subrec_info(sen_records *r, const sen_recordh *rh,
369 int index, void *keybuf, int buf_size,
370 int *key_size, int *section, int *pos, int *score);
371 sen_index *sen_index_create_with_keys(const char *path, sen_sym *keys, int flags,
372 int initial_n_segments, sen_encoding encoding);
373 sen_index *sen_index_open_with_keys(const char *path, sen_sym *keys);
374 sen_index *sen_index_create_with_keys_lexicon(const char *path, sen_sym *keys,
375 sen_sym *lexicon, int initial_n_segments);
376 sen_index *sen_index_open_with_keys_lexicon(const char *path, sen_sym *keys,
378 sen_rc sen_index_update(sen_index *i, const void *key, unsigned int section,
379 sen_values *oldvalues, sen_values *newvalues);
380 sen_rc sen_index_select(sen_index *i,
381 const char *string, unsigned int string_len,
383 sen_sel_operator op, sen_select_optarg *optarg);
384 sen_rc sen_index_info(sen_index *i, int *key_size, int *flags,
385 int *initial_n_segments, sen_encoding *encoding,
386 unsigned *nrecords_keys, unsigned *file_size_keys,
387 unsigned *nrecords_lexicon, unsigned *file_size_lexicon,
388 unsigned long long *inv_seg_size,
389 unsigned long long *inv_chunk_size);
390 int sen_index_path(sen_index *i, char *pathbuf, int buf_size);
392 sen_set *sen_index_related_terms(sen_index *index, const char *string,
393 const char *(*fetcher)(void *, void *),
397 sen_query *sen_query_open(const char *str, unsigned int str_len,
398 sen_sel_operator default_op,
399 int max_exprs, sen_encoding encoding);
400 unsigned int sen_query_rest(sen_query *q, const char ** const rest);
401 sen_rc sen_query_close(sen_query *q);
402 sen_rc sen_query_exec(sen_index *i, sen_query *q, sen_records *r, sen_sel_operator op);
403 void sen_query_term(sen_query *q, query_term_callback func, void *func_arg);
404 sen_rc sen_query_scan(sen_query *q, const char **strs, unsigned int *str_lens,
405 unsigned int nstrs, int flags, int *found, int *score);
406 sen_snip *sen_query_snip(sen_query *query, int flags,
407 unsigned int width, unsigned int max_results,
409 const char **opentags, unsigned int *opentag_lens,
410 const char **closetags, unsigned int *closetag_lens,
411 sen_snip_mapping *mapping);
413 sen_rc sen_index_del(sen_index *i, const void *key);
415 /******** low level API ********/
417 sen_set *sen_set_open(unsigned key_size, unsigned value_size, unsigned init_size);
418 sen_rc sen_set_close(sen_set *set);
419 sen_rc sen_set_info(sen_set *set, unsigned *key_size,
420 unsigned *value_size, unsigned *n_entries);
421 sen_set_eh *sen_set_get(sen_set *set, const void *key, void **value);
422 sen_set_eh *sen_set_at(sen_set *set, const void *key, void **value);
423 sen_rc sen_set_del(sen_set *set, sen_set_eh *e);
424 sen_set_cursor *sen_set_cursor_open(sen_set *set);
425 sen_set_eh *sen_set_cursor_next(sen_set_cursor *cursor, void **key, void **value);
426 sen_rc sen_set_cursor_close(sen_set_cursor *cursor);
427 sen_rc sen_set_element_info(sen_set *set, const sen_set_eh *e,
428 void **key, void **value);
429 sen_set *sen_set_union(sen_set *a, sen_set *b);
430 sen_set *sen_set_subtract(sen_set *a, sen_set *b);
431 sen_set *sen_set_intersect(sen_set *a, sen_set *b);
432 int sen_set_difference(sen_set *a, sen_set *b);
433 sen_set_eh *sen_set_sort(sen_set *set, int limit, sen_set_sort_optarg *optarg);
435 sen_sym *sen_sym_create(const char *path, unsigned key_size,
436 unsigned flags, sen_encoding encoding);
437 sen_sym *sen_sym_open(const char *path);
438 sen_rc sen_sym_info(sen_sym *sym, int *key_size, unsigned *flags,
439 sen_encoding *encoding, unsigned *nrecords, unsigned *file_size);
440 sen_rc sen_sym_close(sen_sym *sym);
441 sen_rc sen_sym_remove(const char *path);
443 /* Lookup the sym table and find the id of the corresponding entry.
444 * If no matches are found, create a new entry, and return that ID
446 sen_id sen_sym_get(sen_sym *sym, const void *key);
448 /* Lookup the sym table and find the id of the corresponding entry.
449 * If no matches are found return SEN_SYM_NIL
451 sen_id sen_sym_at(sen_sym *sym, const void *key);
452 sen_rc sen_sym_del(sen_sym *sym, const void *key);
453 unsigned int sen_sym_size(sen_sym *sym);
454 int sen_sym_key(sen_sym *sym, sen_id id, void *keybuf, int buf_size);
455 sen_set *sen_sym_prefix_search(sen_sym *sym, const void *key);
456 sen_set *sen_sym_suffix_search(sen_sym *sym, const void *key);
457 sen_id sen_sym_common_prefix_search(sen_sym *sym, const void *key);
458 int sen_sym_pocket_get(sen_sym *sym, sen_id id);
459 sen_rc sen_sym_pocket_set(sen_sym *sym, sen_id id, unsigned int value);
460 sen_id sen_sym_next(sen_sym *sym, sen_id id);
461 int sen_sym_scan(sen_sym *sym, const char *str, unsigned int str_len,
462 sen_sym_scan_hit *sh, unsigned int sh_size, const char **rest);
464 /******** utility API ********/
465 sen_snip *sen_snip_open(sen_encoding encoding, int flags, unsigned int width,
466 unsigned int max_results,
467 const char *defaultopentag, unsigned int defaultopentag_len,
468 const char *defaultclosetag, unsigned int defaultclosetag_len,
469 sen_snip_mapping *mapping);
470 sen_rc sen_snip_close(sen_snip *snip);
471 sen_rc sen_snip_add_cond(sen_snip *snip,
472 const char *keyword, unsigned int keyword_len,
473 const char *opentag, unsigned int opentag_len,
474 const char *closetag, unsigned int closetag_len);
475 sen_rc sen_snip_exec(sen_snip *snip,
476 const char *string, unsigned int string_len,
477 unsigned int *nresults, unsigned int *max_tagged_len);
478 sen_rc sen_snip_get_result(sen_snip *snip, const unsigned int index,
479 char *result, unsigned int *result_len);
481 sen_records_heap *sen_records_heap_open(int size, int limit, sen_sort_optarg *optarg);
482 sen_rc sen_records_heap_add(sen_records_heap *h, sen_records *r);
483 int sen_records_heap_next(sen_records_heap *h);
484 sen_records *sen_records_heap_head(sen_records_heap *h);
485 sen_rc sen_records_heap_close(sen_records_heap *h);
487 int sen_inv_entry_info(sen_inv *inv, sen_id key, unsigned *a, unsigned *pocket,
488 unsigned *chunk, unsigned *chunk_size, unsigned *buffer_free,
489 unsigned *nterms, unsigned *nterms_void, unsigned *tid,
490 unsigned *size_in_chunk, unsigned *pos_in_chunk,
491 unsigned *size_in_buffer, unsigned *pos_in_buffer);
493 /* flags for sen_str_normalize */
494 #define SEN_STR_REMOVEBLANK 1
495 #define SEN_STR_WITH_CTYPES 2
496 #define SEN_STR_WITH_CHECKS 4
497 int sen_str_normalize(const char *str, unsigned int str_len,
498 sen_encoding encoding, int flags,
499 char *nstrbuf, int buf_size);
500 int fast_sen_str_normalize(const char *str, unsigned int str_len,
501 char *nstrbuf, int buf_size);
502 unsigned int sen_str_charlen(const char *str, sen_encoding encoding);
506 sen_rc sen_logger_info_set(const sen_logger_info *info);
508 void sen_logger_put(sen_log_level level,
509 const char *file, int line, const char *func, char *fmt, ...);
511 int sen_logger_pass(sen_log_level level);
513 #define SEN_LOG(level,...) \
514 if (sen_logger_pass(level)) {\
515 sen_logger_put((level), __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__);\
518 #ifndef SEN_LOG_DEFAULT_LEVEL
519 #define SEN_LOG_DEFAULT_LEVEL sen_log_notice
520 #endif /* SEN_LOG_DEFAULT_LEVEL */
522 #define sen_log(...) \
523 if (sen_logger_pass(SEN_LOG_DEFAULT_LEVEL)) {\
524 sen_logger_put(SEN_LOG_DEFAULT_LEVEL, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__);\