1 /*-------------------------------------------------------------------------
3 * Portions Copyright (c) 2007-2012, PostgreSQL Global Development Group
4 * Portions Copyright (c) 2013-2016, NTT DATA Corporation
8 * Support full text search using bigrams.
9 * Author: NTT DATA Corporation
11 *-------------------------------------------------------------------------
17 #include "access/gin.h"
18 #include "access/gin_private.h"
19 #include "access/itup.h"
20 #if PG_VERSION_NUM >= 120000
21 #include "access/relation.h"
23 #include "access/skey.h"
24 #if PG_VERSION_NUM < 130000
25 #include "access/tuptoaster.h"
27 #include "access/xlog.h"
28 #if PG_VERSION_NUM > 90500
29 #include "catalog/pg_am.h"
31 #include "catalog/pg_type.h"
33 #include "mb/pg_wchar.h"
34 #include "storage/bufmgr.h"
35 #include "storage/bufpage.h"
36 #include "tsearch/ts_locale.h"
37 #include "utils/array.h"
38 #include "utils/builtins.h"
39 #include "utils/rel.h"
42 PG_FUNCTION_INFO_V1(gin_extract_value_bigm);
43 PG_FUNCTION_INFO_V1(gin_extract_query_bigm);
44 PG_FUNCTION_INFO_V1(gin_bigm_consistent);
45 PG_FUNCTION_INFO_V1(gin_bigm_compare_partial);
46 PG_FUNCTION_INFO_V1(pg_gin_pending_stats);
48 /* triConsistent function is available only in 9.4 or later */
49 #if PG_VERSION_NUM >= 90400
50 PG_FUNCTION_INFO_V1(gin_bigm_triconsistent);
54 * The function prototypes are created as a part of PG_FUNCTION_INFO_V1
55 * macro since 9.4, and hence the declaration of the function prototypes
56 * here is necessary only for 9.3 or before.
58 #if PG_VERSION_NUM < 90400
59 Datum gin_extract_value_bigm(PG_FUNCTION_ARGS);
60 Datum gin_extract_query_bigm(PG_FUNCTION_ARGS);
61 Datum gin_bigm_consistent(PG_FUNCTION_ARGS);
62 Datum gin_bigm_compare_partial(PG_FUNCTION_ARGS);
63 Datum pg_gin_pending_stats(PG_FUNCTION_ARGS);
67 gin_extract_value_bigm(PG_FUNCTION_ARGS)
69 text *val = (text *) PG_GETARG_TEXT_P(0);
70 int32 *nentries = (int32 *) PG_GETARG_POINTER(1);
71 Datum *entries = NULL;
77 bgm = generate_bigm(VARDATA(val), VARSIZE(val) - VARHDRSZ);
78 bgmlen = ARRNELEM(bgm);
86 entries = (Datum *) palloc(sizeof(Datum) * bgmlen);
89 for (i = 0; i < bgmlen; i++)
91 text *item = cstring_to_text_with_len(ptr->str, ptr->bytelen);
93 entries[i] = PointerGetDatum(item);
98 PG_RETURN_POINTER(entries);
102 gin_extract_query_bigm(PG_FUNCTION_ARGS)
104 text *val = (text *) PG_GETARG_TEXT_P(0);
105 int32 *nentries = (int32 *) PG_GETARG_POINTER(1);
106 StrategyNumber strategy = PG_GETARG_UINT16(2);
108 bool **pmatch = (bool **) PG_GETARG_POINTER(3);
109 Pointer **extra_data = (Pointer **) PG_GETARG_POINTER(4);
111 /* bool **nullFlags = (bool **) PG_GETARG_POINTER(5); */
112 int32 *searchMode = (int32 *) PG_GETARG_POINTER(6);
113 Datum *entries = NULL;
122 case LikeStrategyNumber:
124 char *str = VARDATA(val);
125 int slen = VARSIZE(val) - VARHDRSZ;
129 * For wildcard search we extract all the bigrams that every
130 * potentially-matching string must include.
132 bgm = generate_wildcard_bigm(str, slen, &removeDups);
133 bgmlen = ARRNELEM(bgm);
136 * Check whether the heap tuple fetched by index search needs to
137 * be rechecked against the query. If the search word consists of
138 * one or two characters and doesn't contain any space character,
139 * we can guarantee that the index test would be exact. That is,
140 * the heap tuple does match the query, so it doesn't need to be
143 *extra_data = (Pointer *) palloc(sizeof(bool));
144 recheck = (bool *) *extra_data;
145 if (bgmlen == 1 && !removeDups)
150 for (sp = str; (sp - str) < slen;)
158 sp += IS_HIGHBIT_SET(*sp) ? pg_mblen(sp) : 1;
165 case SimilarityStrategyNumber:
167 bgm = generate_bigm(VARDATA(val), VARSIZE(val) - VARHDRSZ);
168 bgmlen = ARRNELEM(bgm);
172 elog(ERROR, "unrecognized strategy number: %d", strategy);
173 bgm = NULL; /* keep compiler quiet */
177 *nentries = (bigm_gin_key_limit == 0) ?
178 bgmlen : Min(bigm_gin_key_limit, bgmlen);
183 entries = (Datum *) palloc(sizeof(Datum) * *nentries);
185 for (i = 0; i < *nentries; i++)
192 *pmatch = (bool *) palloc0(sizeof(bool) * *nentries);
195 item = cstring_to_text_with_len(ptr->str, ptr->bytelen);
196 entries[i] = PointerGetDatum(item);
202 * If no bigram was extracted then we have to scan all the index.
205 *searchMode = GIN_SEARCH_MODE_ALL;
207 PG_RETURN_POINTER(entries);
211 gin_bigm_consistent(PG_FUNCTION_ARGS)
213 bool *check = (bool *) PG_GETARG_POINTER(0);
214 StrategyNumber strategy = PG_GETARG_UINT16(1);
216 /* text *query = PG_GETARG_TEXT_P(2); */
217 int32 nkeys = PG_GETARG_INT32(3);
219 Pointer *extra_data = (Pointer *) PG_GETARG_POINTER(4);
220 bool *recheck = (bool *) PG_GETARG_POINTER(5);
227 case LikeStrategyNumber:
230 * Don't recheck the heap tuple against the query if either
231 * pg_bigm.enable_recheck is disabled or the search word is the
232 * special one so that the index can return the exact result.
234 Assert(extra_data != NULL);
235 *recheck = bigm_enable_recheck &&
236 (*((bool *) extra_data) || (nkeys != 1));
238 /* Check if all extracted bigrams are presented. */
240 for (i = 0; i < nkeys; i++)
249 case SimilarityStrategyNumber:
250 /* Count the matches */
251 *recheck = bigm_enable_recheck;
253 for (i = 0; i < nkeys; i++)
259 /*--------------------
260 * If DIVUNION is defined then similarity formula is:
261 * c / (len1 + len2 - c)
262 * where c is number of common bigrams and it stands as ntrue in
263 * this code. Here we don't know value of len2 but we can assume
264 * that c (ntrue) is a lower bound of len2, so upper bound of
266 * c / (len1 + c - c) => c / len1
267 * If DIVUNION is not defined then similarity formula is:
268 * c / max(len1, len2)
269 * And again, c (ntrue) is a lower bound of len2, but c <= len1
270 * just by definition and, consequently, upper bound of
271 * similarity is just c / len1.
272 * So, independently on DIVUNION the upper bound formula is the same.
274 res = (nkeys == 0) ? false :
275 ((((((float4) ntrue) / ((float4) nkeys))) >=
276 (float4) bigm_similarity_limit) ? true : false);
279 elog(ERROR, "unrecognized strategy number: %d", strategy);
280 res = false; /* keep compiler quiet */
287 /* triConsistent function is available only in 9.4 or later */
288 #if PG_VERSION_NUM >= 90400
290 gin_bigm_triconsistent(PG_FUNCTION_ARGS)
292 GinTernaryValue *check = (GinTernaryValue *) PG_GETARG_POINTER(0);
293 StrategyNumber strategy = PG_GETARG_UINT16(1);
295 /* text *query = PG_GETARG_TEXT_P(2); */
296 int32 nkeys = PG_GETARG_INT32(3);
297 Pointer *extra_data = (Pointer *) PG_GETARG_POINTER(4);
298 GinTernaryValue res = GIN_MAYBE;
304 case LikeStrategyNumber:
306 * Don't recheck the heap tuple against the query if either
307 * pg_bigm.enable_recheck is disabled or the search word is the
308 * special one so that the index can return the exact result.
310 res = (bigm_enable_recheck &&
311 (*((bool *) extra_data) || (nkeys != 1))) ?
312 GIN_MAYBE : GIN_TRUE;
314 /* Check if all extracted bigrams are presented. */
315 for (i = 0; i < nkeys; i++)
317 if (check[i] == GIN_FALSE)
324 case SimilarityStrategyNumber:
325 /* Count the matches */
327 for (i = 0; i < nkeys; i++)
329 if (check[i] != GIN_FALSE)
334 * See comment in gin_bigm_consistent() about upper bound formula
336 res = (nkeys == 0) ? GIN_FALSE :
337 (((((float4) ntrue) / ((float4) nkeys)) >=
338 (float4) bigm_similarity_limit) ? GIN_MAYBE : GIN_FALSE);
340 if (res != GIN_FALSE && !bigm_enable_recheck)
344 elog(ERROR, "unrecognized strategy number: %d", strategy);
345 res = GIN_FALSE; /* keep compiler quiet */
349 PG_RETURN_GIN_TERNARY_VALUE(res);
351 #endif /* PG_VERSION_NUM >= 90400 */
354 gin_bigm_compare_partial(PG_FUNCTION_ARGS)
356 text *arg1 = PG_GETARG_TEXT_PP(0);
357 text *arg2 = PG_GETARG_TEXT_PP(1);
364 a1p = VARDATA_ANY(arg1);
365 a2p = VARDATA_ANY(arg2);
367 mblen1 = pg_mblen(a1p);
368 mblen2 = pg_mblen(a2p);
370 if (mblen1 != mblen2)
373 res = memcmp(a1p, a2p, mblen1) ? 1 : 0;
374 PG_RETURN_INT32(res);
378 * Report both number of pages and number of heap tuples that
379 * are in the pending list.
382 pg_gin_pending_stats(PG_FUNCTION_ARGS)
384 Oid indexOid = PG_GETARG_OID(0);
388 GinMetaPageData *metadata;
394 indexRel = relation_open(indexOid, AccessShareLock);
396 if (indexRel->rd_rel->relkind != RELKIND_INDEX ||
397 indexRel->rd_rel->relam != GIN_AM_OID)
399 (errcode(ERRCODE_WRONG_OBJECT_TYPE),
400 errmsg("relation \"%s\" is not a GIN index",
401 RelationGetRelationName(indexRel))));
404 * Reject attempts to read non-local temporary relations; we would be
405 * likely to get wrong data since we have no visibility into the owning
406 * session's local buffers.
408 if (RELATION_IS_OTHER_TEMP(indexRel))
410 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
411 errmsg("cannot access temporary indexes of other sessions")));
414 * Obtain statistic information from the meta page
416 metabuffer = ReadBuffer(indexRel, GIN_METAPAGE_BLKNO);
417 LockBuffer(metabuffer, GIN_SHARE);
418 metapage = BufferGetPage(metabuffer);
419 metadata = GinPageGetMeta(metapage);
422 * Construct a tuple descriptor for the result row. This must match this
423 * function's pg_bigm--x.x.sql entry.
425 #if PG_VERSION_NUM >= 120000
426 tupdesc = CreateTemplateTupleDesc(2);
428 tupdesc = CreateTemplateTupleDesc(2, false);
430 TupleDescInitEntry(tupdesc, (AttrNumber) 1,
431 "pages", INT4OID, -1, 0);
432 TupleDescInitEntry(tupdesc, (AttrNumber) 2,
433 "tuples", INT8OID, -1, 0);
434 tupdesc = BlessTupleDesc(tupdesc);
437 values[0] = Int32GetDatum(metadata->nPendingPages);
441 values[1] = Int64GetDatum(metadata->nPendingHeapTuples);
444 UnlockReleaseBuffer(metabuffer);
445 relation_close(indexRel, AccessShareLock);
447 tuple = heap_form_tuple(tupdesc, values, isnull);
448 PG_RETURN_DATUM(HeapTupleGetDatum(tuple));