Use more proper formula to calculate the number of bigram entries.

Previously we used the formula that we just borrowed from pg_trgm code,
to calculate the number of bigram entries. Based on the result of that
calculation we allocated the memory to store the bigram entries.
That borrowed formula was not incorrect, but not precise. It caused us to
allocate too much memory than necessary, and which limited the maximum
allowed size of column indexed by pg_bigm to smaller number.

This commit makes the formula more proper, and increases the column
limitation size to about 102MB. Also this installs some checks that the input
string length is not so large as to cause overflow in the calculations of
palloc request sizes. The addition of these checks was inspired by
the pg_trgm related commit c3ccc9e.

This change is simple and useful to alleviate the column size limitation.
So back-patch to all supported versions.

Beena Emerson, reviewed by me.

diff --git a/bigm_op.c b/bigm_op.c
index 617559b..93c77d2 100644 (file)
--- a/bigm_op.c
+++ b/bigm_op.c
@@ -19,6 +19,7 @@
 #include "catalog/pg_type.h"
 #include "tsearch/ts_locale.h"
 #include "utils/array.h"
+#include "utils/memutils.h"
 
 PG_MODULE_MAGIC;
 
@@ -250,7 +251,17 @@ generate_bigm(char *str, int slen)
        char       *bword,
                           *eword;
 
-       bgm = (BIGM *) palloc(VARHDRSZ + sizeof(bigm) * (Size) (slen / 2 + 1) * 3);
+       /*
+        * Guard against possible overflow in the palloc requests below.
+        * We need to prevent integer overflow in the multiplications here.
+        */
+       if ((Size) slen > (MaxAllocSize - VARHDRSZ) / sizeof(bigm) - 1 ||
+               (Size) slen > MaxAllocSize - 4)
+               ereport(ERROR,
+                               (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+                                errmsg("out of memory")));
+
+       bgm = (BIGM *) palloc(VARHDRSZ + sizeof(bigm) * (slen + 1));
        SET_VARSIZE(bgm, VARHDRSZ);
 
        if (slen + LPADDING + RPADDING < 2 || slen == 0)
@@ -480,7 +491,17 @@ generate_wildcard_bigm(const char *str, int slen, bool *removeDups)
 
        *removeDups = false;
 
-       bgm = (BIGM *) palloc(VARHDRSZ + sizeof(bigm) * (Size) (slen / 2 + 1) * 3);
+       /*
+        * Guard against possible overflow in the palloc requests below.
+        * We need to prevent integer overflow in the multiplications here.
+        */
+       if ((Size) slen > (MaxAllocSize - VARHDRSZ) / sizeof(bigm) - 1 ||
+               (Size) slen > MaxAllocSize - 4)
+               ereport(ERROR,
+                               (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+                                errmsg("out of memory")));
+
+       bgm = (BIGM *) palloc(VARHDRSZ + sizeof(bigm) * (slen + 1));
        SET_VARSIZE(bgm, VARHDRSZ);
 
        if (slen + LPADDING + RPADDING < 2 || slen == 0)

diff --git a/html/pg_bigm-1-1.html b/html/pg_bigm-1-1.html
index 3346332..179cac0 100644 (file)
--- a/html/pg_bigm-1-1.html
+++ b/html/pg_bigm-1-1.html
@@ -85,7 +85,7 @@
   <td>対応済(バージョン1.1以降)</td></tr>
 <tr><td>インデックス作成が可能な列の最大サイズ</td>
   <td nowrap>238,609,291Byte (約228MB)</td>
-  <td nowrap>71,582,785Byte (約68MB)</td></tr>
+  <td nowrap>107,374,180 Bytes (約102MB)</td></tr>
 </tbody>
 </table>
 
@@ -496,13 +496,13 @@ GINインデックスの待機リストとFASTUPDATEオプションの詳細は
 <h2 id="limitations">制約</h2>
 <h3 id="indexed_column_size">インデックス作成が可能な列の最大サイズ</h3>
 
-<p>pg_bigmでは、列サイズが71,582,785Byte (約68MB)を越える列へインデックスを作成できません。制限を越える大きさのデータを挿入するとエラーを起こします。</p>
+<p>pg_bigmでは、列サイズが107,374,180Byte (約102MB)を越える列へインデックスを作成できません。制限を越える大きさのデータを挿入するとエラーを起こします。</p>
 
 <pre>
 =# CREATE TABLE t1 (description text);
 =# CREATE INDEX t1_idx ON t1 USING gin (description gin_bigm_ops);
-=# INSERT INTO t1 SELECT repeat('A', 71582786);
-ERROR:  invalid memory alloc request size 1073741824
+=# INSERT INTO t1 SELECT repeat('A', 107374181);
+ERROR:  out of memory
 </pre>
 
 <p>この制約はpg_trgmにも存在しますが、pg_trgmでは最大238,609,291Byte (約228MB)までインデックスの作成が可能です。</p>

diff --git a/html/pg_bigm_en-1-1.html b/html/pg_bigm_en-1-1.html
index a2c69e4..38adc7d 100644 (file)
--- a/html/pg_bigm_en-1-1.html
+++ b/html/pg_bigm_en-1-1.html
@@ -85,7 +85,7 @@ For example, x.y is 1.1 and YYYYMMDD is 20131122 if the file of the version 1.1
   <td>Supported (version 1.1 or later)</td></tr>
 <tr><td>Maximum indexed column size</td>
   <td nowrap>238,609,291 Bytes (~228MB)</td>
-  <td nowrap>71,582,785 Bytes (~68MB)</td></tr>
+  <td nowrap>107,374,180 Bytes (~102MB)</td></tr>
 </tbody>
 </table>
 
@@ -495,13 +495,13 @@ Please see <a href="http://www.postgresql.org/docs/current/static/gin-implementa
 <h2 id="limitations">Limitations</h2>
 <h3 id="indexed_column_size">Indexed Column Size</h3>
 
-<p>The size of the column indexed by bigm GIN index cannot exceed 71,582,785 Bytes (~68MB). Any attempt to enter larger values will result in an error. </p>
+<p>The size of the column indexed by bigm GIN index cannot exceed 107,374,180 Bytes (~102MB). Any attempt to enter larger values will result in an error. </p>
 
 <pre>
 =# CREATE TABLE t1 (description text);
 =# CREATE INDEX t1_idx ON t1 USING gin (description gin_bigm_ops);
-=# INSERT INTO t1 SELECT repeat('A', 71582786);
-ERROR:  invalid memory alloc request size 1073741824
+=# INSERT INTO t1 SELECT repeat('A', 107374181);
+ERROR:  out of memory
 </pre>
 
 <p>pg_trgm also has this limitation. However, the maximum size in the case of trgm indexed column is 238,609,291 Bytes (~228MB).</p>