From cecb6075594a407b7adcd9c9a0c243ca4b43c9a3 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 17 Nov 2005 22:14:56 +0000 Subject: [PATCH] Make SQL arrays support null elements. This commit fixes the core array functionality, but I still need to make another pass looking at places that incidentally use arrays (such as ACL manipulation) to make sure they are null-safe. Contrib needs work too. I have not changed the behaviors that are still under discussion about array comparison and what to do with lower bounds. --- contrib/intagg/int_aggregate.c | 2 +- contrib/intarray/_int_tool.c | 5 +- contrib/tsearch2/query_rewrite.c | 2 +- doc/src/sgml/array.sgml | 110 +- doc/src/sgml/config.sgml | 83 +- doc/src/sgml/func.sgml | 26 +- doc/src/sgml/ref/insert.sgml | 8 +- doc/src/sgml/xfunc.sgml | 18 +- src/backend/catalog/pg_proc.c | 13 +- src/backend/executor/execQual.c | 244 ++-- src/backend/optimizer/util/clauses.c | 7 +- src/backend/utils/adt/acl.c | 4 +- src/backend/utils/adt/array_userfuncs.c | 112 +- src/backend/utils/adt/arrayfuncs.c | 1932 +++++++++++++++++-------- src/backend/utils/adt/arrayutils.c | 110 +- src/backend/utils/adt/float.c | 3 +- src/backend/utils/adt/int.c | 12 +- src/backend/utils/adt/numeric.c | 19 +- src/backend/utils/adt/oid.c | 12 +- src/backend/utils/adt/ruleutils.c | 4 +- src/backend/utils/adt/timestamp.c | 6 +- src/backend/utils/cache/lsyscache.c | 7 +- src/backend/utils/fmgr/funcapi.c | 11 +- src/backend/utils/misc/guc.c | 24 +- src/backend/utils/misc/postgresql.conf.sample | 5 +- src/include/c.h | 10 +- src/include/catalog/catversion.h | 4 +- src/include/catalog/pg_proc.h | 8 +- src/include/utils/acl.h | 6 +- src/include/utils/array.h | 127 +- src/pl/plpgsql/src/pl_comp.c | 6 +- src/pl/plpgsql/src/pl_exec.c | 28 +- src/test/regress/expected/arrays.out | 101 +- src/test/regress/expected/domain.out | 2 +- src/test/regress/sql/arrays.sql | 20 + 35 files changed, 2145 insertions(+), 946 deletions(-) diff --git a/contrib/intagg/int_aggregate.c b/contrib/intagg/int_aggregate.c index 3c7bb7ff87..afe5dd526f 100644 --- a/contrib/intagg/int_aggregate.c +++ b/contrib/intagg/int_aggregate.c @@ -87,7 +87,7 @@ GetPGArray(PGARRAY * p, AggState *aggstate, bool fAdd) p = (PGARRAY *) MemoryContextAlloc(aggstate->aggcontext, cb); p->a.size = cb; p->a.ndim = 1; - p->a.flags = 0; + p->a.dataoffset = 0; /* we don't support nulls, for now */ p->a.elemtype = INT4OID; p->items = 0; p->lower = START_NUM; diff --git a/contrib/intarray/_int_tool.c b/contrib/intarray/_int_tool.c index a3399874ad..13c5d1e9e2 100644 --- a/contrib/intarray/_int_tool.c +++ b/contrib/intarray/_int_tool.c @@ -208,12 +208,13 @@ ArrayType * new_intArrayType(int num) { ArrayType *r; - int nbytes = ARR_OVERHEAD(NDIM) + sizeof(int) * num; + int nbytes = ARR_OVERHEAD_NONULLS(NDIM) + sizeof(int) * num; r = (ArrayType *) palloc0(nbytes); ARR_SIZE(r) = nbytes; ARR_NDIM(r) = NDIM; + r->dataoffset = 0; /* marker for no null bitmap */ ARR_ELEMTYPE(r) = INT4OID; *((int *) ARR_DIMS(r)) = num; *((int *) ARR_LBOUND(r)) = 1; @@ -224,7 +225,7 @@ new_intArrayType(int num) ArrayType * resize_intArrayType(ArrayType *a, int num) { - int nbytes = ARR_OVERHEAD(NDIM) + sizeof(int) * num; + int nbytes = ARR_OVERHEAD_NONULLS(NDIM) + sizeof(int) * num; if (num == ARRNELEMS(a)) return a; diff --git a/contrib/tsearch2/query_rewrite.c b/contrib/tsearch2/query_rewrite.c index 163801c230..e3d40cc44d 100644 --- a/contrib/tsearch2/query_rewrite.c +++ b/contrib/tsearch2/query_rewrite.c @@ -232,7 +232,7 @@ rewrite_accum(PG_FUNCTION_ARGS) { if (ARR_ELEMTYPE(qa) != tsqOid) elog(ERROR, "array should contain tsquery type"); - deconstruct_array(qa, tsqOid, -1, false, 'i', &elemsp, &nelemsp); + deconstruct_array(qa, tsqOid, -1, false, 'i', &elemsp, NULL, &nelemsp); q = (QUERYTYPE*)DatumGetPointer( elemsp[0] ); if ( q->size == 0 ) { diff --git a/doc/src/sgml/array.sgml b/doc/src/sgml/array.sgml index 2d179fd7f1..c24646e43c 100644 --- a/doc/src/sgml/array.sgml +++ b/doc/src/sgml/array.sgml @@ -1,4 +1,4 @@ - + Arrays @@ -111,6 +111,13 @@ CREATE TABLE tictactoe ( + To set an element of an array constant to NULL, write NULL + for the element value. (Any upper- or lower-case variant of + NULL will do.) If you want an actual string value + NULL, you must put double quotes around it. + + + (These kinds of array constants are actually only a special case of the generic type constants discussed in . The constant is initially @@ -125,17 +132,6 @@ CREATE TABLE tictactoe ( INSERT INTO sal_emp VALUES ('Bill', '{10000, 10000, 10000, 10000}', - '{{"meeting", "lunch"}, {"meeting"}}'); -ERROR: multidimensional arrays must have array expressions with matching dimensions - - - Note that multidimensional arrays must have matching extents for each - dimension. A mismatch causes an error report. - - -INSERT INTO sal_emp - VALUES ('Bill', - '{10000, 10000, 10000, 10000}', '{{"meeting", "lunch"}, {"training", "presentation"}}'); INSERT INTO sal_emp @@ -145,15 +141,9 @@ INSERT INTO sal_emp - - A limitation of the present array implementation is that individual - elements of an array cannot be SQL null values. The entire array - can be set to null, but you can't have an array with some elements - null and some not. (This is likely to change in the future.) - - The result of the previous two inserts looks like this: + SELECT * FROM sal_emp; name | pay_by_quarter | schedule @@ -183,6 +173,19 @@ INSERT INTO sal_emp constructor syntax is discussed in more detail in . + + + Multidimensional arrays must have matching extents for each + dimension. A mismatch causes an error report, for example: + + +INSERT INTO sal_emp + VALUES ('Bill', + '{10000, 10000, 10000, 10000}', + '{{"meeting", "lunch"}, {"meeting"}}'); +ERROR: multidimensional arrays must have array expressions with matching dimensions + + @@ -262,14 +265,22 @@ SELECT schedule[1:2][2] FROM sal_emp WHERE name = 'Bill'; - Fetching from outside the current bounds of an array yields a - SQL null value, not an error. For example, if schedule + An array subscript expression will return null if either the array itself or + any of the subscript expressions are null. Also, null is returned if a + subscript is outside the array bounds (this case does not raise an error). + For example, if schedule currently has the dimensions [1:3][1:2] then referencing schedule[3][3] yields NULL. Similarly, an array reference with the wrong number of subscripts yields a null rather than an error. - Fetching an array slice that - is completely outside the current bounds likewise yields a null array; - but if the requested slice partially overlaps the array bounds, then it + + + + An array slice expression likewise yields null if the array itself or + any of the subscript expressions are null. However, in other corner + cases such as selecting an array slice that + is completely outside the current array bounds, a slice expression + yields an empty (zero-dimensional) array instead of null. + If the requested slice partially overlaps the array bounds, then it is silently reduced to just the overlapping region. @@ -349,7 +360,7 @@ UPDATE sal_emp SET pay_by_quarter[1:2] = '{27000,27000}' - Array slice assignment allows creation of arrays that do not use one-based + Subscripted assignment allows creation of arrays that do not use one-based subscripts. For example one might assign to myarray[-2:7] to create an array with subscript values running from -2 to 7. @@ -442,7 +453,7 @@ SELECT array_dims(ARRAY[1,2] || ARRAY[[3,4],[5,6]]); arrays, but array_cat supports multidimensional arrays. Note that the concatenation operator discussed above is preferred over - direct use of these functions. In fact, the functions are primarily for use + direct use of these functions. In fact, the functions exist primarily for use in implementing the concatenation operator. However, they may be directly useful in the creation of user-defined aggregates. Some examples: @@ -544,8 +555,9 @@ SELECT * FROM sal_emp WHERE 10000 = ALL (pay_by_quarter); The array output routine will put double quotes around element values - if they are empty strings or contain curly braces, delimiter characters, - double quotes, backslashes, or white space. Double quotes and backslashes + if they are empty strings, contain curly braces, delimiter characters, + double quotes, backslashes, or white space, or match the word + NULL. Double quotes and backslashes embedded in element values will be backslash-escaped. For numeric data types it is safe to assume that double quotes will never appear, but for textual data types one should be prepared to cope with either presence @@ -555,35 +567,15 @@ SELECT * FROM sal_emp WHERE 10000 = ALL (pay_by_quarter); By default, the lower bound index value of an array's dimensions is - set to one. If any of an array's dimensions has a lower bound index not - equal to one, an additional decoration that indicates the actual - array dimensions will precede the array structure decoration. + set to one. To represent arrays with other lower bounds, the array + subscript ranges can be specified explicitly before writing the + array contents. This decoration consists of square brackets ([]) around each array dimension's lower and upper bounds, with a colon (:) delimiter character in between. The array dimension decoration is followed by an equal sign (=). For example: -SELECT 1 || ARRAY[2,3] AS array; - - array ---------------- - [0:2]={1,2,3} -(1 row) - -SELECT ARRAY[1,2] || ARRAY[[3,4]] AS array; - - array --------------------------- - [0:1][1:2]={{1,2},{3,4}} -(1 row) - - - - - This syntax can also be used to specify non-default array subscripts - in an array literal. For example: - SELECT f1[1][-2][3] AS e1, f1[1][-1][5] AS e2 FROM (SELECT '[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int[] AS f1) AS ss; @@ -592,6 +584,18 @@ SELECT f1[1][-2][3] AS e1, f1[1][-1][5] AS e2 1 | 6 (1 row) + The array output routine will include explicit dimensions in its result + only when there are one or more lower bounds different from one. + + + + If the value written for an element is NULL (in any case + variant), the element is taken to be NULL. The presence of any quotes + or backslashes disables this and allows the literal string value + NULL to be entered. Also, for backwards compatibility with + pre-8.2 versions of PostgreSQL, the configuration parameter may be turned + off to suppress recognition of NULL as a NULL. @@ -600,7 +604,9 @@ SELECT f1[1][-2][3] AS e1, f1[1][-1][5] AS e2 if the element value would otherwise confuse the array-value parser. For example, elements containing curly braces, commas (or whatever the delimiter character is), double quotes, backslashes, or leading or trailing - whitespace must be double-quoted. To put a double quote or backslash in a + whitespace must be double-quoted. Empty strings and strings matching the + word NULL must be quoted, too. To put a double quote or + backslash in a quoted array element value, precede it with a backslash. Alternatively, you can use backslash-escaping to protect all data characters that would otherwise be taken as array syntax. diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index aabcebd7cd..89dc122327 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1,5 +1,5 @@ Server Configuration @@ -3614,6 +3614,7 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' Previous PostgreSQL Versions + @@ -3647,40 +3648,27 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' - - regex_flavor (string) - regular expressions + + array_nulls (boolean) - regex_flavor configuration parameter + array_nulls configuration parameter - The regular expression flavor can be set to - advanced, extended, or basic. - The default is advanced. The extended - setting may be useful for exact backwards compatibility with - pre-7.4 releases of PostgreSQL. See - for details. + This controls whether the array input parser recognizes + unquoted NULL as specifying a NULL array element. + By default, this is on, allowing array values containing + NULLs to be entered. However, PostgreSQL versions + before 8.2 did not support NULLs in arrays, and therefore would + treat NULL as specifying a normal array element with + the string value NULL. For backwards compatibility with + applications that require the old behavior, this variable can be + turned off. - - - - sql_inheritance (boolean) - - sql_inheritance configuration parameter - - inheritance - - This controls the inheritance semantics, in particular whether - subtables are included by various commands by default. They were - not included in versions prior to 7.1. If you need the old - behavior you can set this variable to off, but in - the long run you are encouraged to change your applications to - use the ONLY key word to exclude subtables. - See for more information about - inheritance. + Note that it is possible to create array values containing NULLs + even when this variable is off. @@ -3736,8 +3724,47 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' + + regex_flavor (string) + regular expressions + + regex_flavor configuration parameter + + + + The regular expression flavor can be set to + advanced, extended, or basic. + The default is advanced. The extended + setting may be useful for exact backwards compatibility with + pre-7.4 releases of PostgreSQL. See + for details. + + + + + + sql_inheritance (boolean) + + sql_inheritance configuration parameter + + inheritance + + + This controls the inheritance semantics, in particular whether + subtables are included by various commands by default. They were + not included in versions prior to 7.1. If you need the old + behavior you can set this variable to off, but in + the long run you are encouraged to change your applications to + use the ONLY key word to exclude subtables. + See for more information about + inheritance. + + + + + Platform and Client Compatibility diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 4b7a0cafba..8bc963b02f 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -1,5 +1,5 @@ @@ -8324,6 +8324,18 @@ AND + If the array expression yields a null array, the result of + ANY will be null. If the left-hand expression yields null, + the result of ANY is ordinarily null (though a non-strict + comparison operator could possibly yield a different result). + Also, if the right-hand array contains any null elements and no true + comparison result is obtained, the result of ANY + will be null, not false (again, assuming a strict comparison operator). + This is in accordance with SQL's normal rules for Boolean combinations + of null values. + + + SOME is a synonym for ANY. @@ -8346,6 +8358,18 @@ AND (including the special case where the array has zero elements). The result is false if any false result is found. + + + If the array expression yields a null array, the result of + ALL will be null. If the left-hand expression yields null, + the result of ALL is ordinarily null (though a non-strict + comparison operator could possibly yield a different result). + Also, if the right-hand array contains any null elements and no false + comparison result is obtained, the result of ALL + will be null, not true (again, assuming a strict comparison operator). + This is in accordance with SQL's normal rules for Boolean combinations + of null values. + diff --git a/doc/src/sgml/ref/insert.sgml b/doc/src/sgml/ref/insert.sgml index a3d03a745a..4e589b599b 100644 --- a/doc/src/sgml/ref/insert.sgml +++ b/doc/src/sgml/ref/insert.sgml @@ -1,5 +1,5 @@ @@ -206,11 +206,11 @@ INSERT INTO films SELECT * FROM tmp_films WHERE date_prod < '2004-05-07'; -- Create an empty 3x3 gameboard for noughts-and-crosses --- (these commands create the same board) INSERT INTO tictactoe (game, board[1:3][1:3]) - VALUES (1,'{{"","",""},{"","",""},{"","",""}}'); + VALUES (1, '{{" "," "," "},{" "," "," "},{" "," "," "}}'); +-- The subscripts in the above example aren't really needed INSERT INTO tictactoe (game, board) - VALUES (2,'{{,,},{,,},{,,}}'); + VALUES (2, '{{X," "," "},{" ",O," "},{" ",X," "}}'); diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index ff461884db..e5dfe9d48c 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -1,5 +1,5 @@ @@ -2790,6 +2790,7 @@ make_array(PG_FUNCTION_ARGS) ArrayType *result; Oid element_type = get_fn_expr_argtype(fcinfo->flinfo, 0); Datum element; + bool isnull; int16 typlen; bool typbyval; char typalign; @@ -2800,8 +2801,12 @@ make_array(PG_FUNCTION_ARGS) if (!OidIsValid(element_type)) elog(ERROR, "could not determine data type of input"); - /* get the provided element */ - element = PG_GETARG_DATUM(0); + /* get the provided element, being careful in case it's NULL */ + isnull = PG_ARGISNULL(0); + if (isnull) + element = (Datum) 0; + else + element = PG_GETARG_DATUM(0); /* we have one dimension */ ndims = 1; @@ -2814,7 +2819,7 @@ make_array(PG_FUNCTION_ARGS) get_typlenbyvalalign(element_type, &typlen, &typbyval, &typalign); /* now build the array */ - result = construct_md_array(&element, ndims, dims, lbs, + result = construct_md_array(&element, &isnull, ndims, dims, lbs, element_type, typlen, typbyval, typalign); PG_RETURN_ARRAYTYPE_P(result); @@ -2829,11 +2834,8 @@ make_array(PG_FUNCTION_ARGS) CREATE FUNCTION make_array(anyelement) RETURNS anyarray AS 'DIRECTORY/funcs', 'make_array' - LANGUAGE C STRICT; + LANGUAGE C IMMUTABLE; - - Note the use of STRICT; this is essential - since the code is not bothering to test for a null input. diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c index b2559a0e77..d443646724 100644 --- a/src/backend/catalog/pg_proc.c +++ b/src/backend/catalog/pg_proc.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/pg_proc.c,v 1.135 2005/10/29 00:31:50 petere Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/pg_proc.c,v 1.136 2005/11/17 22:14:51 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -119,12 +119,15 @@ ProcedureCreate(const char *procedureName, * need to use deconstruct_array() since the array data is just going * to look like a C array of OID values. */ - allParamCount = ARR_DIMS(DatumGetPointer(allParameterTypes))[0]; - if (ARR_NDIM(DatumGetPointer(allParameterTypes)) != 1 || + ArrayType *allParamArray = (ArrayType *) DatumGetPointer(allParameterTypes); + + allParamCount = ARR_DIMS(allParamArray)[0]; + if (ARR_NDIM(allParamArray) != 1 || allParamCount <= 0 || - ARR_ELEMTYPE(DatumGetPointer(allParameterTypes)) != OIDOID) + ARR_HASNULL(allParamArray) || + ARR_ELEMTYPE(allParamArray) != OIDOID) elog(ERROR, "allParameterTypes is not a 1-D Oid array"); - allParams = (Oid *) ARR_DATA_PTR(DatumGetPointer(allParameterTypes)); + allParams = (Oid *) ARR_DATA_PTR(allParamArray); Assert(allParamCount >= parameterCount); /* we assume caller got the contents right */ } diff --git a/src/backend/executor/execQual.c b/src/backend/executor/execQual.c index 4ee9a4ca62..7debc3fcd5 100644 --- a/src/backend/executor/execQual.c +++ b/src/backend/executor/execQual.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/executor/execQual.c,v 1.183 2005/10/19 22:30:30 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/executor/execQual.c,v 1.184 2005/11/17 22:14:51 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -202,16 +202,8 @@ static Datum ExecEvalRelabelType(GenericExprState *exprstate, * if it's a simple reference, or the modified array value if it's * an array assignment (i.e., array element or slice insertion). * - * NOTE: if we get a NULL result from a subexpression, we return NULL when - * it's an array reference, or the unmodified source array when it's an - * array assignment. This may seem peculiar, but if we return NULL (as was - * done in versions up through 7.0) then an assignment like - * UPDATE table SET arrayfield[4] = NULL - * will result in setting the whole array to NULL, which is certainly not - * very desirable. By returning the source array we make the assignment - * into a no-op, instead. (Eventually we need to redesign arrays so that - * individual elements can be NULL, but for now, let's try to protect users - * from shooting themselves in the foot.) + * NOTE: if we get a NULL result from a subscript expression, we return NULL + * when it's an array reference, or raise an error when it's an assignment. * * NOTE: we deliberately refrain from applying DatumGetArrayTypeP() here, * even though that might seem natural, because this code needs to support @@ -270,15 +262,15 @@ ExecEvalArrayRef(ArrayRefExprState *astate, econtext, &eisnull, NULL)); - /* If any index expr yields NULL, result is NULL or source array */ + /* If any index expr yields NULL, result is NULL or error */ if (eisnull) { - if (!isAssignment) - { - *isNull = true; - return (Datum) NULL; - } - return PointerGetDatum(array_source); + if (isAssignment) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("array subscript in assignment must not be NULL"))); + *isNull = true; + return (Datum) NULL; } } @@ -298,18 +290,15 @@ ExecEvalArrayRef(ArrayRefExprState *astate, econtext, &eisnull, NULL)); - - /* - * If any index expr yields NULL, result is NULL or source array - */ + /* If any index expr yields NULL, result is NULL or error */ if (eisnull) { - if (!isAssignment) - { - *isNull = true; - return (Datum) NULL; - } - return PointerGetDatum(array_source); + if (isAssignment) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("array subscript in assignment must not be NULL"))); + *isNull = true; + return (Datum) NULL; } } /* this can't happen unless parser messed up */ @@ -327,8 +316,8 @@ ExecEvalArrayRef(ArrayRefExprState *astate, /* * Evaluate the value to be assigned into the array. * - * XXX At some point we'll need to look into making the old value of the - * array element available via CaseTestExpr, as is done by + * XXX At some point we'll need to look into making the old value of + * the array element available via CaseTestExpr, as is done by * ExecEvalFieldStore. This is not needed now but will be needed to * support arrays of composite types; in an assignment to a field of * an array member, the parser would generate a FieldStore that @@ -340,29 +329,23 @@ ExecEvalArrayRef(ArrayRefExprState *astate, NULL); /* - * For now, can't cope with inserting NULL into an array, so make it a - * no-op per discussion above... + * For an assignment to a fixed-length array type, both the original + * array and the value to be assigned into it must be non-NULL, else + * we punt and return the original array. */ - if (eisnull) - return PointerGetDatum(array_source); + if (astate->refattrlength > 0) /* fixed-length array? */ + if (eisnull || *isNull) + return PointerGetDatum(array_source); /* - * For an assignment, if all the subscripts and the input expression - * are non-null but the original array is null, then substitute an - * empty (zero-dimensional) array and proceed with the assignment. - * This only works for varlena arrays, though; for fixed-length array - * types we punt and return the null input array. + * For assignment to varlena arrays, we handle a NULL original array + * by substituting an empty (zero-dimensional) array; insertion of + * the new element will result in a singleton array value. It does + * not matter whether the new element is NULL. */ if (*isNull) { - if (astate->refattrlength > 0) /* fixed-length array? */ - return PointerGetDatum(array_source); - - array_source = construct_md_array(NULL, 0, NULL, NULL, - arrayRef->refelemtype, - astate->refelemlength, - astate->refelembyval, - astate->refelemalign); + array_source = construct_empty_array(arrayRef->refelemtype); *isNull = false; } @@ -370,20 +353,20 @@ ExecEvalArrayRef(ArrayRefExprState *astate, resultArray = array_set(array_source, i, upper.indx, sourceData, + eisnull, astate->refattrlength, astate->refelemlength, astate->refelembyval, - astate->refelemalign, - isNull); + astate->refelemalign); else resultArray = array_set_slice(array_source, i, upper.indx, lower.indx, (ArrayType *) DatumGetPointer(sourceData), + eisnull, astate->refattrlength, astate->refelemlength, astate->refelembyval, - astate->refelemalign, - isNull); + astate->refelemalign); return PointerGetDatum(resultArray); } @@ -401,8 +384,7 @@ ExecEvalArrayRef(ArrayRefExprState *astate, astate->refattrlength, astate->refelemlength, astate->refelembyval, - astate->refelemalign, - isNull); + astate->refelemalign); return PointerGetDatum(resultArray); } } @@ -1620,6 +1602,8 @@ ExecEvalScalarArrayOp(ScalarArrayOpExprState *sstate, bool typbyval; char typalign; char *s; + bits8 *bitmap; + int bitmask; /* Set default values for result flags: non-null, not a set result */ *isNull = false; @@ -1668,9 +1652,8 @@ ExecEvalScalarArrayOp(ScalarArrayOpExprState *sstate, return BoolGetDatum(!useOr); /* - * If the scalar is NULL, and the function is strict, return NULL. This is - * just to avoid having to test for strictness inside the loop. (XXX but - * if arrays could have null elements, we'd need a test anyway.) + * If the scalar is NULL, and the function is strict, return NULL; + * no point in iterating the loop. */ if (fcinfo.argnull[0] && sstate->fxprstate.func.fn_strict) { @@ -1699,22 +1682,40 @@ ExecEvalScalarArrayOp(ScalarArrayOpExprState *sstate, /* Loop over the array elements */ s = (char *) ARR_DATA_PTR(arr); + bitmap = ARR_NULLBITMAP(arr); + bitmask = 1; + for (i = 0; i < nitems; i++) { Datum elt; Datum thisresult; - /* Get array element */ - elt = fetch_att(s, typbyval, typlen); - - s = att_addlength(s, typlen, PointerGetDatum(s)); - s = (char *) att_align(s, typalign); + /* Get array element, checking for NULL */ + if (bitmap && (*bitmap & bitmask) == 0) + { + fcinfo.arg[1] = (Datum) 0; + fcinfo.argnull[1] = true; + } + else + { + elt = fetch_att(s, typbyval, typlen); + s = att_addlength(s, typlen, PointerGetDatum(s)); + s = (char *) att_align(s, typalign); + fcinfo.arg[1] = elt; + fcinfo.argnull[1] = false; + } /* Call comparison function */ - fcinfo.arg[1] = elt; - fcinfo.argnull[1] = false; - fcinfo.isnull = false; - thisresult = FunctionCallInvoke(&fcinfo); + if (fcinfo.argnull[1] && sstate->fxprstate.func.fn_strict) + { + fcinfo.isnull = true; + thisresult = (Datum) 0; + } + else + { + fcinfo.isnull = false; + thisresult = FunctionCallInvoke(&fcinfo); + } /* Combine results per OR or AND semantics */ if (fcinfo.isnull) @@ -1737,6 +1738,17 @@ ExecEvalScalarArrayOp(ScalarArrayOpExprState *sstate, break; /* needn't look at any more elements */ } } + + /* advance bitmap pointer if any */ + if (bitmap) + { + bitmask <<= 1; + if (bitmask == 0x100) + { + bitmap++; + bitmask = 1; + } + } } *isNull = resultnull; @@ -2053,10 +2065,6 @@ ExecEvalCaseTestExpr(ExprState *exprstate, /* ---------------------------------------------------------------- * ExecEvalArray - ARRAY[] expressions - * - * NOTE: currently, if any input value is NULL then we return a NULL array, - * so the ARRAY[] construct can be considered strict. Eventually this will - * change; when it does, be sure to fix contain_nonstrict_functions(). * ---------------------------------------------------------------- */ static Datum @@ -2081,39 +2089,33 @@ ExecEvalArray(ArrayExprState *astate, ExprContext *econtext, /* Elements are presumably of scalar type */ int nelems; Datum *dvalues; + bool *dnulls; int i = 0; ndims = 1; nelems = list_length(astate->elements); - /* Shouldn't happen here, but if length is 0, return NULL */ + /* Shouldn't happen here, but if length is 0, return empty array */ if (nelems == 0) - { - *isNull = true; - return (Datum) 0; - } + return PointerGetDatum(construct_empty_array(element_type)); dvalues = (Datum *) palloc(nelems * sizeof(Datum)); + dnulls = (bool *) palloc(nelems * sizeof(bool)); /* loop through and build array of datums */ foreach(element, astate->elements) { ExprState *e = (ExprState *) lfirst(element); - bool eisnull; - dvalues[i++] = ExecEvalExpr(e, econtext, &eisnull, NULL); - if (eisnull) - { - *isNull = true; - return (Datum) 0; - } + dvalues[i] = ExecEvalExpr(e, econtext, &dnulls[i], NULL); + i++; } /* setup for 1-D array of the given length */ dims[0] = nelems; lbs[0] = 1; - result = construct_md_array(dvalues, ndims, dims, lbs, + result = construct_md_array(dvalues, dnulls, ndims, dims, lbs, element_type, astate->elemlength, astate->elembyval, @@ -2122,15 +2124,28 @@ ExecEvalArray(ArrayExprState *astate, ExprContext *econtext, else { /* Must be nested array expressions */ - char *dat = NULL; - Size ndatabytes = 0; - int nbytes; - int outer_nelems = list_length(astate->elements); + int nbytes = 0; + int nitems = 0; + int outer_nelems = 0; int elem_ndims = 0; int *elem_dims = NULL; int *elem_lbs = NULL; bool firstone = true; + bool havenulls = false; + char **subdata; + bits8 **subbitmaps; + int *subbytes; + int *subnitems; int i; + int32 dataoffset; + char *dat; + int iitem; + + i = list_length(astate->elements); + subdata = (char **) palloc(i * sizeof(char *)); + subbitmaps = (bits8 **) palloc(i * sizeof(bits8 *)); + subbytes = (int *) palloc(i * sizeof(int)); + subnitems = (int *) palloc(i * sizeof(int)); /* loop through and get data area from each element */ foreach(element, astate->elements) @@ -2139,14 +2154,11 @@ ExecEvalArray(ArrayExprState *astate, ExprContext *econtext, bool eisnull; Datum arraydatum; ArrayType *array; - int elem_ndatabytes; arraydatum = ExecEvalExpr(e, econtext, &eisnull, NULL); + /* ignore null subarrays */ if (eisnull) - { - *isNull = true; - return (Datum) 0; - } + continue; array = DatumGetArrayTypeP(arraydatum); @@ -2192,16 +2204,15 @@ ExecEvalArray(ArrayExprState *astate, ExprContext *econtext, "expressions with matching dimensions"))); } - elem_ndatabytes = ARR_SIZE(array) - ARR_OVERHEAD(elem_ndims); - ndatabytes += elem_ndatabytes; - if (dat == NULL) - dat = (char *) palloc(ndatabytes); - else - dat = (char *) repalloc(dat, ndatabytes); - - memcpy(dat + (ndatabytes - elem_ndatabytes), - ARR_DATA_PTR(array), - elem_ndatabytes); + subdata[outer_nelems] = ARR_DATA_PTR(array); + subbitmaps[outer_nelems] = ARR_NULLBITMAP(array); + subbytes[outer_nelems] = ARR_SIZE(array) - ARR_DATA_OFFSET(array); + nbytes += subbytes[outer_nelems]; + subnitems[outer_nelems] = ArrayGetNItems(ARR_NDIM(array), + ARR_DIMS(array)); + nitems += subnitems[outer_nelems]; + havenulls |= ARR_HASNULL(array); + outer_nelems++; } /* setup for multi-D array */ @@ -2213,20 +2224,37 @@ ExecEvalArray(ArrayExprState *astate, ExprContext *econtext, lbs[i] = elem_lbs[i - 1]; } - nbytes = ndatabytes + ARR_OVERHEAD(ndims); - result = (ArrayType *) palloc(nbytes); + if (havenulls) + { + dataoffset = ARR_OVERHEAD_WITHNULLS(ndims, nitems); + nbytes += dataoffset; + } + else + { + dataoffset = 0; /* marker for no null bitmap */ + nbytes += ARR_OVERHEAD_NONULLS(ndims); + } + result = (ArrayType *) palloc(nbytes); result->size = nbytes; result->ndim = ndims; - result->flags = 0; + result->dataoffset = dataoffset; result->elemtype = element_type; memcpy(ARR_DIMS(result), dims, ndims * sizeof(int)); memcpy(ARR_LBOUND(result), lbs, ndims * sizeof(int)); - if (ndatabytes > 0) - memcpy(ARR_DATA_PTR(result), dat, ndatabytes); - if (dat != NULL) - pfree(dat); + dat = ARR_DATA_PTR(result); + iitem = 0; + for (i = 0; i < outer_nelems; i++) + { + memcpy(dat, subdata[i], subbytes[i]); + dat += subbytes[i]; + if (havenulls) + array_bitmap_copy(ARR_NULLBITMAP(result), iitem, + subbitmaps[i], 0, + subnitems[i]); + iitem += subnitems[i]; + } } return PointerGetDatum(result); diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 5e2718dc63..088612cb4e 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/optimizer/util/clauses.c,v 1.201 2005/10/15 02:49:21 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/optimizer/util/clauses.c,v 1.202 2005/11/17 22:14:52 tgl Exp $ * * HISTORY * AUTHOR DATE MAJOR EVENT @@ -784,7 +784,7 @@ contain_nonstrict_functions_walker(Node *node, void *context) } if (IsA(node, ArrayRef)) { - /* array assignment is nonstrict */ + /* array assignment is nonstrict, but subscripting is strict */ if (((ArrayRef *) node)->refassgnexpr != NULL) return true; /* else fall through to check args */ @@ -842,7 +842,8 @@ contain_nonstrict_functions_walker(Node *node, void *context) return true; if (IsA(node, CaseWhen)) return true; - /* NB: ArrayExpr might someday be nonstrict */ + if (IsA(node, ArrayExpr)) + return true; if (IsA(node, RowExpr)) return true; if (IsA(node, CoalesceExpr)) diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 6d1402356e..a1080b59f6 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/acl.c,v 1.127 2005/11/04 17:25:15 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/acl.c,v 1.128 2005/11/17 22:14:52 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -352,7 +352,7 @@ allocacl(int n) new_acl = (Acl *) palloc0(size); new_acl->size = size; new_acl->ndim = 1; - new_acl->flags = 0; + new_acl->dataoffset = 0; /* we never put in any nulls */ new_acl->elemtype = ACLITEMOID; ARR_LBOUND(new_acl)[0] = 1; ARR_DIMS(new_acl)[0] = n; diff --git a/src/backend/utils/adt/array_userfuncs.c b/src/backend/utils/adt/array_userfuncs.c index 08a7072634..468e444e13 100644 --- a/src/backend/utils/adt/array_userfuncs.c +++ b/src/backend/utils/adt/array_userfuncs.c @@ -6,7 +6,7 @@ * Copyright (c) 2003-2005, PostgreSQL Global Development Group * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/array_userfuncs.c,v 1.16 2005/10/15 02:49:27 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/array_userfuncs.c,v 1.17 2005/11/17 22:14:52 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -19,6 +19,7 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" + /*----------------------------------------------------------------------------- * array_push : * push an element onto either end of a one-dimensional array @@ -29,11 +30,11 @@ array_push(PG_FUNCTION_ARGS) { ArrayType *v; Datum newelem; + bool isNull; int *dimv, *lb; ArrayType *result; int indx; - bool isNull; Oid element_type; int16 typlen; bool typbyval; @@ -54,15 +55,27 @@ array_push(PG_FUNCTION_ARGS) if (arg0_elemid != InvalidOid) { - v = PG_GETARG_ARRAYTYPE_P(0); - element_type = ARR_ELEMTYPE(v); - newelem = PG_GETARG_DATUM(1); + if (PG_ARGISNULL(0)) + v = construct_empty_array(arg0_elemid); + else + v = PG_GETARG_ARRAYTYPE_P(0); + isNull = PG_ARGISNULL(1); + if (isNull) + newelem = (Datum) 0; + else + newelem = PG_GETARG_DATUM(1); } else if (arg1_elemid != InvalidOid) { - v = PG_GETARG_ARRAYTYPE_P(1); - element_type = ARR_ELEMTYPE(v); - newelem = PG_GETARG_DATUM(0); + if (PG_ARGISNULL(1)) + v = construct_empty_array(arg1_elemid); + else + v = PG_GETARG_ARRAYTYPE_P(1); + isNull = PG_ARGISNULL(0); + if (isNull) + newelem = (Datum) 0; + else + newelem = PG_GETARG_DATUM(0); } else { @@ -73,6 +86,8 @@ array_push(PG_FUNCTION_ARGS) PG_RETURN_NULL(); /* keep compiler quiet */ } + element_type = ARR_ELEMTYPE(v); + if (ARR_NDIM(v) == 1) { lb = ARR_LBOUND(v); @@ -84,11 +99,21 @@ array_push(PG_FUNCTION_ARGS) int ub = dimv[0] + lb[0] - 1; indx = ub + 1; + /* overflow? */ + if (indx < ub) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("integer out of range"))); } else { /* prepend newelem */ indx = lb[0] - 1; + /* overflow? */ + if (indx > lb[0]) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("integer out of range"))); } } else if (ARR_NDIM(v) == 0) @@ -108,7 +133,7 @@ array_push(PG_FUNCTION_ARGS) fcinfo->flinfo->fn_extra = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, sizeof(ArrayMetaState)); my_extra = (ArrayMetaState *) fcinfo->flinfo->fn_extra; - my_extra->element_type = InvalidOid; + my_extra->element_type = ~element_type; } if (my_extra->element_type != element_type) @@ -124,8 +149,8 @@ array_push(PG_FUNCTION_ARGS) typbyval = my_extra->typbyval; typalign = my_extra->typalign; - result = array_set(v, 1, &indx, newelem, -1, - typlen, typbyval, typalign, &isNull); + result = array_set(v, 1, &indx, newelem, isNull, + -1, typlen, typbyval, typalign); PG_RETURN_ARRAYTYPE_P(result); } @@ -141,26 +166,46 @@ array_cat(PG_FUNCTION_ARGS) { ArrayType *v1, *v2; + ArrayType *result; int *dims, *lbs, ndims, + nitems, ndatabytes, nbytes; int *dims1, *lbs1, ndims1, + nitems1, ndatabytes1; int *dims2, *lbs2, ndims2, + nitems2, ndatabytes2; int i; char *dat1, *dat2; + bits8 *bitmap1, + *bitmap2; Oid element_type; Oid element_type1; Oid element_type2; - ArrayType *result; + int32 dataoffset; + + /* Concatenating a null array is a no-op, just return the other input */ + if (PG_ARGISNULL(0)) + { + if (PG_ARGISNULL(1)) + PG_RETURN_NULL(); + result = PG_GETARG_ARRAYTYPE_P(1); + PG_RETURN_ARRAYTYPE_P(result); + } + if (PG_ARGISNULL(1)) + { + result = PG_GETARG_ARRAYTYPE_P(0); + PG_RETURN_ARRAYTYPE_P(result); + } v1 = PG_GETARG_ARRAYTYPE_P(0); v2 = PG_GETARG_ARRAYTYPE_P(1); @@ -223,8 +268,12 @@ array_cat(PG_FUNCTION_ARGS) dims2 = ARR_DIMS(v2); dat1 = ARR_DATA_PTR(v1); dat2 = ARR_DATA_PTR(v2); - ndatabytes1 = ARR_SIZE(v1) - ARR_OVERHEAD(ndims1); - ndatabytes2 = ARR_SIZE(v2) - ARR_OVERHEAD(ndims2); + bitmap1 = ARR_NULLBITMAP(v1); + bitmap2 = ARR_NULLBITMAP(v2); + nitems1 = ArrayGetNItems(ndims1, dims1); + nitems2 = ArrayGetNItems(ndims2, dims2); + ndatabytes1 = ARR_SIZE(v1) - ARR_DATA_OFFSET(v1); + ndatabytes2 = ARR_SIZE(v2) - ARR_DATA_OFFSET(v2); if (ndims1 == ndims2) { @@ -310,20 +359,41 @@ array_cat(PG_FUNCTION_ARGS) } } + /* Do this mainly for overflow checking */ + nitems = ArrayGetNItems(ndims, dims); + /* build the result array */ ndatabytes = ndatabytes1 + ndatabytes2; - nbytes = ndatabytes + ARR_OVERHEAD(ndims); + if (ARR_HASNULL(v1) || ARR_HASNULL(v2)) + { + dataoffset = ARR_OVERHEAD_WITHNULLS(ndims, nitems); + nbytes = ndatabytes + dataoffset; + } + else + { + dataoffset = 0; /* marker for no null bitmap */ + nbytes = ndatabytes + ARR_OVERHEAD_NONULLS(ndims); + } result = (ArrayType *) palloc(nbytes); - result->size = nbytes; result->ndim = ndims; - result->flags = 0; + result->dataoffset = dataoffset; result->elemtype = element_type; memcpy(ARR_DIMS(result), dims, ndims * sizeof(int)); memcpy(ARR_LBOUND(result), lbs, ndims * sizeof(int)); /* data area is arg1 then arg2 */ memcpy(ARR_DATA_PTR(result), dat1, ndatabytes1); memcpy(ARR_DATA_PTR(result) + ndatabytes1, dat2, ndatabytes2); + /* handle the null bitmap if needed */ + if (ARR_HASNULL(result)) + { + array_bitmap_copy(ARR_NULLBITMAP(result), 0, + bitmap1, 0, + nitems1); + array_bitmap_copy(ARR_NULLBITMAP(result), nitems1, + bitmap2, 0, + nitems2); + } PG_RETURN_ARRAYTYPE_P(result); } @@ -347,10 +417,6 @@ create_singleton_array(FunctionCallInfo fcinfo, int i; ArrayMetaState *my_extra; - if (element_type == 0) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("invalid array element type OID: %u", element_type))); if (ndims < 1) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -379,7 +445,7 @@ create_singleton_array(FunctionCallInfo fcinfo, fcinfo->flinfo->fn_extra = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, sizeof(ArrayMetaState)); my_extra = (ArrayMetaState *) fcinfo->flinfo->fn_extra; - my_extra->element_type = InvalidOid; + my_extra->element_type = ~element_type; } if (my_extra->element_type != element_type) @@ -395,6 +461,6 @@ create_singleton_array(FunctionCallInfo fcinfo, typbyval = my_extra->typbyval; typalign = my_extra->typalign; - return construct_md_array(dvalues, ndims, dims, lbs, element_type, + return construct_md_array(dvalues, NULL, ndims, dims, lbs, element_type, typlen, typbyval, typalign); } diff --git a/src/backend/utils/adt/arrayfuncs.c b/src/backend/utils/adt/arrayfuncs.c index 5304d47fa8..3818b18190 100644 --- a/src/backend/utils/adt/arrayfuncs.c +++ b/src/backend/utils/adt/arrayfuncs.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/arrayfuncs.c,v 1.123 2005/10/15 02:49:27 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/arrayfuncs.c,v 1.124 2005/11/17 22:14:52 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -31,84 +31,73 @@ #include "utils/typcache.h" -/*---------- - * A standard varlena array has the following internal structure: - * - total number of bytes (also, TOAST info flags) - * - number of dimensions of the array - * - bit mask of flags - * - element type OID - * - size of each array axis (C array of int) - * - lower boundary of each dimension (C array of int) - * - whatever is the stored data - * The actual data starts on a MAXALIGN boundary. Individual items in the - * array are aligned as specified by the array element type. - * - * NOTE: it is important that array elements of toastable datatypes NOT be - * toasted, since the tupletoaster won't know they are there. (We could - * support compressed toasted items; only out-of-line items are dangerous. - * However, it seems preferable to store such items uncompressed and allow - * the toaster to compress the whole array as one input.) - * - * There is currently no support for NULL elements in arrays, either. - * A reasonable (and backwards-compatible) way to add support would be to - * add a nulls bitmap following the array, which would be present - * if needed; and its presence would be signaled by a bit in the flags word. - * - * - * There are also some "fixed-length array" datatypes, such as NAME and - * POINT. These are simply a sequence of a fixed number of items each - * of a fixed-length datatype, with no overhead; the item size must be - * a multiple of its alignment requirement, because we do no padding. - * We support subscripting on these types, but array_in() and array_out() - * only work with varlena arrays. - *---------- +/* + * GUC parameter */ +bool Array_nulls = true; - -/* ---------- +/* * Local definitions - * ---------- */ #define ASSGN "=" -#define RETURN_NULL(type) do { *isNull = true; return (type) 0; } while (0) +typedef enum +{ + ARRAY_NO_LEVEL, + ARRAY_LEVEL_STARTED, + ARRAY_ELEM_STARTED, + ARRAY_ELEM_COMPLETED, + ARRAY_QUOTED_ELEM_STARTED, + ARRAY_QUOTED_ELEM_COMPLETED, + ARRAY_ELEM_DELIMITED, + ARRAY_LEVEL_COMPLETED, + ARRAY_LEVEL_DELIMITED +} ArrayParseState; -static int ArrayCount(char *str, int *dim, char typdelim); -static Datum *ReadArrayStr(char *arrayStr, const char *origStr, +static int ArrayCount(const char *str, int *dim, char typdelim); +static void ReadArrayStr(char *arrayStr, const char *origStr, int nitems, int ndim, int *dim, FmgrInfo *inputproc, Oid typioparam, int32 typmod, char typdelim, int typlen, bool typbyval, char typalign, - int *nbytes); -static Datum *ReadArrayBinary(StringInfo buf, int nitems, + Datum *values, bool *nulls, + bool *hasnulls, int32 *nbytes); +static void ReadArrayBinary(StringInfo buf, int nitems, FmgrInfo *receiveproc, Oid typioparam, int32 typmod, int typlen, bool typbyval, char typalign, - int *nbytes); -static void CopyArrayEls(char *p, Datum *values, int nitems, - int typlen, bool typbyval, char typalign, - bool freedata); + Datum *values, bool *nulls, + bool *hasnulls, int32 *nbytes); +static void CopyArrayEls(ArrayType *array, + Datum *values, bool *nulls, int nitems, + int typlen, bool typbyval, char typalign, + bool freedata); +static bool array_get_isnull(const bits8 *nullbitmap, int offset); +static void array_set_isnull(bits8 *nullbitmap, int offset, bool isNull); static Datum ArrayCast(char *value, bool byval, int len); static int ArrayCastAndSet(Datum src, int typlen, bool typbyval, char typalign, char *dest); -static int array_nelems_size(char *ptr, int nitems, - int typlen, bool typbyval, char typalign); -static char *array_seek(char *ptr, int nitems, - int typlen, bool typbyval, char typalign); -static int array_copy(char *destptr, int nitems, char *srcptr, - int typlen, bool typbyval, char typalign); -static int array_slice_size(int ndim, int *dim, int *lb, char *arraydataptr, - int *st, int *endp, - int typlen, bool typbyval, char typalign); -static void array_extract_slice(int ndim, int *dim, int *lb, - char *arraydataptr, - int *st, int *endp, char *destPtr, - int typlen, bool typbyval, char typalign); -static void array_insert_slice(int ndim, int *dim, int *lb, - char *origPtr, int origdatasize, - char *destPtr, - int *st, int *endp, char *srcPtr, - int typlen, bool typbyval, char typalign); +static char *array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems, + int typlen, bool typbyval, char typalign); +static int array_nelems_size(char *ptr, int offset, bits8 *nullbitmap, + int nitems, int typlen, bool typbyval, char typalign); +static int array_copy(char *destptr, int nitems, + char *srcptr, int offset, bits8 *nullbitmap, + int typlen, bool typbyval, char typalign); +static int array_slice_size(char *arraydataptr, bits8 *arraynullsptr, + int ndim, int *dim, int *lb, + int *st, int *endp, + int typlen, bool typbyval, char typalign); +static void array_extract_slice(ArrayType *newarray, + int ndim, int *dim, int *lb, + char *arraydataptr, bits8 *arraynullsptr, + int *st, int *endp, + int typlen, bool typbyval, char typalign); +static void array_insert_slice(ArrayType *destArray, ArrayType *origArray, + ArrayType *srcArray, + int ndim, int *dim, int *lb, + int *st, int *endp, + int typlen, bool typbyval, char typalign); static int array_cmp(FunctionCallInfo fcinfo); static Datum array_type_length_coerce_internal(ArrayType *src, int32 desttypmod, @@ -116,13 +105,13 @@ static Datum array_type_length_coerce_internal(ArrayType *src, FmgrInfo *fmgr_info); -/*--------------------------------------------------------------------- +/* * array_in : * converts an array from the external format in "string" to * its internal format. + * * return value : * the internal representation of the input array - *-------------------------------------------------------------------- */ Datum array_in(PG_FUNCTION_ARGS) @@ -140,8 +129,11 @@ array_in(PG_FUNCTION_ARGS) *p; int i, nitems; - int32 nbytes; Datum *dataPtr; + bool *nullsPtr; + bool hasnulls; + int32 nbytes; + int32 dataoffset; ArrayType *retval; int ndim, dim[MAXDIM], @@ -189,8 +181,8 @@ array_in(PG_FUNCTION_ARGS) * Otherwise, we require the input to be in curly-brace style, and we * prescan the input to determine dimensions. * - * Dimension info takes the form of one or more [n] or [m:n] items. The outer - * loop iterates once per dimension item. + * Dimension info takes the form of one or more [n] or [m:n] items. + * The outer loop iterates once per dimension item. */ p = string_save; ndim = 0; @@ -310,60 +302,60 @@ array_in(PG_FUNCTION_ARGS) printf(") for %s\n", string); #endif + /* This checks for overflow of the array dimensions */ nitems = ArrayGetNItems(ndim, dim); + /* Empty array? */ if (nitems == 0) + PG_RETURN_ARRAYTYPE_P(construct_empty_array(element_type)); + + dataPtr = (Datum *) palloc(nitems * sizeof(Datum)); + nullsPtr = (bool *) palloc(nitems * sizeof(bool)); + ReadArrayStr(p, string, + nitems, ndim, dim, + &my_extra->proc, typioparam, typmod, + typdelim, + typlen, typbyval, typalign, + dataPtr, nullsPtr, + &hasnulls, &nbytes); + if (hasnulls) { - /* Return empty array */ - retval = (ArrayType *) palloc0(sizeof(ArrayType)); - retval->size = sizeof(ArrayType); - retval->elemtype = element_type; - PG_RETURN_ARRAYTYPE_P(retval); + dataoffset = ARR_OVERHEAD_WITHNULLS(ndim, nitems); + nbytes += dataoffset; } - - if (*p != '{') - ereport(ERROR, - (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("missing left brace"))); - dataPtr = ReadArrayStr(p, string, - nitems, ndim, dim, &my_extra->proc, typioparam, - typmod, typdelim, typlen, typbyval, typalign, - &nbytes); - nbytes += ARR_OVERHEAD(ndim); - retval = (ArrayType *) palloc0(nbytes); + else + { + dataoffset = 0; /* marker for no null bitmap */ + nbytes += ARR_OVERHEAD_NONULLS(ndim); + } + retval = (ArrayType *) palloc(nbytes); retval->size = nbytes; retval->ndim = ndim; + retval->dataoffset = dataoffset; retval->elemtype = element_type; memcpy(ARR_DIMS(retval), dim, ndim * sizeof(int)); memcpy(ARR_LBOUND(retval), lBound, ndim * sizeof(int)); - CopyArrayEls(ARR_DATA_PTR(retval), dataPtr, nitems, - typlen, typbyval, typalign, true); + CopyArrayEls(retval, + dataPtr, nullsPtr, nitems, + typlen, typbyval, typalign, + true); + pfree(dataPtr); + pfree(nullsPtr); pfree(string_save); + PG_RETURN_ARRAYTYPE_P(retval); } -/*----------------------------------------------------------------------------- +/* * ArrayCount - * Counts the number of dimensions and the *dim array for an array string. - * The syntax for array input is C-like nested curly braces - *----------------------------------------------------------------------------- + * Determines the dimensions for an array string. + * + * Returns number of dimensions as function result. The axis lengths are + * returned in dim[], which must be of size MAXDIM. */ -typedef enum -{ - ARRAY_NO_LEVEL, - ARRAY_LEVEL_STARTED, - ARRAY_ELEM_STARTED, - ARRAY_ELEM_COMPLETED, - ARRAY_QUOTED_ELEM_STARTED, - ARRAY_QUOTED_ELEM_COMPLETED, - ARRAY_ELEM_DELIMITED, - ARRAY_LEVEL_COMPLETED, - ARRAY_LEVEL_DELIMITED -} ArrayParseState; - static int -ArrayCount(char *str, int *dim, char typdelim) +ArrayCount(const char *str, int *dim, char typdelim) { int nest_level = 0, i; @@ -374,7 +366,7 @@ ArrayCount(char *str, int *dim, char typdelim) bool in_quotes = false; bool eoArray = false; bool empty_array = true; - char *ptr; + const char *ptr; ArrayParseState parse_state = ARRAY_NO_LEVEL; for (i = 0; i < MAXDIM; ++i) @@ -383,10 +375,6 @@ ArrayCount(char *str, int *dim, char typdelim) nelems_last[i] = nelems[i] = 1; } - /* special case for an empty array */ - if (strcmp(str, "{}") == 0) - return 0; - ptr = str; while (!eoArray) { @@ -588,24 +576,35 @@ ArrayCount(char *str, int *dim, char typdelim) return ndim; } -/*--------------------------------------------------------------------------- +/* * ReadArrayStr : - * parses the array string pointed by "arrayStr" and converts it to - * internal format. The external format expected is like C array - * declaration. Unspecified elements are initialized to zero for fixed length - * base types and to empty varlena structures for variable length base - * types. (This is pretty bogus; NULL would be much safer.) + * parses the array string pointed to by "arrayStr" and converts the values + * to internal format. Unspecified elements are initialized to nulls. + * The array dimensions must already have been determined. * - * result : - * returns a palloc'd array of Datum representations of the array elements. - * If element type is pass-by-ref, the Datums point to palloc'd values. - * *nbytes is set to the amount of data space needed for the array, - * including alignment padding but not including array header overhead. + * Inputs: + * arrayStr: the string to parse. + * CAUTION: the contents of "arrayStr" will be modified! + * origStr: the unmodified input string, used only in error messages. + * nitems: total number of array elements, as already determined. + * ndim: number of array dimensions + * dim[]: array axis lengths + * inputproc: type-specific input procedure for element datatype. + * typioparam, typmod: auxiliary values to pass to inputproc. + * typdelim: the value delimiter (type-specific). + * typlen, typbyval, typalign: storage parameters of element datatype. * - * CAUTION: the contents of "arrayStr" will be modified! - *--------------------------------------------------------------------------- + * Outputs: + * values[]: filled with converted data values. + * nulls[]: filled with is-null markers. + * *hasnulls: set TRUE iff there are any null elements. + * *nbytes: set to total size of data area needed (including alignment + * padding but not including array header overhead). + * + * Note that values[] and nulls[] are allocated by the caller, and must have + * nitems elements. */ -static Datum * +static void ReadArrayStr(char *arrayStr, const char *origStr, int nitems, @@ -618,31 +617,36 @@ ReadArrayStr(char *arrayStr, int typlen, bool typbyval, char typalign, - int *nbytes) + Datum *values, + bool *nulls, + bool *hasnulls, + int32 *nbytes) { int i, nest_level = 0; - Datum *values; char *srcptr; bool in_quotes = false; bool eoArray = false; - int totbytes; + bool hasnull; + int32 totbytes; int indx[MAXDIM], prod[MAXDIM]; mda_get_prod(ndim, dim, prod); - values = (Datum *) palloc0(nitems * sizeof(Datum)); MemSet(indx, 0, sizeof(indx)); + /* Initialize is-null markers to true */ + memset(nulls, true, nitems * sizeof(bool)); + /* * We have to remove " and \ characters to create a clean item value to * pass to the datatype input routine. We overwrite each item value * in-place within arrayStr to do this. srcptr is the current scan point, * and dstptr is where we are copying to. * - * We also want to suppress leading and trailing unquoted whitespace. We use - * the leadingspace flag to suppress leading space. Trailing space is - * tracked by using dstendptr to point to the last significant output + * We also want to suppress leading and trailing unquoted whitespace. + * We use the leadingspace flag to suppress leading space. Trailing space + * is tracked by using dstendptr to point to the last significant output * character. * * The error checking in this routine is mostly pro-forma, since we expect @@ -653,6 +657,7 @@ ReadArrayStr(char *arrayStr, { bool itemdone = false; bool leadingspace = true; + bool hasquoting = false; char *itemstart; char *dstptr; char *dstendptr; @@ -683,6 +688,7 @@ ReadArrayStr(char *arrayStr, /* Treat the escaped character as non-whitespace */ leadingspace = false; dstendptr = dstptr; + hasquoting = true; /* can't be a NULL marker */ break; case '\"': in_quotes = !in_quotes; @@ -697,6 +703,7 @@ ReadArrayStr(char *arrayStr, */ dstendptr = dstptr; } + hasquoting = true; /* can't be a NULL marker */ srcptr++; break; case '{': @@ -776,66 +783,57 @@ ReadArrayStr(char *arrayStr, errmsg("malformed array literal: \"%s\"", origStr))); - values[i] = FunctionCall3(inputproc, - CStringGetDatum(itemstart), - ObjectIdGetDatum(typioparam), - Int32GetDatum(typmod)); + if (Array_nulls && !hasquoting && + pg_strcasecmp(itemstart, "NULL") == 0) + { + /* it's a NULL item */ + nulls[i] = true; + } + else + { + values[i] = FunctionCall3(inputproc, + CStringGetDatum(itemstart), + ObjectIdGetDatum(typioparam), + Int32GetDatum(typmod)); + nulls[i] = false; + } } /* - * Initialize any unset items and compute total data space needed + * Check for nulls, compute total data space needed */ - if (typlen > 0) - { - totbytes = nitems * att_align(typlen, typalign); - if (!typbyval) - for (i = 0; i < nitems; i++) - if (values[i] == (Datum) 0) - values[i] = PointerGetDatum(palloc0(typlen)); - } - else + hasnull = false; + totbytes = 0; + for (i = 0; i < nitems; i++) { - Assert(!typbyval); - totbytes = 0; - for (i = 0; i < nitems; i++) + if (nulls[i]) + hasnull = true; + else { - if (values[i] != (Datum) 0) - { - /* let's just make sure data is not toasted */ - if (typlen == -1) - values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); - totbytes = att_addlength(totbytes, typlen, values[i]); - totbytes = att_align(totbytes, typalign); - } - else if (typlen == -1) - { - /* dummy varlena value (XXX bogus, see notes above) */ - values[i] = PointerGetDatum(palloc(sizeof(int32))); - VARATT_SIZEP(DatumGetPointer(values[i])) = sizeof(int32); - totbytes += sizeof(int32); - totbytes = att_align(totbytes, typalign); - } - else - { - /* dummy cstring value */ - Assert(typlen == -2); - values[i] = PointerGetDatum(palloc(1)); - *((char *) DatumGetPointer(values[i])) = '\0'; - totbytes += 1; - totbytes = att_align(totbytes, typalign); - } + /* let's just make sure data is not toasted */ + if (typlen == -1) + values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); + totbytes = att_addlength(totbytes, typlen, values[i]); + totbytes = att_align(totbytes, typalign); + /* check for overflow of total request */ + if (!AllocSizeIsValid(totbytes)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed (%d)", + (int) MaxAllocSize))); } } + *hasnulls = hasnull; *nbytes = totbytes; - return values; } -/*---------- +/* * Copy data into an array object from a temporary array of Datums. * - * p: pointer to start of array data area + * array: array object (with header fields already filled in) * values: array of Datums to be copied + * nulls: array of is-null flags (can be NULL if no nulls) * nitems: number of Datums to be copied * typbyval, typlen, typalign: info about element datatype * freedata: if TRUE and element type is pass-by-ref, pfree data values @@ -844,17 +842,21 @@ ReadArrayStr(char *arrayStr, * If the input data is of varlena type, the caller must have ensured that * the values are not toasted. (Doing it here doesn't work since the * caller has already allocated space for the array...) - *---------- */ static void -CopyArrayEls(char *p, +CopyArrayEls(ArrayType *array, Datum *values, + bool *nulls, int nitems, int typlen, bool typbyval, char typalign, bool freedata) { + char *p = ARR_DATA_PTR(array); + bits8 *bitmap = ARR_NULLBITMAP(array); + int bitval = 0; + int bitmask = 1; int i; if (typbyval) @@ -862,23 +864,45 @@ CopyArrayEls(char *p, for (i = 0; i < nitems; i++) { - p += ArrayCastAndSet(values[i], typlen, typbyval, typalign, p); - if (freedata) - pfree(DatumGetPointer(values[i])); + if (nulls && nulls[i]) + { + if (!bitmap) /* shouldn't happen */ + elog(ERROR, "null array element where not supported"); + /* bitmap bit stays 0 */ + } + else + { + bitval |= bitmask; + p += ArrayCastAndSet(values[i], typlen, typbyval, typalign, p); + if (freedata) + pfree(DatumGetPointer(values[i])); + } + if (bitmap) + { + bitmask <<= 1; + if (bitmask == 0x100) + { + *bitmap++ = bitval; + bitval = 0; + bitmask = 1; + } + } } + + if (bitmap && bitmask != 1) + *bitmap = bitval; } -/*------------------------------------------------------------------------- +/* * array_out : * takes the internal representation of an array and returns a string * containing the array in its external format. - *------------------------------------------------------------------------- */ Datum array_out(PG_FUNCTION_ARGS) { ArrayType *v = PG_GETARG_ARRAYTYPE_P(0); - Oid element_type; + Oid element_type = ARR_ELEMTYPE(v); int typlen; bool typbyval; char typalign; @@ -887,13 +911,14 @@ array_out(PG_FUNCTION_ARGS) *tmp, *retval, **values, - + dims_str[(MAXDIM * 33) + 2]; /* * 33 per dim since we assume 15 digits per number + ':' +'[]' * * +2 allows for assignment operator + trailing null */ - dims_str[(MAXDIM * 33) + 2]; + bits8 *bitmap; + int bitmask; bool *needquotes, needdims = false; int nitems, @@ -907,8 +932,6 @@ array_out(PG_FUNCTION_ARGS) *lb; ArrayMetaState *my_extra; - element_type = ARR_ELEMTYPE(v); - /* * We arrange to look up info about element type, including its output * conversion proc, only once per series of calls, assuming the element @@ -920,7 +943,7 @@ array_out(PG_FUNCTION_ARGS) fcinfo->flinfo->fn_extra = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, sizeof(ArrayMetaState)); my_extra = (ArrayMetaState *) fcinfo->flinfo->fn_extra; - my_extra->element_type = InvalidOid; + my_extra->element_type = ~element_type; } if (my_extra->element_type != element_type) @@ -972,41 +995,57 @@ array_out(PG_FUNCTION_ARGS) */ values = (char **) palloc(nitems * sizeof(char *)); needquotes = (bool *) palloc(nitems * sizeof(bool)); - p = ARR_DATA_PTR(v); overall_length = 1; /* don't forget to count \0 at end. */ + p = ARR_DATA_PTR(v); + bitmap = ARR_NULLBITMAP(v); + bitmask = 1; + for (i = 0; i < nitems; i++) { - Datum itemvalue; bool needquote; - itemvalue = fetch_att(p, typbyval, typlen); - values[i] = DatumGetCString(FunctionCall1(&my_extra->proc, - itemvalue)); - p = att_addlength(p, typlen, PointerGetDatum(p)); - p = (char *) att_align(p, typalign); - - /* count data plus backslashes; detect chars needing quotes */ - if (values[i][0] == '\0') - needquote = true; /* force quotes for empty string */ - else + /* Get source element, checking for NULL */ + if (bitmap && (*bitmap & bitmask) == 0) + { + values[i] = pstrdup("NULL"); + overall_length += 4; needquote = false; - - for (tmp = values[i]; *tmp != '\0'; tmp++) + } + else { - char ch = *tmp; + Datum itemvalue; + + itemvalue = fetch_att(p, typbyval, typlen); + values[i] = DatumGetCString(FunctionCall1(&my_extra->proc, + itemvalue)); + p = att_addlength(p, typlen, PointerGetDatum(p)); + p = (char *) att_align(p, typalign); + + /* count data plus backslashes; detect chars needing quotes */ + if (values[i][0] == '\0') + needquote = true; /* force quotes for empty string */ + else if (pg_strcasecmp(values[i], "NULL") == 0) + needquote = true; /* force quotes for literal NULL */ + else + needquote = false; - overall_length += 1; - if (ch == '"' || ch == '\\') + for (tmp = values[i]; *tmp != '\0'; tmp++) { - needquote = true; -#ifndef TCL_ARRAYS + char ch = *tmp; + overall_length += 1; + if (ch == '"' || ch == '\\') + { + needquote = true; +#ifndef TCL_ARRAYS + overall_length += 1; #endif + } + else if (ch == '{' || ch == '}' || ch == typdelim || + isspace((unsigned char) ch)) + needquote = true; } - else if (ch == '{' || ch == '}' || ch == typdelim || - isspace((unsigned char) ch)) - needquote = true; } needquotes[i] = needquote; @@ -1014,9 +1053,19 @@ array_out(PG_FUNCTION_ARGS) /* Count the pair of double quotes, if needed */ if (needquote) overall_length += 2; - /* and the comma */ overall_length += 1; + + /* advance bitmap pointer if any */ + if (bitmap) + { + bitmask <<= 1; + if (bitmask == 0x100) + { + bitmap++; + bitmask = 1; + } + } } /* @@ -1104,13 +1153,13 @@ array_out(PG_FUNCTION_ARGS) PG_RETURN_CSTRING(retval); } -/*--------------------------------------------------------------------- +/* * array_recv : * converts an array from the external binary format to * its internal format. + * * return value : * the internal representation of the input array - *-------------------------------------------------------------------- */ Datum array_recv(PG_FUNCTION_ARGS) @@ -1126,8 +1175,11 @@ array_recv(PG_FUNCTION_ARGS) Oid typioparam; int i, nitems; - int32 nbytes; Datum *dataPtr; + bool *nullsPtr; + bool hasnulls; + int32 nbytes; + int32 dataoffset; ArrayType *retval; int ndim, flags, @@ -1148,7 +1200,7 @@ array_recv(PG_FUNCTION_ARGS) ndim, MAXDIM))); flags = pq_getmsgint(buf, 4); - if (flags != 0) + if (flags != 0 && flags != 1) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("invalid array flags"))); @@ -1167,6 +1219,8 @@ array_recv(PG_FUNCTION_ARGS) dim[i] = pq_getmsgint(buf, 4); lBound[i] = pq_getmsgint(buf, 4); } + + /* This checks for overflow of array dimensions */ nitems = ArrayGetNItems(ndim, dim); /* @@ -1203,10 +1257,7 @@ array_recv(PG_FUNCTION_ARGS) if (nitems == 0) { /* Return empty array ... but not till we've validated element_type */ - retval = (ArrayType *) palloc0(sizeof(ArrayType)); - retval->size = sizeof(ArrayType); - retval->elemtype = element_type; - PG_RETURN_ARRAYTYPE_P(retval); + PG_RETURN_ARRAYTYPE_P(construct_empty_array(element_type)); } typlen = my_extra->typlen; @@ -1214,37 +1265,64 @@ array_recv(PG_FUNCTION_ARGS) typalign = my_extra->typalign; typioparam = my_extra->typioparam; - dataPtr = ReadArrayBinary(buf, nitems, &my_extra->proc, - typioparam, typmod, - typlen, typbyval, typalign, - &nbytes); - nbytes += ARR_OVERHEAD(ndim); - - retval = (ArrayType *) palloc0(nbytes); + dataPtr = (Datum *) palloc(nitems * sizeof(Datum)); + nullsPtr = (bool *) palloc(nitems * sizeof(bool)); + ReadArrayBinary(buf, nitems, + &my_extra->proc, typioparam, typmod, + typlen, typbyval, typalign, + dataPtr, nullsPtr, + &hasnulls, &nbytes); + if (hasnulls) + { + dataoffset = ARR_OVERHEAD_WITHNULLS(ndim, nitems); + nbytes += dataoffset; + } + else + { + dataoffset = 0; /* marker for no null bitmap */ + nbytes += ARR_OVERHEAD_NONULLS(ndim); + } + retval = (ArrayType *) palloc(nbytes); retval->size = nbytes; retval->ndim = ndim; + retval->dataoffset = dataoffset; retval->elemtype = element_type; memcpy(ARR_DIMS(retval), dim, ndim * sizeof(int)); memcpy(ARR_LBOUND(retval), lBound, ndim * sizeof(int)); - CopyArrayEls(ARR_DATA_PTR(retval), dataPtr, nitems, - typlen, typbyval, typalign, true); + CopyArrayEls(retval, + dataPtr, nullsPtr, nitems, + typlen, typbyval, typalign, + true); + pfree(dataPtr); + pfree(nullsPtr); PG_RETURN_ARRAYTYPE_P(retval); } -/*--------------------------------------------------------------------------- +/* * ReadArrayBinary: * collect the data elements of an array being read in binary style. - * result : - * returns a palloc'd array of Datum representations of the array elements. - * If element type is pass-by-ref, the Datums point to palloc'd values. - * *nbytes is set to the amount of data space needed for the array, - * including alignment padding but not including array header overhead. - *--------------------------------------------------------------------------- + * + * Inputs: + * buf: the data buffer to read from. + * nitems: total number of array elements (already read). + * receiveproc: type-specific receive procedure for element datatype. + * typioparam, typmod: auxiliary values to pass to receiveproc. + * typlen, typbyval, typalign: storage parameters of element datatype. + * + * Outputs: + * values[]: filled with converted data values. + * nulls[]: filled with is-null markers. + * *hasnulls: set TRUE iff there are any null elements. + * *nbytes: set to total size of data area needed (including alignment + * padding but not including array header overhead). + * + * Note that values[] and nulls[] are allocated by the caller, and must have + * nitems elements. */ -static Datum * +static void ReadArrayBinary(StringInfo buf, int nitems, FmgrInfo *receiveproc, @@ -1253,12 +1331,14 @@ ReadArrayBinary(StringInfo buf, int typlen, bool typbyval, char typalign, - int *nbytes) + Datum *values, + bool *nulls, + bool *hasnulls, + int32 *nbytes) { - Datum *values; int i; - - values = (Datum *) palloc(nitems * sizeof(Datum)); + bool hasnull; + int32 totbytes; for (i = 0; i < nitems; i++) { @@ -1268,11 +1348,18 @@ ReadArrayBinary(StringInfo buf, /* Get and check the item length */ itemlen = pq_getmsgint(buf, 4); - if (itemlen < 0 || itemlen > (buf->len - buf->cursor)) + if (itemlen < -1 || itemlen > (buf->len - buf->cursor)) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("insufficient data left in message"))); + if (itemlen == -1) + { + /* -1 length means NULL */ + nulls[i] = true; + continue; + } + /* * Rather than copying data around, we just set up a phony StringInfo * pointing to the correct portion of the input buffer. We assume we @@ -1294,6 +1381,7 @@ ReadArrayBinary(StringInfo buf, PointerGetDatum(&elem_buf), ObjectIdGetDatum(typioparam), Int32GetDatum(typmod)); + nulls[i] = false; /* Trouble if it didn't eat the whole buffer */ if (elem_buf.cursor != itemlen) @@ -1306,43 +1394,50 @@ ReadArrayBinary(StringInfo buf, } /* - * Compute total data space needed + * Check for nulls, compute total data space needed */ - if (typlen > 0) - *nbytes = nitems * att_align(typlen, typalign); - else + hasnull = false; + totbytes = 0; + for (i = 0; i < nitems; i++) { - Assert(!typbyval); - *nbytes = 0; - for (i = 0; i < nitems; i++) + if (nulls[i]) + hasnull = true; + else { /* let's just make sure data is not toasted */ if (typlen == -1) values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); - *nbytes = att_addlength(*nbytes, typlen, values[i]); - *nbytes = att_align(*nbytes, typalign); + totbytes = att_addlength(totbytes, typlen, values[i]); + totbytes = att_align(totbytes, typalign); + /* check for overflow of total request */ + if (!AllocSizeIsValid(totbytes)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed (%d)", + (int) MaxAllocSize))); } } - - return values; + *hasnulls = hasnull; + *nbytes = totbytes; } -/*------------------------------------------------------------------------- +/* * array_send : - * takes the internal representation of an array and returns a bytea + * takes the internal representation of an array and returns a bytea * containing the array in its external binary format. - *------------------------------------------------------------------------- */ Datum array_send(PG_FUNCTION_ARGS) { ArrayType *v = PG_GETARG_ARRAYTYPE_P(0); - Oid element_type; + Oid element_type = ARR_ELEMTYPE(v); int typlen; bool typbyval; char typalign; char *p; + bits8 *bitmap; + int bitmask; int nitems, i; int ndim, @@ -1350,9 +1445,6 @@ array_send(PG_FUNCTION_ARGS) StringInfoData buf; ArrayMetaState *my_extra; - /* Get information about the element type and the array dimensions */ - element_type = ARR_ELEMTYPE(v); - /* * We arrange to look up info about element type, including its send * conversion proc, only once per series of calls, assuming the element @@ -1364,7 +1456,7 @@ array_send(PG_FUNCTION_ARGS) fcinfo->flinfo->fn_extra = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt, sizeof(ArrayMetaState)); my_extra = (ArrayMetaState *) fcinfo->flinfo->fn_extra; - my_extra->element_type = InvalidOid; + my_extra->element_type = ~element_type; } if (my_extra->element_type != element_type) @@ -1395,7 +1487,7 @@ array_send(PG_FUNCTION_ARGS) /* Send the array header information */ pq_sendint(&buf, ndim, 4); - pq_sendint(&buf, v->flags, 4); + pq_sendint(&buf, ARR_HASNULL(v) ? 1 : 0, 4); pq_sendint(&buf, element_type, sizeof(Oid)); for (i = 0; i < ndim; i++) { @@ -1405,32 +1497,54 @@ array_send(PG_FUNCTION_ARGS) /* Send the array elements using the element's own sendproc */ p = ARR_DATA_PTR(v); + bitmap = ARR_NULLBITMAP(v); + bitmask = 1; + for (i = 0; i < nitems; i++) { - Datum itemvalue; - bytea *outputbytes; + /* Get source element, checking for NULL */ + if (bitmap && (*bitmap & bitmask) == 0) + { + /* -1 length means a NULL */ + pq_sendint(&buf, -1, 4); + } + else + { + Datum itemvalue; + bytea *outputbytes; - itemvalue = fetch_att(p, typbyval, typlen); + itemvalue = fetch_att(p, typbyval, typlen); - outputbytes = DatumGetByteaP(FunctionCall1(&my_extra->proc, - itemvalue)); - /* We assume the result will not have been toasted */ - pq_sendint(&buf, VARSIZE(outputbytes) - VARHDRSZ, 4); - pq_sendbytes(&buf, VARDATA(outputbytes), - VARSIZE(outputbytes) - VARHDRSZ); - pfree(outputbytes); + outputbytes = DatumGetByteaP(FunctionCall1(&my_extra->proc, + itemvalue)); + /* We assume the result will not have been toasted */ + pq_sendint(&buf, VARSIZE(outputbytes) - VARHDRSZ, 4); + pq_sendbytes(&buf, VARDATA(outputbytes), + VARSIZE(outputbytes) - VARHDRSZ); + pfree(outputbytes); - p = att_addlength(p, typlen, PointerGetDatum(p)); - p = (char *) att_align(p, typalign); + p = att_addlength(p, typlen, PointerGetDatum(p)); + p = (char *) att_align(p, typalign); + } + + /* advance bitmap pointer if any */ + if (bitmap) + { + bitmask <<= 1; + if (bitmask == 0x100) + { + bitmap++; + bitmask = 1; + } + } } PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } -/*----------------------------------------------------------------------------- +/* * array_dims : * returns the dimensions of the array pointed to by "v", as a "text" - *---------------------------------------------------------------------------- */ Datum array_dims(PG_FUNCTION_ARGS) @@ -1471,11 +1585,10 @@ array_dims(PG_FUNCTION_ARGS) PG_RETURN_TEXT_P(result); } -/*----------------------------------------------------------------------------- +/* * array_lower : * returns the lower dimension, of the DIM requested, for * the array pointed to by "v", as an int4 - *---------------------------------------------------------------------------- */ Datum array_lower(PG_FUNCTION_ARGS) @@ -1499,11 +1612,10 @@ array_lower(PG_FUNCTION_ARGS) PG_RETURN_INT32(result); } -/*----------------------------------------------------------------------------- +/* * array_upper : * returns the upper dimension, of the DIM requested, for * the array pointed to by "v", as an int4 - *---------------------------------------------------------------------------- */ Datum array_upper(PG_FUNCTION_ARGS) @@ -1530,18 +1642,32 @@ array_upper(PG_FUNCTION_ARGS) PG_RETURN_INT32(result); } -/*--------------------------------------------------------------------------- +/* * array_ref : - * This routine takes an array pointer and an index array and returns + * This routine takes an array pointer and a subscript array and returns * the referenced item as a Datum. Note that for a pass-by-reference * datatype, the returned Datum is a pointer into the array object. - *--------------------------------------------------------------------------- + * + * This handles both ordinary varlena arrays and fixed-length arrays. + * + * Inputs: + * array: the array object (mustn't be NULL) + * nSubscripts: number of subscripts supplied + * indx[]: the subscript values + * arraytyplen: pg_type.typlen for the array type + * elmlen: pg_type.typlen for the array's element type + * elmbyval: pg_type.typbyval for the array's element type + * elmalign: pg_type.typalign for the array's element type + * + * Outputs: + * The return value is the element Datum. + * *isNull is set to indicate whether the element is NULL. */ Datum array_ref(ArrayType *array, int nSubscripts, int *indx, - int arraylen, + int arraytyplen, int elmlen, bool elmbyval, char elmalign, @@ -1556,21 +1682,20 @@ array_ref(ArrayType *array, fixedLb[1]; char *arraydataptr, *retptr; + bits8 *arraynullsptr; - if (array == NULL) - RETURN_NULL(Datum); - - if (arraylen > 0) + if (arraytyplen > 0) { /* * fixed-length arrays -- these are assumed to be 1-d, 0-based */ ndim = 1; - fixedDim[0] = arraylen / elmlen; + fixedDim[0] = arraytyplen / elmlen; fixedLb[0] = 0; dim = fixedDim; lb = fixedLb; arraydataptr = (char *) array; + arraynullsptr = NULL; } else { @@ -1581,49 +1706,84 @@ array_ref(ArrayType *array, dim = ARR_DIMS(array); lb = ARR_LBOUND(array); arraydataptr = ARR_DATA_PTR(array); + arraynullsptr = ARR_NULLBITMAP(array); } /* * Return NULL for invalid subscript */ if (ndim != nSubscripts || ndim <= 0 || ndim > MAXDIM) - RETURN_NULL(Datum); + { + *isNull = true; + return (Datum) 0; + } for (i = 0; i < ndim; i++) + { if (indx[i] < lb[i] || indx[i] >= (dim[i] + lb[i])) - RETURN_NULL(Datum); + { + *isNull = true; + return (Datum) 0; + } + } /* - * OK, get the element + * Calculate the element number */ offset = ArrayGetOffset(nSubscripts, dim, lb, indx); - retptr = array_seek(arraydataptr, offset, elmlen, elmbyval, elmalign); + /* + * Check for NULL array element + */ + if (array_get_isnull(arraynullsptr, offset)) + { + *isNull = true; + return (Datum) 0; + } + /* + * OK, get the element + */ *isNull = false; + retptr = array_seek(arraydataptr, 0, arraynullsptr, offset, + elmlen, elmbyval, elmalign); return ArrayCast(retptr, elmbyval, elmlen); } -/*----------------------------------------------------------------------------- +/* * array_get_slice : * This routine takes an array and a range of indices (upperIndex and * lowerIndx), creates a new array structure for the referred elements * and returns a pointer to it. * - * NOTE: we assume it is OK to scribble on the provided index arrays + * This handles both ordinary varlena arrays and fixed-length arrays. + * + * Inputs: + * array: the array object (mustn't be NULL) + * nSubscripts: number of subscripts supplied (must be same for upper/lower) + * upperIndx[]: the upper subscript values + * lowerIndx[]: the lower subscript values + * arraytyplen: pg_type.typlen for the array type + * elmlen: pg_type.typlen for the array's element type + * elmbyval: pg_type.typbyval for the array's element type + * elmalign: pg_type.typalign for the array's element type + * + * Outputs: + * The return value is the new array Datum (it's never NULL) + * + * NOTE: we assume it is OK to scribble on the provided subscript arrays * lowerIndx[] and upperIndx[]. These are generally just temporaries. - *----------------------------------------------------------------------------- */ ArrayType * array_get_slice(ArrayType *array, int nSubscripts, int *upperIndx, int *lowerIndx, - int arraylen, + int arraytyplen, int elmlen, bool elmbyval, - char elmalign, - bool *isNull) + char elmalign) { + ArrayType *newarray; int i, ndim, *dim, @@ -1631,15 +1791,14 @@ array_get_slice(ArrayType *array, *newlb; int fixedDim[1], fixedLb[1]; + Oid elemtype; char *arraydataptr; - ArrayType *newarray; + bits8 *arraynullsptr; + int32 dataoffset; int bytes, span[MAXDIM]; - if (array == NULL) - RETURN_NULL(ArrayType *); - - if (arraylen > 0) + if (arraytyplen > 0) { /* * fixed-length arrays -- currently, cannot slice these because parser @@ -1652,15 +1811,18 @@ array_get_slice(ArrayType *array, errmsg("slices of fixed-length arrays not implemented"))); /* - * fixed-length arrays -- these are assumed to be 1-d, 0-based XXX - * where would we get the correct ELEMTYPE from? + * fixed-length arrays -- these are assumed to be 1-d, 0-based + * + * XXX where would we get the correct ELEMTYPE from? */ ndim = 1; - fixedDim[0] = arraylen / elmlen; + fixedDim[0] = arraytyplen / elmlen; fixedLb[0] = 0; dim = fixedDim; lb = fixedLb; + elemtype = InvalidOid; /* XXX */ arraydataptr = (char *) array; + arraynullsptr = NULL; } else { @@ -1670,16 +1832,18 @@ array_get_slice(ArrayType *array, ndim = ARR_NDIM(array); dim = ARR_DIMS(array); lb = ARR_LBOUND(array); + elemtype = ARR_ELEMTYPE(array); arraydataptr = ARR_DATA_PTR(array); + arraynullsptr = ARR_NULLBITMAP(array); } /* * Check provided subscripts. A slice exceeding the current array limits * is silently truncated to the array limits. If we end up with an empty - * slice, return NULL (should it be an empty array instead?) + * slice, return an empty array. */ if (ndim < nSubscripts || ndim <= 0 || ndim > MAXDIM) - RETURN_NULL(ArrayType *); + return construct_empty_array(elemtype); for (i = 0; i < nSubscripts; i++) { @@ -1688,7 +1852,7 @@ array_get_slice(ArrayType *array, if (upperIndx[i] >= (dim[i] + lb[i])) upperIndx[i] = dim[i] + lb[i] - 1; if (lowerIndx[i] > upperIndx[i]) - RETURN_NULL(ArrayType *); + return construct_empty_array(elemtype); } /* fill any missing subscript positions with full array range */ for (; i < ndim; i++) @@ -1696,21 +1860,36 @@ array_get_slice(ArrayType *array, lowerIndx[i] = lb[i]; upperIndx[i] = dim[i] + lb[i] - 1; if (lowerIndx[i] > upperIndx[i]) - RETURN_NULL(ArrayType *); + return construct_empty_array(elemtype); } mda_get_range(ndim, span, lowerIndx, upperIndx); - bytes = array_slice_size(ndim, dim, lb, arraydataptr, + bytes = array_slice_size(arraydataptr, arraynullsptr, + ndim, dim, lb, lowerIndx, upperIndx, elmlen, elmbyval, elmalign); - bytes += ARR_OVERHEAD(ndim); + + /* + * Currently, we put a null bitmap in the result if the source has one; + * could be smarter ... + */ + if (arraynullsptr) + { + dataoffset = ARR_OVERHEAD_WITHNULLS(ndim, ArrayGetNItems(ndim, span)); + bytes += dataoffset; + } + else + { + dataoffset = 0; /* marker for no null bitmap */ + bytes += ARR_OVERHEAD_NONULLS(ndim); + } newarray = (ArrayType *) palloc(bytes); newarray->size = bytes; newarray->ndim = ndim; - newarray->flags = 0; - newarray->elemtype = ARR_ELEMTYPE(array); + newarray->dataoffset = dataoffset; + newarray->elemtype = elemtype; memcpy(ARR_DIMS(newarray), span, ndim * sizeof(int)); /* @@ -1721,63 +1900,77 @@ array_get_slice(ArrayType *array, for (i = 0; i < ndim; i++) newlb[i] = 1; - array_extract_slice(ndim, dim, lb, arraydataptr, - lowerIndx, upperIndx, ARR_DATA_PTR(newarray), + array_extract_slice(newarray, + ndim, dim, lb, + arraydataptr, arraynullsptr, + lowerIndx, upperIndx, elmlen, elmbyval, elmalign); return newarray; } -/*----------------------------------------------------------------------------- +/* * array_set : - * This routine sets the value of an array location (specified by - * an index array) to a new value specified by "dataValue". - * result : + * This routine sets the value of an array element (specified by + * a subscript array) to a new value specified by "dataValue". + * + * This handles both ordinary varlena arrays and fixed-length arrays. + * + * Inputs: + * array: the initial array object (mustn't be NULL) + * nSubscripts: number of subscripts supplied + * indx[]: the subscript values + * dataValue: the datum to be inserted at the given position + * isNull: whether dataValue is NULL + * arraytyplen: pg_type.typlen for the array type + * elmlen: pg_type.typlen for the array's element type + * elmbyval: pg_type.typbyval for the array's element type + * elmalign: pg_type.typalign for the array's element type + * + * Result: * A new array is returned, just like the old except for the one - * modified entry. + * modified entry. The original array object is not changed. * * For one-dimensional arrays only, we allow the array to be extended * by assigning to the position one above or one below the existing range. - * (We could be more flexible if we had a way to represent NULL elements.) + * (XXX we could be more flexible: perhaps allow NULL fill?) * * NOTE: For assignments, we throw an error for invalid subscripts etc, - * rather than returning a NULL as the fetch operations do. The reasoning - * is that returning a NULL would cause the user's whole array to be replaced - * with NULL, which will probably not make him happy. - *----------------------------------------------------------------------------- + * rather than returning a NULL as the fetch operations do. */ ArrayType * array_set(ArrayType *array, int nSubscripts, int *indx, Datum dataValue, - int arraylen, + bool isNull, + int arraytyplen, int elmlen, bool elmbyval, - char elmalign, - bool *isNull) + char elmalign) { + ArrayType *newarray; int i, ndim, dim[MAXDIM], lb[MAXDIM], offset; - ArrayType *newarray; char *elt_ptr; bool extendbefore = false; bool extendafter = false; - int olddatasize, + bool newhasnulls; + bits8 *oldnullbitmap; + int oldnitems, + olddatasize, newsize, olditemlen, newitemlen, overheadlen, + oldoverheadlen, lenbefore, lenafter; - if (array == NULL) - RETURN_NULL(ArrayType *); - - if (arraylen > 0) + if (arraytyplen > 0) { /* * fixed-length arrays -- these are assumed to be 1-d, 0-based. We @@ -1788,20 +1981,30 @@ array_set(ArrayType *array, (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), errmsg("invalid array subscripts"))); - if (indx[0] < 0 || indx[0] * elmlen >= arraylen) + if (indx[0] < 0 || indx[0] * elmlen >= arraytyplen) ereport(ERROR, (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), errmsg("invalid array subscripts"))); - newarray = (ArrayType *) palloc(arraylen); - memcpy(newarray, array, arraylen); + if (isNull) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("cannot assign NULL to an element of a fixed-length array"))); + + newarray = (ArrayType *) palloc(arraytyplen); + memcpy(newarray, array, arraytyplen); elt_ptr = (char *) newarray + indx[0] * elmlen; ArrayCastAndSet(dataValue, elmlen, elmbyval, elmalign, elt_ptr); return newarray; } + if (nSubscripts <= 0 || nSubscripts > MAXDIM) + ereport(ERROR, + (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), + errmsg("invalid array subscripts"))); + /* make sure item to be inserted is not toasted */ - if (elmlen == -1) + if (elmlen == -1 && !isNull) dataValue = PointerGetDatum(PG_DETOAST_DATUM(dataValue)); /* detoast input array if necessary */ @@ -1824,11 +2027,12 @@ array_set(ArrayType *array, lb[i] = indx[i]; } - return construct_md_array(&dataValue, nSubscripts, dim, lb, elmtype, + return construct_md_array(&dataValue, &isNull, nSubscripts, + dim, lb, elmtype, elmlen, elmbyval, elmalign); } - if (ndim != nSubscripts || ndim <= 0 || ndim > MAXDIM) + if (ndim != nSubscripts) ereport(ERROR, (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), errmsg("invalid array subscripts"))); @@ -1872,16 +2076,31 @@ array_set(ArrayType *array, /* * Compute sizes of items and areas to copy */ - overheadlen = ARR_OVERHEAD(ndim); - olddatasize = ARR_SIZE(array) - overheadlen; + if (ARR_HASNULL(array) || isNull) + { + newhasnulls = true; + overheadlen = ARR_OVERHEAD_WITHNULLS(ndim, + ArrayGetNItems(ndim, dim)); + } + else + { + newhasnulls = false; + overheadlen = ARR_OVERHEAD_NONULLS(ndim); + } + oldnitems = ArrayGetNItems(ndim, ARR_DIMS(array)); + oldnullbitmap = ARR_NULLBITMAP(array); + oldoverheadlen = ARR_DATA_OFFSET(array); + olddatasize = ARR_SIZE(array) - oldoverheadlen; if (extendbefore) { + offset = 0; lenbefore = 0; olditemlen = 0; lenafter = olddatasize; } else if (extendafter) { + offset = oldnitems; lenbefore = olddatasize; olditemlen = 0; lenafter = 0; @@ -1889,59 +2108,112 @@ array_set(ArrayType *array, else { offset = ArrayGetOffset(nSubscripts, dim, lb, indx); - elt_ptr = array_seek(ARR_DATA_PTR(array), offset, + elt_ptr = array_seek(ARR_DATA_PTR(array), 0, oldnullbitmap, offset, elmlen, elmbyval, elmalign); lenbefore = (int) (elt_ptr - ARR_DATA_PTR(array)); - olditemlen = att_addlength(0, elmlen, PointerGetDatum(elt_ptr)); - olditemlen = att_align(olditemlen, elmalign); + if (array_get_isnull(oldnullbitmap, offset)) + olditemlen = 0; + else + { + olditemlen = att_addlength(0, elmlen, PointerGetDatum(elt_ptr)); + olditemlen = att_align(olditemlen, elmalign); + } lenafter = (int) (olddatasize - lenbefore - olditemlen); } - newitemlen = att_addlength(0, elmlen, dataValue); - newitemlen = att_align(newitemlen, elmalign); + if (isNull) + newitemlen = 0; + else + { + newitemlen = att_addlength(0, elmlen, dataValue); + newitemlen = att_align(newitemlen, elmalign); + } newsize = overheadlen + lenbefore + newitemlen + lenafter; /* - * OK, do the assignment + * OK, create the new array and fill in header/dimensions */ newarray = (ArrayType *) palloc(newsize); newarray->size = newsize; newarray->ndim = ndim; - newarray->flags = 0; + newarray->dataoffset = newhasnulls ? overheadlen : 0; newarray->elemtype = ARR_ELEMTYPE(array); memcpy(ARR_DIMS(newarray), dim, ndim * sizeof(int)); memcpy(ARR_LBOUND(newarray), lb, ndim * sizeof(int)); + + /* + * Fill in data + */ memcpy((char *) newarray + overheadlen, - (char *) array + overheadlen, + (char *) array + oldoverheadlen, lenbefore); + if (!isNull) + ArrayCastAndSet(dataValue, elmlen, elmbyval, elmalign, + (char *) newarray + overheadlen + lenbefore); memcpy((char *) newarray + overheadlen + lenbefore + newitemlen, - (char *) array + overheadlen + lenbefore + olditemlen, + (char *) array + oldoverheadlen + lenbefore + olditemlen, lenafter); - ArrayCastAndSet(dataValue, elmlen, elmbyval, elmalign, - (char *) newarray + overheadlen + lenbefore); + /* + * Fill in nulls bitmap if needed + * + * Note: it's possible we just replaced the last NULL with a non-NULL, + * and could get rid of the bitmap. Seems not worth testing for though. + */ + if (newhasnulls) + { + bits8 *newnullbitmap = ARR_NULLBITMAP(newarray); + + array_set_isnull(newnullbitmap, offset, isNull); + if (extendbefore) + array_bitmap_copy(newnullbitmap, 1, + oldnullbitmap, 0, + oldnitems); + else + { + array_bitmap_copy(newnullbitmap, 0, + oldnullbitmap, 0, + offset); + if (!extendafter) + array_bitmap_copy(newnullbitmap, offset+1, + oldnullbitmap, offset+1, + oldnitems - offset - 1); + } + } return newarray; } -/*---------------------------------------------------------------------------- +/* * array_set_slice : * This routine sets the value of a range of array locations (specified - * by upper and lower index values ) to new values passed as - * another array - * result : + * by upper and lower subscript values) to new values passed as + * another array. + * + * This handles both ordinary varlena arrays and fixed-length arrays. + * + * Inputs: + * array: the initial array object (mustn't be NULL) + * nSubscripts: number of subscripts supplied (must be same for upper/lower) + * upperIndx[]: the upper subscript values + * lowerIndx[]: the lower subscript values + * srcArray: the source for the inserted values + * isNull: indicates whether srcArray is NULL + * arraytyplen: pg_type.typlen for the array type + * elmlen: pg_type.typlen for the array's element type + * elmbyval: pg_type.typbyval for the array's element type + * elmalign: pg_type.typalign for the array's element type + * + * Result: * A new array is returned, just like the old except for the - * modified range. + * modified range. The original array object is not changed. * * NOTE: we assume it is OK to scribble on the provided index arrays * lowerIndx[] and upperIndx[]. These are generally just temporaries. * * NOTE: For assignments, we throw an error for silly subscripts etc, - * rather than returning a NULL as the fetch operations do. The reasoning - * is that returning a NULL would cause the user's whole array to be replaced - * with NULL, which will probably not make him happy. - *---------------------------------------------------------------------------- + * rather than returning a NULL or empty array as the fetch operations do. */ ArrayType * array_set_slice(ArrayType *array, @@ -1949,33 +2221,38 @@ array_set_slice(ArrayType *array, int *upperIndx, int *lowerIndx, ArrayType *srcArray, - int arraylen, + bool isNull, + int arraytyplen, int elmlen, bool elmbyval, - char elmalign, - bool *isNull) + char elmalign) { + ArrayType *newarray; int i, ndim, dim[MAXDIM], lb[MAXDIM], span[MAXDIM]; - ArrayType *newarray; - int nsrcitems, + bool newhasnulls; + int nitems, + nsrcitems, olddatasize, newsize, olditemsize, newitemsize, overheadlen, + oldoverheadlen, lenbefore, - lenafter; + lenafter, + itemsbefore, + itemsafter, + nolditems; - if (array == NULL) - RETURN_NULL(ArrayType *); - if (srcArray == NULL) + /* Currently, assignment from a NULL source array is a no-op */ + if (isNull) return array; - if (arraylen > 0) + if (arraytyplen > 0) { /* * fixed-length arrays -- not got round to doing this... @@ -2001,11 +2278,12 @@ array_set_slice(ArrayType *array, if (ndim == 0) { Datum *dvalues; + bool *dnulls; int nelems; Oid elmtype = ARR_ELEMTYPE(array); deconstruct_array(srcArray, elmtype, elmlen, elmbyval, elmalign, - &dvalues, &nelems); + &dvalues, &dnulls, &nelems); for (i = 0; i < nSubscripts; i++) { @@ -2019,7 +2297,8 @@ array_set_slice(ArrayType *array, (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), errmsg("source array too small"))); - return construct_md_array(dvalues, nSubscripts, dim, lb, elmtype, + return construct_md_array(dvalues, dnulls, nSubscripts, + dim, lb, elmtype, elmlen, elmbyval, elmalign); } @@ -2076,6 +2355,9 @@ array_set_slice(ArrayType *array, errmsg("invalid array subscripts"))); } + /* Do this mainly to check for overflow */ + nitems = ArrayGetNItems(ndim, dim); + /* * Make sure source array has enough entries. Note we ignore the shape of * the source array and just read entries serially. @@ -2091,20 +2373,34 @@ array_set_slice(ArrayType *array, * Compute space occupied by new entries, space occupied by replaced * entries, and required space for new array. */ - newitemsize = array_nelems_size(ARR_DATA_PTR(srcArray), nsrcitems, + if (ARR_HASNULL(array) || ARR_HASNULL(srcArray)) + { + newhasnulls = true; + overheadlen = ARR_OVERHEAD_WITHNULLS(ndim, nitems); + } + else + { + newhasnulls = false; + overheadlen = ARR_OVERHEAD_NONULLS(ndim); + } + newitemsize = array_nelems_size(ARR_DATA_PTR(srcArray), 0, + ARR_NULLBITMAP(srcArray), nsrcitems, elmlen, elmbyval, elmalign); - overheadlen = ARR_OVERHEAD(ndim); - olddatasize = ARR_SIZE(array) - overheadlen; + oldoverheadlen = ARR_DATA_OFFSET(array); + olddatasize = ARR_SIZE(array) - oldoverheadlen; if (ndim > 1) { /* * here we do not need to cope with extension of the array; it would * be a lot more complicated if we had to do so... */ - olditemsize = array_slice_size(ndim, dim, lb, ARR_DATA_PTR(array), + olditemsize = array_slice_size(ARR_DATA_PTR(array), + ARR_NULLBITMAP(array), + ndim, dim, lb, lowerIndx, upperIndx, elmlen, elmbyval, elmalign); lenbefore = lenafter = 0; /* keep compiler quiet */ + itemsbefore = itemsafter = nolditems = 0; } else { @@ -2116,15 +2412,26 @@ array_set_slice(ArrayType *array, int slicelb = Max(oldlb, lowerIndx[0]); int sliceub = Min(oldub, upperIndx[0]); char *oldarraydata = ARR_DATA_PTR(array); + bits8 *oldarraybitmap = ARR_NULLBITMAP(array); - lenbefore = array_nelems_size(oldarraydata, slicelb - oldlb, + itemsbefore = slicelb - oldlb; + lenbefore = array_nelems_size(oldarraydata, 0, oldarraybitmap, + itemsbefore, elmlen, elmbyval, elmalign); if (slicelb > sliceub) + { + nolditems = 0; olditemsize = 0; + } else + { + nolditems = sliceub - slicelb + 1; olditemsize = array_nelems_size(oldarraydata + lenbefore, - sliceub - slicelb + 1, + itemsbefore, oldarraybitmap, + nolditems, elmlen, elmbyval, elmalign); + } + itemsafter = oldub - sliceub; lenafter = olddatasize - lenbefore - olditemsize; } @@ -2133,7 +2440,7 @@ array_set_slice(ArrayType *array, newarray = (ArrayType *) palloc(newsize); newarray->size = newsize; newarray->ndim = ndim; - newarray->flags = 0; + newarray->dataoffset = newhasnulls ? overheadlen : 0; newarray->elemtype = ARR_ELEMTYPE(array); memcpy(ARR_DIMS(newarray), dim, ndim * sizeof(int)); memcpy(ARR_LBOUND(newarray), lb, ndim * sizeof(int)); @@ -2144,22 +2451,39 @@ array_set_slice(ArrayType *array, * here we do not need to cope with extension of the array; it would * be a lot more complicated if we had to do so... */ - array_insert_slice(ndim, dim, lb, ARR_DATA_PTR(array), olddatasize, - ARR_DATA_PTR(newarray), - lowerIndx, upperIndx, ARR_DATA_PTR(srcArray), + array_insert_slice(newarray, array, srcArray, + ndim, dim, lb, + lowerIndx, upperIndx, elmlen, elmbyval, elmalign); } else { + /* fill in data */ memcpy((char *) newarray + overheadlen, - (char *) array + overheadlen, + (char *) array + oldoverheadlen, lenbefore); memcpy((char *) newarray + overheadlen + lenbefore, ARR_DATA_PTR(srcArray), newitemsize); memcpy((char *) newarray + overheadlen + lenbefore + newitemsize, - (char *) array + overheadlen + lenbefore + olditemsize, + (char *) array + oldoverheadlen + lenbefore + olditemsize, lenafter); + /* fill in nulls bitmap if needed */ + if (newhasnulls) + { + bits8 *newnullbitmap = ARR_NULLBITMAP(newarray); + bits8 *oldnullbitmap = ARR_NULLBITMAP(array); + + array_bitmap_copy(newnullbitmap, 0, + oldnullbitmap, 0, + itemsbefore); + array_bitmap_copy(newnullbitmap, itemsbefore, + ARR_NULLBITMAP(srcArray), 0, + nsrcitems); + array_bitmap_copy(newnullbitmap, itemsbefore+nsrcitems, + oldnullbitmap, itemsbefore+nolditems, + itemsafter); + } } return newarray; @@ -2192,9 +2516,8 @@ array_set_slice(ArrayType *array, * but better performance can be had if the state can be preserved across * a series of calls. * - * NB: caller must assure that input array is not NULL. Currently, - * any additional parameters passed to fn() may not be specified as NULL - * either. + * NB: caller must assure that input array is not NULL. NULL elements in + * the array are OK however. */ Datum array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, @@ -2203,12 +2526,15 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, ArrayType *v; ArrayType *result; Datum *values; + bool *nulls; Datum elt; int *dim; int ndim; int nitems; int i; - int nbytes = 0; + int32 nbytes = 0; + int32 dataoffset; + bool hasnulls; int inp_typlen; bool inp_typbyval; char inp_typalign; @@ -2216,6 +2542,8 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, bool typbyval; char typalign; char *s; + bits8 *bitmap; + int bitmask; ArrayMetaState *inp_extra; ArrayMetaState *ret_extra; @@ -2236,10 +2564,7 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, if (nitems <= 0) { /* Return empty array */ - result = (ArrayType *) palloc0(sizeof(ArrayType)); - result->size = sizeof(ArrayType); - result->elemtype = retType; - PG_RETURN_ARRAYTYPE_P(result); + PG_RETURN_ARRAYTYPE_P(construct_empty_array(retType)); } /* @@ -2274,79 +2599,137 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, typbyval = ret_extra->typbyval; typalign = ret_extra->typalign; - /* Allocate temporary array for new values */ + /* Allocate temporary arrays for new values */ values = (Datum *) palloc(nitems * sizeof(Datum)); + nulls = (bool *) palloc(nitems * sizeof(bool)); /* Loop over source data */ - s = (char *) ARR_DATA_PTR(v); + s = ARR_DATA_PTR(v); + bitmap = ARR_NULLBITMAP(v); + bitmask = 1; + hasnulls = false; + for (i = 0; i < nitems; i++) { - /* Get source element */ - elt = fetch_att(s, inp_typbyval, inp_typlen); + bool callit = true; - s = att_addlength(s, inp_typlen, PointerGetDatum(s)); - s = (char *) att_align(s, inp_typalign); + /* Get source element, checking for NULL */ + if (bitmap && (*bitmap & bitmask) == 0) + { + fcinfo->argnull[0] = true; + } + else + { + elt = fetch_att(s, inp_typbyval, inp_typlen); + s = att_addlength(s, inp_typlen, elt); + s = (char *) att_align(s, inp_typalign); + fcinfo->arg[0] = elt; + fcinfo->argnull[0] = false; + } /* * Apply the given function to source elt and extra args. - * - * We assume the extra args are non-NULL, so need not check whether fn() - * is strict. Would need to do more work here to support arrays - * containing nulls, too. */ - fcinfo->arg[0] = elt; - fcinfo->argnull[0] = false; - fcinfo->isnull = false; - values[i] = FunctionCallInvoke(fcinfo); - if (fcinfo->isnull) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("null array elements not supported"))); + if (fcinfo->flinfo->fn_strict) + { + int j; - /* Ensure data is not toasted */ - if (typlen == -1) - values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); + for (j = 0; j < fcinfo->nargs; j++) + { + if (fcinfo->argnull[j]) + { + callit = false; + break; + } + } + } - /* Update total result size */ - nbytes = att_addlength(nbytes, typlen, values[i]); - nbytes = att_align(nbytes, typalign); + if (callit) + { + fcinfo->isnull = false; + values[i] = FunctionCallInvoke(fcinfo); + } + else + fcinfo->isnull = true; + + nulls[i] = fcinfo->isnull; + if (fcinfo->isnull) + hasnulls = true; + else + { + /* Ensure data is not toasted */ + if (typlen == -1) + values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); + /* Update total result size */ + nbytes = att_addlength(nbytes, typlen, values[i]); + nbytes = att_align(nbytes, typalign); + /* check for overflow of total request */ + if (!AllocSizeIsValid(nbytes)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed (%d)", + (int) MaxAllocSize))); + } + + /* advance bitmap pointer if any */ + if (bitmap) + { + bitmask <<= 1; + if (bitmask == 0x100) + { + bitmap++; + bitmask = 1; + } + } } /* Allocate and initialize the result array */ - nbytes += ARR_OVERHEAD(ndim); - result = (ArrayType *) palloc0(nbytes); - + if (hasnulls) + { + dataoffset = ARR_OVERHEAD_WITHNULLS(ndim, nitems); + nbytes += dataoffset; + } + else + { + dataoffset = 0; /* marker for no null bitmap */ + nbytes += ARR_OVERHEAD_NONULLS(ndim); + } + result = (ArrayType *) palloc(nbytes); result->size = nbytes; result->ndim = ndim; + result->dataoffset = dataoffset; result->elemtype = retType; memcpy(ARR_DIMS(result), ARR_DIMS(v), 2 * ndim * sizeof(int)); /* * Note: do not risk trying to pfree the results of the called function */ - CopyArrayEls(ARR_DATA_PTR(result), values, nitems, - typlen, typbyval, typalign, false); + CopyArrayEls(result, + values, nulls, nitems, + typlen, typbyval, typalign, + false); + pfree(values); + pfree(nulls); PG_RETURN_ARRAYTYPE_P(result); } -/*---------- +/* * construct_array --- simple method for constructing an array object * * elems: array of Datum items to become the array contents + * (NULL element values are not supported). * nelems: number of items * elmtype, elmlen, elmbyval, elmalign: info for the datatype of the items * * A palloc'd 1-D array object is constructed and returned. Note that * elem values will be copied into the object even if pass-by-ref type. - * NULL element values are not supported. * * NOTE: it would be cleaner to look up the elmlen/elmbval/elmalign info * from the system catalogs, given the elmtype. However, the caller is * in a better position to cache this info across multiple uses, or even * to hard-wire values if the element type is hard-wired. - *---------- */ ArrayType * construct_array(Datum *elems, int nelems, @@ -2359,15 +2742,16 @@ construct_array(Datum *elems, int nelems, dims[0] = nelems; lbs[0] = 1; - return construct_md_array(elems, 1, dims, lbs, + return construct_md_array(elems, NULL, 1, dims, lbs, elmtype, elmlen, elmbyval, elmalign); } -/*---------- +/* * construct_md_array --- simple method for constructing an array object - * with arbitrary dimensions + * with arbitrary dimensions and possible NULLs * * elems: array of Datum items to become the array contents + * nulls: array of is-null flags (can be NULL if no nulls) * ndims: number of dimensions * dims: integer array with size of each dimension * lbs: integer array with lower bound of each dimension @@ -2375,23 +2759,24 @@ construct_array(Datum *elems, int nelems, * * A palloc'd ndims-D array object is constructed and returned. Note that * elem values will be copied into the object even if pass-by-ref type. - * NULL element values are not supported. * * NOTE: it would be cleaner to look up the elmlen/elmbval/elmalign info * from the system catalogs, given the elmtype. However, the caller is * in a better position to cache this info across multiple uses, or even * to hard-wire values if the element type is hard-wired. - *---------- */ ArrayType * construct_md_array(Datum *elems, + bool *nulls, int ndims, int *dims, int *lbs, Oid elmtype, int elmlen, bool elmbyval, char elmalign) { ArrayType *result; - int nbytes; + bool hasnulls; + int32 nbytes; + int32 dataoffset; int i; int nelems; @@ -2407,57 +2792,89 @@ construct_md_array(Datum *elems, /* fast track for empty array */ if (ndims == 0) - { - /* Allocate and initialize 0-D result array */ - result = (ArrayType *) palloc0(sizeof(ArrayType)); - result->size = sizeof(ArrayType); - result->elemtype = elmtype; - return result; - } + return construct_empty_array(elmtype); nelems = ArrayGetNItems(ndims, dims); /* compute required space */ - if (elmlen > 0) - nbytes = nelems * att_align(elmlen, elmalign); - else + nbytes = 0; + hasnulls = false; + for (i = 0; i < nelems; i++) { - Assert(!elmbyval); - nbytes = 0; - for (i = 0; i < nelems; i++) + if (nulls && nulls[i]) { - /* make sure data is not toasted */ - if (elmlen == -1) - elems[i] = PointerGetDatum(PG_DETOAST_DATUM(elems[i])); - nbytes = att_addlength(nbytes, elmlen, elems[i]); - nbytes = att_align(nbytes, elmalign); + hasnulls = true; + continue; } + /* make sure data is not toasted */ + if (elmlen == -1) + elems[i] = PointerGetDatum(PG_DETOAST_DATUM(elems[i])); + nbytes = att_addlength(nbytes, elmlen, elems[i]); + nbytes = att_align(nbytes, elmalign); + /* check for overflow of total request */ + if (!AllocSizeIsValid(nbytes)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed (%d)", + (int) MaxAllocSize))); } - /* Allocate and initialize ndims-D result array */ - nbytes += ARR_OVERHEAD(ndims); + /* Allocate and initialize result array */ + if (hasnulls) + { + dataoffset = ARR_OVERHEAD_WITHNULLS(ndims, nelems); + nbytes += dataoffset; + } + else + { + dataoffset = 0; /* marker for no null bitmap */ + nbytes += ARR_OVERHEAD_NONULLS(ndims); + } result = (ArrayType *) palloc(nbytes); - result->size = nbytes; result->ndim = ndims; - result->flags = 0; + result->dataoffset = dataoffset; result->elemtype = elmtype; memcpy(ARR_DIMS(result), dims, ndims * sizeof(int)); memcpy(ARR_LBOUND(result), lbs, ndims * sizeof(int)); - CopyArrayEls(ARR_DATA_PTR(result), elems, nelems, - elmlen, elmbyval, elmalign, false); + CopyArrayEls(result, + elems, nulls, nelems, + elmlen, elmbyval, elmalign, + false); + + return result; +} + +/* + * construct_empty_array --- make a zero-dimensional array of given type + */ +ArrayType * +construct_empty_array(Oid elmtype) +{ + ArrayType *result; + + result = (ArrayType *) palloc(sizeof(ArrayType)); + result->size = sizeof(ArrayType); + result->ndim = 0; + result->dataoffset = 0; + result->elemtype = elmtype; return result; } -/*---------- +/* * deconstruct_array --- simple method for extracting data from an array * * array: array object to examine (must not be NULL) * elmtype, elmlen, elmbyval, elmalign: info for the datatype of the items * elemsp: return value, set to point to palloc'd array of Datum values + * nullsp: return value, set to point to palloc'd array of isnull markers * nelemsp: return value, set to number of extracted values * + * The caller may pass nullsp == NULL if it does not support NULLs in the + * array. Note that this produces a very uninformative error message, + * so do it only in cases where a NULL is really not expected. + * * If array elements are pass-by-ref data type, the returned Datums will * be pointers into the array object. * @@ -2465,42 +2882,72 @@ construct_md_array(Datum *elems, * from the system catalogs, given the elmtype. However, in most current * uses the type is hard-wired into the caller and so we can save a lookup * cycle by hard-wiring the type info as well. - *---------- */ void deconstruct_array(ArrayType *array, Oid elmtype, int elmlen, bool elmbyval, char elmalign, - Datum **elemsp, int *nelemsp) + Datum **elemsp, bool **nullsp, int *nelemsp) { Datum *elems; + bool *nulls; int nelems; char *p; + bits8 *bitmap; + int bitmask; int i; Assert(ARR_ELEMTYPE(array) == elmtype); nelems = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); - if (nelems <= 0) - { - *elemsp = NULL; - *nelemsp = 0; - return; - } *elemsp = elems = (Datum *) palloc(nelems * sizeof(Datum)); + if (nullsp) + *nullsp = nulls = (bool *) palloc(nelems * sizeof(bool)); + else + nulls = NULL; *nelemsp = nelems; p = ARR_DATA_PTR(array); + bitmap = ARR_NULLBITMAP(array); + bitmask = 1; + for (i = 0; i < nelems; i++) { - elems[i] = fetch_att(p, elmbyval, elmlen); - p = att_addlength(p, elmlen, PointerGetDatum(p)); - p = (char *) att_align(p, elmalign); + /* Get source element, checking for NULL */ + if (bitmap && (*bitmap & bitmask) == 0) + { + elems[i] = (Datum) 0; + if (nulls) + nulls[i] = true; + else + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("NULL array element not allowed in this context"))); + } + else + { + elems[i] = fetch_att(p, elmbyval, elmlen); + if (nulls) + nulls[i] = false; + p = att_addlength(p, elmlen, PointerGetDatum(p)); + p = (char *) att_align(p, elmalign); + } + + /* advance bitmap pointer if any */ + if (bitmap) + { + bitmask <<= 1; + if (bitmask == 0x100) + { + bitmap++; + bitmask = 1; + } + } } } -/*----------------------------------------------------------------------------- +/* * array_eq : * compares two arrays for equality * result : @@ -2508,15 +2955,12 @@ deconstruct_array(ArrayType *array, * * Note: we do not use array_cmp here, since equality may be meaningful in * datatypes that don't have a total ordering (and hence no btree support). - *----------------------------------------------------------------------------- */ Datum array_eq(PG_FUNCTION_ARGS) { ArrayType *array1 = PG_GETARG_ARRAYTYPE_P(0); ArrayType *array2 = PG_GETARG_ARRAYTYPE_P(1); - char *p1 = (char *) ARR_DATA_PTR(array1); - char *p2 = (char *) ARR_DATA_PTR(array2); int ndims1 = ARR_NDIM(array1); int ndims2 = ARR_NDIM(array2); int *dims1 = ARR_DIMS(array1); @@ -2529,6 +2973,11 @@ array_eq(PG_FUNCTION_ARGS) int typlen; bool typbyval; char typalign; + char *ptr1; + char *ptr2; + bits8 *bitmap1; + bits8 *bitmap2; + int bitmask; int i; FunctionCallInfoData locfcinfo; @@ -2572,21 +3021,68 @@ array_eq(PG_FUNCTION_ARGS) NULL, NULL); /* Loop over source data */ + ptr1 = ARR_DATA_PTR(array1); + ptr2 = ARR_DATA_PTR(array2); + bitmap1 = ARR_NULLBITMAP(array1); + bitmap2 = ARR_NULLBITMAP(array2); + bitmask = 1; /* use same bitmask for both arrays */ + for (i = 0; i < nitems1; i++) { Datum elt1; Datum elt2; + bool isnull1; + bool isnull2; bool oprresult; - /* Get element pair */ - elt1 = fetch_att(p1, typbyval, typlen); - elt2 = fetch_att(p2, typbyval, typlen); + /* Get elements, checking for NULL */ + if (bitmap1 && (*bitmap1 & bitmask) == 0) + { + isnull1 = true; + elt1 = (Datum) 0; + } + else + { + isnull1 = false; + elt1 = fetch_att(ptr1, typbyval, typlen); + ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1)); + ptr1 = (char *) att_align(ptr1, typalign); + } + + if (bitmap2 && (*bitmap2 & bitmask) == 0) + { + isnull2 = true; + elt2 = (Datum) 0; + } + else + { + isnull2 = false; + elt2 = fetch_att(ptr2, typbyval, typlen); + ptr2 = att_addlength(ptr2, typlen, PointerGetDatum(ptr2)); + ptr2 = (char *) att_align(ptr2, typalign); + } - p1 = att_addlength(p1, typlen, PointerGetDatum(p1)); - p1 = (char *) att_align(p1, typalign); + /* advance bitmap pointers if any */ + bitmask <<= 1; + if (bitmask == 0x100) + { + if (bitmap1) + bitmap1++; + if (bitmap2) + bitmap2++; + bitmask = 1; + } - p2 = att_addlength(p2, typlen, PointerGetDatum(p2)); - p2 = (char *) att_align(p2, typalign); + /* + * We consider two NULLs equal; NULL and not-NULL are unequal. + */ + if (isnull1 && isnull2) + continue; + if (isnull1 || isnull2) + { + result = false; + break; + } /* * Apply the operator to the element pair @@ -2621,6 +3117,7 @@ array_eq(PG_FUNCTION_ARGS) * character-by-character. *---------------------------------------------------------------------------- */ + Datum array_ne(PG_FUNCTION_ARGS) { @@ -2668,8 +3165,6 @@ array_cmp(FunctionCallInfo fcinfo) { ArrayType *array1 = PG_GETARG_ARRAYTYPE_P(0); ArrayType *array2 = PG_GETARG_ARRAYTYPE_P(1); - char *p1 = (char *) ARR_DATA_PTR(array1); - char *p2 = (char *) ARR_DATA_PTR(array2); int ndims1 = ARR_NDIM(array1); int ndims2 = ARR_NDIM(array2); int *dims1 = ARR_DIMS(array1); @@ -2683,6 +3178,11 @@ array_cmp(FunctionCallInfo fcinfo) bool typbyval; char typalign; int min_nitems; + char *ptr1; + char *ptr2; + bits8 *bitmap1; + bits8 *bitmap2; + int bitmask; int i; FunctionCallInfoData locfcinfo; @@ -2721,22 +3221,76 @@ array_cmp(FunctionCallInfo fcinfo) NULL, NULL); /* Loop over source data */ + ptr1 = ARR_DATA_PTR(array1); + ptr2 = ARR_DATA_PTR(array2); + bitmap1 = ARR_NULLBITMAP(array1); + bitmap2 = ARR_NULLBITMAP(array2); + bitmask = 1; /* use same bitmask for both arrays */ + min_nitems = Min(nitems1, nitems2); for (i = 0; i < min_nitems; i++) { Datum elt1; Datum elt2; + bool isnull1; + bool isnull2; int32 cmpresult; - /* Get element pair */ - elt1 = fetch_att(p1, typbyval, typlen); - elt2 = fetch_att(p2, typbyval, typlen); + /* Get elements, checking for NULL */ + if (bitmap1 && (*bitmap1 & bitmask) == 0) + { + isnull1 = true; + elt1 = (Datum) 0; + } + else + { + isnull1 = false; + elt1 = fetch_att(ptr1, typbyval, typlen); + ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1)); + ptr1 = (char *) att_align(ptr1, typalign); + } + + if (bitmap2 && (*bitmap2 & bitmask) == 0) + { + isnull2 = true; + elt2 = (Datum) 0; + } + else + { + isnull2 = false; + elt2 = fetch_att(ptr2, typbyval, typlen); + ptr2 = att_addlength(ptr2, typlen, PointerGetDatum(ptr2)); + ptr2 = (char *) att_align(ptr2, typalign); + } - p1 = att_addlength(p1, typlen, PointerGetDatum(p1)); - p1 = (char *) att_align(p1, typalign); + /* advance bitmap pointers if any */ + bitmask <<= 1; + if (bitmask == 0x100) + { + if (bitmap1) + bitmap1++; + if (bitmap2) + bitmap2++; + bitmask = 1; + } - p2 = att_addlength(p2, typlen, PointerGetDatum(p2)); - p2 = (char *) att_align(p2, typalign); + /* + * We consider two NULLs equal; NULL > not-NULL. + */ + if (isnull1 && isnull2) + continue; + if (isnull1) + { + /* arg1 is greater than arg2 */ + result = 1; + break; + } + if (isnull2) + { + /* arg1 is less than arg2 */ + result = -1; + break; + } /* Compare the pair of elements */ locfcinfo.arg[0] = elt1; @@ -2779,7 +3333,45 @@ array_cmp(FunctionCallInfo fcinfo) /***************************************************************************/ /* + * Check whether a specific array element is NULL + * + * nullbitmap: pointer to array's null bitmap (NULL if none) + * offset: 0-based linear element number of array element + */ +static bool +array_get_isnull(const bits8 *nullbitmap, int offset) +{ + if (nullbitmap == NULL) + return false; /* assume not null */ + if (nullbitmap[offset / 8] & (1 << (offset % 8))) + return false; /* not null */ + return true; +} + +/* + * Set a specific array element's null-bitmap entry + * + * nullbitmap: pointer to array's null bitmap (mustn't be NULL) + * offset: 0-based linear element number of array element + * isNull: null status to set + */ +static void +array_set_isnull(bits8 *nullbitmap, int offset, bool isNull) +{ + int bitmask; + + nullbitmap += offset / 8; + bitmask = 1 << (offset % 8); + if (isNull) + *nullbitmap &= ~bitmask; + else + *nullbitmap |= bitmask; +} + +/* * Fetch array element at pointer, converted correctly to a Datum + * + * Caller must have handled case of NULL element */ static Datum ArrayCast(char *value, bool byval, int len) @@ -2789,6 +3381,8 @@ ArrayCast(char *value, bool byval, int len) /* * Copy datum to *dest and return total space used (including align padding) + * + * Caller must have handled case of NULL element */ static int ArrayCastAndSet(Datum src, @@ -2819,67 +3413,194 @@ ArrayCastAndSet(Datum src, } /* - * Compute total size of the nitems array elements starting at *ptr + * Advance ptr over nitems array elements + * + * ptr: starting location in array + * offset: 0-based linear element number of first element (the one at *ptr) + * nullbitmap: start of array's null bitmap, or NULL if none + * nitems: number of array elements to advance over (>= 0) + * typlen, typbyval, typalign: storage parameters of array element datatype + * + * It is caller's responsibility to ensure that nitems is within range */ -static int -array_nelems_size(char *ptr, int nitems, - int typlen, bool typbyval, char typalign) +static char * +array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems, + int typlen, bool typbyval, char typalign) { - char *origptr; + int bitmask; int i; - /* fixed-size elements? */ - if (typlen > 0) - return nitems * att_align(typlen, typalign); + /* easy if fixed-size elements and no NULLs */ + if (typlen > 0 && !nullbitmap) + return ptr + nitems * ((Size) att_align(typlen, typalign)); - Assert(!typbyval); - origptr = ptr; - for (i = 0; i < nitems; i++) + /* seems worth having separate loops for NULL and no-NULLs cases */ + if (nullbitmap) { - ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr)); - ptr = (char *) att_align(ptr, typalign); + nullbitmap += offset / 8; + bitmask = 1 << (offset % 8); + + for (i = 0; i < nitems; i++) + { + if (*nullbitmap & bitmask) + { + ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr)); + ptr = (char *) att_align(ptr, typalign); + } + bitmask <<= 1; + if (bitmask == 0x100) + { + nullbitmap++; + bitmask = 1; + } + } } - return ptr - origptr; + else + { + for (i = 0; i < nitems; i++) + { + ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr)); + ptr = (char *) att_align(ptr, typalign); + } + } + return ptr; } /* - * Advance ptr over nitems array elements + * Compute total size of the nitems array elements starting at *ptr + * + * Parameters same as for array_seek */ -static char * -array_seek(char *ptr, int nitems, - int typlen, bool typbyval, char typalign) +static int +array_nelems_size(char *ptr, int offset, bits8 *nullbitmap, int nitems, + int typlen, bool typbyval, char typalign) { - return ptr + array_nelems_size(ptr, nitems, - typlen, typbyval, typalign); + return array_seek(ptr, offset, nullbitmap, nitems, + typlen, typbyval, typalign) - ptr; } /* * Copy nitems array elements from srcptr to destptr * + * destptr: starting destination location (must be enough room!) + * nitems: number of array elements to copy (>= 0) + * srcptr: starting location in source array + * offset: 0-based linear element number of first element (the one at *srcptr) + * nullbitmap: start of source array's null bitmap, or NULL if none + * typlen, typbyval, typalign: storage parameters of array element datatype + * * Returns number of bytes copied + * + * NB: this does not take care of setting up the destination's null bitmap! */ static int -array_copy(char *destptr, int nitems, char *srcptr, +array_copy(char *destptr, int nitems, + char *srcptr, int offset, bits8 *nullbitmap, int typlen, bool typbyval, char typalign) { - int numbytes = array_nelems_size(srcptr, nitems, - typlen, typbyval, typalign); + int numbytes; - memmove(destptr, srcptr, numbytes); + numbytes = array_nelems_size(srcptr, offset, nullbitmap, nitems, + typlen, typbyval, typalign); + memcpy(destptr, srcptr, numbytes); return numbytes; } /* + * Copy nitems null-bitmap bits from source to destination + * + * destbitmap: start of destination array's null bitmap (mustn't be NULL) + * destoffset: 0-based linear element number of first dest element + * srcbitmap: start of source array's null bitmap, or NULL if none + * srcoffset: 0-based linear element number of first source element + * nitems: number of bits to copy (>= 0) + * + * If srcbitmap is NULL then we assume the source is all-non-NULL and + * fill 1's into the destination bitmap. Note that only the specified + * bits in the destination map are changed, not any before or after. + * + * Note: this could certainly be optimized using standard bitblt methods. + * However, it's not clear that the typical Postgres array has enough elements + * to make it worth worrying too much. For the moment, KISS. + */ +void +array_bitmap_copy(bits8 *destbitmap, int destoffset, + const bits8 *srcbitmap, int srcoffset, + int nitems) +{ + int destbitmask, + destbitval, + srcbitmask, + srcbitval; + + Assert(destbitmap); + if (nitems <= 0) + return; /* don't risk fetch off end of memory */ + destbitmap += destoffset / 8; + destbitmask = 1 << (destoffset % 8); + destbitval = *destbitmap; + if (srcbitmap) + { + srcbitmap += srcoffset / 8; + srcbitmask = 1 << (srcoffset % 8); + srcbitval = *srcbitmap; + while (nitems-- > 0) + { + if (srcbitval & srcbitmask) + destbitval |= destbitmask; + else + destbitval &= ~destbitmask; + destbitmask <<= 1; + if (destbitmask == 0x100) + { + *destbitmap++ = destbitval; + destbitmask = 1; + if (nitems > 0) + destbitval = *destbitmap; + } + srcbitmask <<= 1; + if (srcbitmask == 0x100) + { + srcbitmap++; + srcbitmask = 1; + if (nitems > 0) + srcbitval = *srcbitmap; + } + } + if (destbitmask != 1) + *destbitmap = destbitval; + } + else + { + while (nitems-- > 0) + { + destbitval |= destbitmask; + destbitmask <<= 1; + if (destbitmask == 0x100) + { + *destbitmap++ = destbitval; + destbitmask = 1; + if (nitems > 0) + destbitval = *destbitmap; + } + } + if (destbitmask != 1) + *destbitmap = destbitval; + } +} + +/* * Compute space needed for a slice of an array * * We assume the caller has verified that the slice coordinates are valid. */ static int -array_slice_size(int ndim, int *dim, int *lb, char *arraydataptr, +array_slice_size(char *arraydataptr, bits8 *arraynullsptr, + int ndim, int *dim, int *lb, int *st, int *endp, int typlen, bool typbyval, char typalign) { - int st_pos, + int src_offset, span[MAXDIM], prod[MAXDIM], dist[MAXDIM], @@ -2892,13 +3613,13 @@ array_slice_size(int ndim, int *dim, int *lb, char *arraydataptr, mda_get_range(ndim, span, st, endp); - /* Pretty easy for fixed element length ... */ - if (typlen > 0) + /* Pretty easy for fixed element length without nulls ... */ + if (typlen > 0 && !arraynullsptr) return ArrayGetNItems(ndim, span) * att_align(typlen, typalign); /* Else gotta do it the hard way */ - st_pos = ArrayGetOffset(ndim, dim, lb, st); - ptr = array_seek(arraydataptr, st_pos, + src_offset = ArrayGetOffset(ndim, dim, lb, st); + ptr = array_seek(arraydataptr, 0, arraynullsptr, src_offset, typlen, typbyval, typalign); mda_get_prod(ndim, dim, prod); mda_get_offset_values(ndim, dist, prod, span); @@ -2907,131 +3628,197 @@ array_slice_size(int ndim, int *dim, int *lb, char *arraydataptr, j = ndim - 1; do { - ptr = array_seek(ptr, dist[j], - typlen, typbyval, typalign); - inc = att_addlength(0, typlen, PointerGetDatum(ptr)); - inc = att_align(inc, typalign); - ptr += inc; - count += inc; + if (dist[j]) + { + ptr = array_seek(ptr, src_offset, arraynullsptr, dist[j], + typlen, typbyval, typalign); + src_offset += dist[j]; + } + if (!array_get_isnull(arraynullsptr, src_offset)) + { + inc = att_addlength(0, typlen, PointerGetDatum(ptr)); + inc = att_align(inc, typalign); + ptr += inc; + count += inc; + } + src_offset++; } while ((j = mda_next_tuple(ndim, indx, span)) != -1); return count; } /* - * Extract a slice of an array into consecutive elements at *destPtr. + * Extract a slice of an array into consecutive elements in the destination + * array. * - * We assume the caller has verified that the slice coordinates are valid - * and allocated enough storage at *destPtr. + * We assume the caller has verified that the slice coordinates are valid, + * allocated enough storage for the result, and initialized the header + * of the new array. */ static void -array_extract_slice(int ndim, +array_extract_slice(ArrayType *newarray, + int ndim, int *dim, int *lb, char *arraydataptr, + bits8 *arraynullsptr, int *st, int *endp, - char *destPtr, int typlen, bool typbyval, char typalign) { - int st_pos, + char *destdataptr = ARR_DATA_PTR(newarray); + bits8 *destnullsptr = ARR_NULLBITMAP(newarray); + char *srcdataptr; + int src_offset, + dest_offset, prod[MAXDIM], span[MAXDIM], dist[MAXDIM], indx[MAXDIM]; - char *srcPtr; int i, j, inc; - st_pos = ArrayGetOffset(ndim, dim, lb, st); - srcPtr = array_seek(arraydataptr, st_pos, + src_offset = ArrayGetOffset(ndim, dim, lb, st); + srcdataptr = array_seek(arraydataptr, 0, arraynullsptr, src_offset, typlen, typbyval, typalign); mda_get_prod(ndim, dim, prod); mda_get_range(ndim, span, st, endp); mda_get_offset_values(ndim, dist, prod, span); for (i = 0; i < ndim; i++) indx[i] = 0; + dest_offset = 0; j = ndim - 1; do { - srcPtr = array_seek(srcPtr, dist[j], - typlen, typbyval, typalign); - inc = array_copy(destPtr, 1, srcPtr, + if (dist[j]) + { + /* skip unwanted elements */ + srcdataptr = array_seek(srcdataptr, src_offset, arraynullsptr, + dist[j], + typlen, typbyval, typalign); + src_offset += dist[j]; + } + inc = array_copy(destdataptr, 1, + srcdataptr, src_offset, arraynullsptr, typlen, typbyval, typalign); - destPtr += inc; - srcPtr += inc; + if (destnullsptr) + array_bitmap_copy(destnullsptr, dest_offset, + arraynullsptr, src_offset, + 1); + destdataptr += inc; + srcdataptr += inc; + src_offset++; + dest_offset++; } while ((j = mda_next_tuple(ndim, indx, span)) != -1); } /* * Insert a slice into an array. * - * ndim/dim/lb are dimensions of the dest array, which has data area - * starting at origPtr. A new array with those same dimensions is to - * be constructed; its data area starts at destPtr. + * ndim/dim[]/lb[] are dimensions of the original array. A new array with + * those same dimensions is to be constructed. destArray must already + * have been allocated and its header initialized. * - * Elements within the slice volume are taken from consecutive locations - * at srcPtr; elements outside it are copied from origPtr. + * st[]/endp[] identify the slice to be replaced. Elements within the slice + * volume are taken from consecutive elements of the srcArray; elements + * outside it are copied from origArray. * - * We assume the caller has verified that the slice coordinates are valid - * and allocated enough storage at *destPtr. + * We assume the caller has verified that the slice coordinates are valid. */ static void -array_insert_slice(int ndim, +array_insert_slice(ArrayType *destArray, + ArrayType *origArray, + ArrayType *srcArray, + int ndim, int *dim, int *lb, - char *origPtr, - int origdatasize, - char *destPtr, int *st, int *endp, - char *srcPtr, int typlen, bool typbyval, char typalign) { - int st_pos, + char *destPtr = ARR_DATA_PTR(destArray); + char *origPtr = ARR_DATA_PTR(origArray); + char *srcPtr = ARR_DATA_PTR(srcArray); + bits8 *destBitmap = ARR_NULLBITMAP(destArray); + bits8 *origBitmap = ARR_NULLBITMAP(origArray); + bits8 *srcBitmap = ARR_NULLBITMAP(srcArray); + int orignitems = ArrayGetNItems(ARR_NDIM(origArray), + ARR_DIMS(origArray)); + int dest_offset, + orig_offset, + src_offset, prod[MAXDIM], span[MAXDIM], dist[MAXDIM], indx[MAXDIM]; - char *origEndpoint = origPtr + origdatasize; int i, j, inc; - st_pos = ArrayGetOffset(ndim, dim, lb, st); - inc = array_copy(destPtr, st_pos, origPtr, + dest_offset = ArrayGetOffset(ndim, dim, lb, st); + /* copy items before the slice start */ + inc = array_copy(destPtr, dest_offset, + origPtr, 0, origBitmap, typlen, typbyval, typalign); destPtr += inc; origPtr += inc; + if (destBitmap) + array_bitmap_copy(destBitmap, 0, origBitmap, 0, dest_offset); + orig_offset = dest_offset; mda_get_prod(ndim, dim, prod); mda_get_range(ndim, span, st, endp); mda_get_offset_values(ndim, dist, prod, span); for (i = 0; i < ndim; i++) indx[i] = 0; + src_offset = 0; j = ndim - 1; do { /* Copy/advance over elements between here and next part of slice */ - inc = array_copy(destPtr, dist[j], origPtr, - typlen, typbyval, typalign); - destPtr += inc; - origPtr += inc; + if (dist[j]) + { + inc = array_copy(destPtr, dist[j], + origPtr, orig_offset, origBitmap, + typlen, typbyval, typalign); + destPtr += inc; + origPtr += inc; + if (destBitmap) + array_bitmap_copy(destBitmap, dest_offset, + origBitmap, orig_offset, + dist[j]); + dest_offset += dist[j]; + orig_offset += dist[j]; + } /* Copy new element at this slice position */ - inc = array_copy(destPtr, 1, srcPtr, + inc = array_copy(destPtr, 1, + srcPtr, src_offset, srcBitmap, typlen, typbyval, typalign); + if (destBitmap) + array_bitmap_copy(destBitmap, dest_offset, + srcBitmap, src_offset, + 1); destPtr += inc; srcPtr += inc; + dest_offset++; + src_offset++; /* Advance over old element at this slice position */ - origPtr = array_seek(origPtr, 1, + origPtr = array_seek(origPtr, orig_offset, origBitmap, 1, typlen, typbyval, typalign); + orig_offset++; } while ((j = mda_next_tuple(ndim, indx, span)) != -1); /* don't miss any data at the end */ - memcpy(destPtr, origPtr, origEndpoint - origPtr); + array_copy(destPtr, orignitems - orig_offset, + origPtr, orig_offset, origBitmap, + typlen, typbyval, typalign); + if (destBitmap) + array_bitmap_copy(destBitmap, dest_offset, + origBitmap, orig_offset, + orignitems - orig_offset); } /* @@ -3280,6 +4067,8 @@ accumArrayResult(ArrayBuildState *astate, astate->mcontext = arr_context; astate->dvalues = (Datum *) palloc(ARRAY_ELEMS_CHUNKSIZE * sizeof(Datum)); + astate->dnulls = (bool *) + palloc(ARRAY_ELEMS_CHUNKSIZE * sizeof(bool)); astate->nelems = 0; astate->element_type = element_type; get_typlenbyvalalign(element_type, @@ -3291,21 +4080,25 @@ accumArrayResult(ArrayBuildState *astate, { oldcontext = MemoryContextSwitchTo(astate->mcontext); Assert(astate->element_type == element_type); - /* enlarge dvalues[] if needed */ + /* enlarge dvalues[]/dnulls[] if needed */ if ((astate->nelems % ARRAY_ELEMS_CHUNKSIZE) == 0) + { astate->dvalues = (Datum *) repalloc(astate->dvalues, (astate->nelems + ARRAY_ELEMS_CHUNKSIZE) * sizeof(Datum)); + astate->dnulls = (bool *) + repalloc(astate->dnulls, + (astate->nelems + ARRAY_ELEMS_CHUNKSIZE) * sizeof(bool)); + } } - if (disnull) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("null array elements not supported"))); - /* Use datumCopy to ensure pass-by-ref stuff is copied into mcontext */ - astate->dvalues[astate->nelems++] = - datumCopy(dvalue, astate->typbyval, astate->typlen); + if (!disnull && !astate->typbyval) + dvalue = datumCopy(dvalue, astate->typbyval, astate->typlen); + + astate->dvalues[astate->nelems] = dvalue; + astate->dnulls[astate->nelems] = disnull; + astate->nelems++; MemoryContextSwitchTo(oldcontext); @@ -3354,6 +4147,7 @@ makeMdArrayResult(ArrayBuildState *astate, oldcontext = MemoryContextSwitchTo(rcontext); result = construct_md_array(astate->dvalues, + astate->dnulls, ndims, dims, lbs, diff --git a/src/backend/utils/adt/arrayutils.c b/src/backend/utils/adt/arrayutils.c index c6a66531db..c7355968d7 100644 --- a/src/backend/utils/adt/arrayutils.c +++ b/src/backend/utils/adt/arrayutils.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/arrayutils.c,v 1.18 2004/12/31 22:01:21 pgsql Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/arrayutils.c,v 1.19 2005/11/17 22:14:52 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -16,11 +16,17 @@ #include "postgres.h" #include "utils/array.h" +#include "utils/memutils.h" -/* Convert subscript list into linear element number (from 0) */ +/* + * Convert subscript list into linear element number (from 0) + * + * We assume caller has already range-checked the dimensions and subscripts, + * so no overflow is possible. + */ int -ArrayGetOffset(int n, int *dim, int *lb, int *indx) +ArrayGetOffset(int n, const int *dim, const int *lb, const int *indx) { int i, scale = 1, @@ -34,11 +40,12 @@ ArrayGetOffset(int n, int *dim, int *lb, int *indx) return offset; } -/* Same, but subscripts are assumed 0-based, and use a scale array +/* + * Same, but subscripts are assumed 0-based, and use a scale array * instead of raw dimension data (see mda_get_prod to create scale array) */ int -ArrayGetOffset0(int n, int *tup, int *scale) +ArrayGetOffset0(int n, const int *tup, const int *scale) { int i, lin = 0; @@ -48,24 +55,66 @@ ArrayGetOffset0(int n, int *tup, int *scale) return lin; } -/* Convert array dimensions into number of elements */ +/* + * Convert array dimensions into number of elements + * + * This must do overflow checking, since it is used to validate that a user + * dimensionality request doesn't overflow what we can handle. + * + * We limit array sizes to at most about a quarter billion elements, + * so that it's not necessary to check for overflow in quite so many + * places --- for instance when palloc'ing Datum arrays. + * + * The multiplication overflow check only works on machines that have int64 + * arithmetic, but that is nearly all platforms these days, and doing check + * divides for those that don't seems way too expensive. + */ int -ArrayGetNItems(int ndim, int *dims) +ArrayGetNItems(int ndim, const int *dims) { - int i, - ret; + int32 ret; + int i; + +#define MaxArraySize ((Size) (MaxAllocSize / sizeof(Datum))) if (ndim <= 0) return 0; ret = 1; for (i = 0; i < ndim; i++) - ret *= dims[i]; - return ret; + { + int64 prod; + + /* A negative dimension implies that UB-LB overflowed ... */ + if (dims[i] < 0) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed (%d)", + (int) MaxArraySize))); + + prod = (int64) ret * (int64) dims[i]; + ret = (int32) prod; + if ((int64) ret != prod) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed (%d)", + (int) MaxArraySize))); + } + Assert(ret >= 0); + if ((Size) ret > MaxArraySize) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("array size exceeds the maximum allowed (%d)", + (int) MaxArraySize))); + return (int) ret; } -/* Compute ranges (sub-array dimensions) for an array slice */ +/* + * Compute ranges (sub-array dimensions) for an array slice + * + * We assume caller has validated slice endpoints, so overflow is impossible + */ void -mda_get_range(int n, int *span, int *st, int *endp) +mda_get_range(int n, int *span, const int *st, const int *endp) { int i; @@ -73,9 +122,13 @@ mda_get_range(int n, int *span, int *st, int *endp) span[i] = endp[i] - st[i] + 1; } -/* Compute products of array dimensions, ie, scale factors for subscripts */ +/* + * Compute products of array dimensions, ie, scale factors for subscripts + * + * We assume caller has validated dimensions, so overflow is impossible + */ void -mda_get_prod(int n, int *range, int *prod) +mda_get_prod(int n, const int *range, int *prod) { int i; @@ -84,11 +137,14 @@ mda_get_prod(int n, int *range, int *prod) prod[i] = prod[i + 1] * range[i + 1]; } -/* From products of whole-array dimensions and spans of a sub-array, +/* + * From products of whole-array dimensions and spans of a sub-array, * compute offset distances needed to step through subarray within array + * + * We assume caller has validated dimensions, so overflow is impossible */ void -mda_get_offset_values(int n, int *dist, int *prod, int *span) +mda_get_offset_values(int n, int *dist, const int *prod, const int *span) { int i, j; @@ -102,16 +158,18 @@ mda_get_offset_values(int n, int *dist, int *prod, int *span) } } -/*----------------------------------------------------------------------------- - generates the tuple that is lexicographically one greater than the current - n-tuple in "curr", with the restriction that the i-th element of "curr" is - less than the i-th element of "span". - Returns -1 if no next tuple exists, else the subscript position (0..n-1) - corresponding to the dimension to advance along. - ----------------------------------------------------------------------------- -*/ +/* + * Generates the tuple that is lexicographically one greater than the current + * n-tuple in "curr", with the restriction that the i-th element of "curr" is + * less than the i-th element of "span". + * + * Returns -1 if no next tuple exists, else the subscript position (0..n-1) + * corresponding to the dimension to advance along. + * + * We assume caller has validated dimensions, so overflow is impossible + */ int -mda_next_tuple(int n, int *curr, int *span) +mda_next_tuple(int n, int *curr, const int *span) { int i; diff --git a/src/backend/utils/adt/float.c b/src/backend/utils/adt/float.c index fb37e36624..f77c54f9cc 100644 --- a/src/backend/utils/adt/float.c +++ b/src/backend/utils/adt/float.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/float.c,v 1.115 2005/10/15 02:49:28 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/float.c,v 1.116 2005/11/17 22:14:52 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -1886,6 +1886,7 @@ check_float8_array(ArrayType *transarray, const char *caller) */ if (ARR_NDIM(transarray) != 1 || ARR_DIMS(transarray)[0] != 3 || + ARR_HASNULL(transarray) || ARR_ELEMTYPE(transarray) != FLOAT8OID) elog(ERROR, "%s: expected 3-element float8 array", caller); return (float8 *) ARR_DATA_PTR(transarray); diff --git a/src/backend/utils/adt/int.c b/src/backend/utils/adt/int.c index e41e584ffe..d47dbfdab6 100644 --- a/src/backend/utils/adt/int.c +++ b/src/backend/utils/adt/int.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/int.c,v 1.68 2005/10/15 02:49:28 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/int.c,v 1.69 2005/11/17 22:14:53 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -133,7 +133,7 @@ buildint2vector(const int2 *int2s, int n) */ result->size = Int2VectorSize(n); result->ndim = 1; - result->flags = 0; + result->dataoffset = 0; /* never any nulls */ result->elemtype = INT2OID; result->dim1 = n; result->lbound1 = 0; @@ -171,7 +171,7 @@ int2vectorin(PG_FUNCTION_ARGS) result->size = Int2VectorSize(n); result->ndim = 1; - result->flags = 0; + result->dataoffset = 0; /* never any nulls */ result->elemtype = INT2OID; result->dim1 = n; result->lbound1 = 0; @@ -220,9 +220,9 @@ int2vectorrecv(PG_FUNCTION_ARGS) ObjectIdGetDatum(INT2OID), Int32GetDatum(-1))); /* sanity checks: int2vector must be 1-D, no nulls */ - if (result->ndim != 1 || - result->flags != 0 || - result->elemtype != INT2OID) + if (ARR_NDIM(result) != 1 || + ARR_HASNULL(result) || + ARR_ELEMTYPE(result) != INT2OID) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("invalid int2vector data"))); diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index a8becf990d..8a69a936dc 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -14,7 +14,7 @@ * Copyright (c) 1998-2005, PostgreSQL Global Development Group * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/numeric.c,v 1.86 2005/10/15 02:49:29 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/numeric.c,v 1.87 2005/11/17 22:14:53 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -2070,7 +2070,7 @@ do_numeric_accum(ArrayType *transarray, Numeric newval) /* We assume the input is array of numeric */ deconstruct_array(transarray, NUMERICOID, -1, false, 'i', - &transdatums, &ndatums); + &transdatums, NULL, &ndatums); if (ndatums != 3) elog(ERROR, "expected 3-element numeric array"); N = transdatums[0]; @@ -2161,7 +2161,7 @@ numeric_avg(PG_FUNCTION_ARGS) /* We assume the input is array of numeric */ deconstruct_array(transarray, NUMERICOID, -1, false, 'i', - &transdatums, &ndatums); + &transdatums, NULL, &ndatums); if (ndatums != 3) elog(ERROR, "expected 3-element numeric array"); N = DatumGetNumeric(transdatums[0]); @@ -2197,7 +2197,7 @@ numeric_variance(PG_FUNCTION_ARGS) /* We assume the input is array of numeric */ deconstruct_array(transarray, NUMERICOID, -1, false, 'i', - &transdatums, &ndatums); + &transdatums, NULL, &ndatums); if (ndatums != 3) elog(ERROR, "expected 3-element numeric array"); N = DatumGetNumeric(transdatums[0]); @@ -2273,7 +2273,7 @@ numeric_stddev(PG_FUNCTION_ARGS) /* We assume the input is array of numeric */ deconstruct_array(transarray, NUMERICOID, -1, false, 'i', - &transdatums, &ndatums); + &transdatums, NULL, &ndatums); if (ndatums != 3) elog(ERROR, "expected 3-element numeric array"); N = DatumGetNumeric(transdatums[0]); @@ -2511,7 +2511,8 @@ int2_avg_accum(PG_FUNCTION_ARGS) else transarray = PG_GETARG_ARRAYTYPE_P_COPY(0); - if (ARR_SIZE(transarray) != ARR_OVERHEAD(1) + sizeof(Int8TransTypeData)) + if (ARR_HASNULL(transarray) || + ARR_SIZE(transarray) != ARR_OVERHEAD_NONULLS(1) + sizeof(Int8TransTypeData)) elog(ERROR, "expected 2-element int8 array"); transdata = (Int8TransTypeData *) ARR_DATA_PTR(transarray); @@ -2538,7 +2539,8 @@ int4_avg_accum(PG_FUNCTION_ARGS) else transarray = PG_GETARG_ARRAYTYPE_P_COPY(0); - if (ARR_SIZE(transarray) != ARR_OVERHEAD(1) + sizeof(Int8TransTypeData)) + if (ARR_HASNULL(transarray) || + ARR_SIZE(transarray) != ARR_OVERHEAD_NONULLS(1) + sizeof(Int8TransTypeData)) elog(ERROR, "expected 2-element int8 array"); transdata = (Int8TransTypeData *) ARR_DATA_PTR(transarray); @@ -2556,7 +2558,8 @@ int8_avg(PG_FUNCTION_ARGS) Datum countd, sumd; - if (ARR_SIZE(transarray) != ARR_OVERHEAD(1) + sizeof(Int8TransTypeData)) + if (ARR_HASNULL(transarray) || + ARR_SIZE(transarray) != ARR_OVERHEAD_NONULLS(1) + sizeof(Int8TransTypeData)) elog(ERROR, "expected 2-element int8 array"); transdata = (Int8TransTypeData *) ARR_DATA_PTR(transarray); diff --git a/src/backend/utils/adt/oid.c b/src/backend/utils/adt/oid.c index 62db042bbd..e400c9a1b4 100644 --- a/src/backend/utils/adt/oid.c +++ b/src/backend/utils/adt/oid.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/oid.c,v 1.64 2005/10/15 02:49:29 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/oid.c,v 1.65 2005/11/17 22:14:53 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -176,7 +176,7 @@ buildoidvector(const Oid *oids, int n) */ result->size = OidVectorSize(n); result->ndim = 1; - result->flags = 0; + result->dataoffset = 0; /* never any nulls */ result->elemtype = OIDOID; result->dim1 = n; result->lbound1 = 0; @@ -213,7 +213,7 @@ oidvectorin(PG_FUNCTION_ARGS) result->size = OidVectorSize(n); result->ndim = 1; - result->flags = 0; + result->dataoffset = 0; /* never any nulls */ result->elemtype = OIDOID; result->dim1 = n; result->lbound1 = 0; @@ -262,9 +262,9 @@ oidvectorrecv(PG_FUNCTION_ARGS) ObjectIdGetDatum(OIDOID), Int32GetDatum(-1))); /* sanity checks: oidvector must be 1-D, no nulls */ - if (result->ndim != 1 || - result->flags != 0 || - result->elemtype != OIDOID) + if (ARR_NDIM(result) != 1 || + ARR_HASNULL(result) || + ARR_ELEMTYPE(result) != OIDOID) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("invalid oidvector data"))); diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 04e8eb5516..5411e6ab8c 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -3,7 +3,7 @@ * back to source text * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/ruleutils.c,v 1.207 2005/10/15 02:49:29 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/ruleutils.c,v 1.208 2005/11/17 22:14:53 tgl Exp $ * * This software is copyrighted by Jan Wieck - Hamburg. * @@ -1107,7 +1107,7 @@ decompile_column_index_array(Datum column_index_array, Oid relId, /* Extract data from array of int16 */ deconstruct_array(DatumGetArrayTypeP(column_index_array), INT2OID, 2, true, 's', - &keys, &nKeys); + &keys, NULL, &nKeys); for (j = 0; j < nKeys; j++) { diff --git a/src/backend/utils/adt/timestamp.c b/src/backend/utils/adt/timestamp.c index 43956597e3..ec2e80fc2a 100644 --- a/src/backend/utils/adt/timestamp.c +++ b/src/backend/utils/adt/timestamp.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/timestamp.c,v 1.157 2005/10/27 02:45:22 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/timestamp.c,v 1.158 2005/11/17 22:14:53 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -2434,7 +2434,7 @@ interval_accum(PG_FUNCTION_ARGS) deconstruct_array(transarray, INTERVALOID, sizeof(Interval), false, 'd', - &transdatums, &ndatums); + &transdatums, NULL, &ndatums); if (ndatums != 2) elog(ERROR, "expected 2-element interval array"); @@ -2475,7 +2475,7 @@ interval_avg(PG_FUNCTION_ARGS) deconstruct_array(transarray, INTERVALOID, sizeof(Interval), false, 'd', - &transdatums, &ndatums); + &transdatums, NULL, &ndatums); if (ndatums != 2) elog(ERROR, "expected 2-element interval array"); diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 096a3cb942..40dd680659 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -7,7 +7,7 @@ * Portions Copyright (c) 1994, Regents of the University of California * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/cache/lsyscache.c,v 1.129 2005/10/15 02:49:31 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/cache/lsyscache.c,v 1.130 2005/11/17 22:14:53 tgl Exp $ * * NOTES * Eventually, the index information should go through here, too. @@ -1896,13 +1896,13 @@ get_attstatsslot(HeapTuple statstuple, elog(ERROR, "cache lookup failed for type %u", atttype); typeForm = (Form_pg_type) GETSTRUCT(typeTuple); - /* Deconstruct array into Datum elements */ + /* Deconstruct array into Datum elements; NULLs not expected */ deconstruct_array(statarray, atttype, typeForm->typlen, typeForm->typbyval, typeForm->typalign, - values, nvalues); + values, NULL, nvalues); /* * If the element type is pass-by-reference, we now have a bunch of @@ -1944,6 +1944,7 @@ get_attstatsslot(HeapTuple statstuple, */ narrayelem = ARR_DIMS(statarray)[0]; if (ARR_NDIM(statarray) != 1 || narrayelem <= 0 || + ARR_HASNULL(statarray) || ARR_ELEMTYPE(statarray) != FLOAT4OID) elog(ERROR, "stanumbers is not a 1-D float4 array"); *numbers = (float4 *) palloc(narrayelem * sizeof(float4)); diff --git a/src/backend/utils/fmgr/funcapi.c b/src/backend/utils/fmgr/funcapi.c index 0a51f7ae0f..b545928d9b 100644 --- a/src/backend/utils/fmgr/funcapi.c +++ b/src/backend/utils/fmgr/funcapi.c @@ -7,7 +7,7 @@ * Copyright (c) 2002-2005, PostgreSQL Global Development Group * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/fmgr/funcapi.c,v 1.26 2005/10/15 02:49:32 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/fmgr/funcapi.c,v 1.27 2005/11/17 22:14:53 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -686,16 +686,18 @@ get_func_result_name(Oid functionId) numargs = ARR_DIMS(arr)[0]; if (ARR_NDIM(arr) != 1 || numargs < 0 || + ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != CHAROID) elog(ERROR, "proargmodes is not a 1-D char array"); argmodes = (char *) ARR_DATA_PTR(arr); arr = DatumGetArrayTypeP(proargnames); /* ensure not toasted */ if (ARR_NDIM(arr) != 1 || ARR_DIMS(arr)[0] != numargs || + ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != TEXTOID) elog(ERROR, "proargnames is not a 1-D text array"); deconstruct_array(arr, TEXTOID, -1, false, 'i', - &argnames, &nargnames); + &argnames, NULL, &nargnames); Assert(nargnames == numargs); /* scan for output argument(s) */ @@ -818,12 +820,14 @@ build_function_result_tupdesc_d(Datum proallargtypes, numargs = ARR_DIMS(arr)[0]; if (ARR_NDIM(arr) != 1 || numargs < 0 || + ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != OIDOID) elog(ERROR, "proallargtypes is not a 1-D Oid array"); argtypes = (Oid *) ARR_DATA_PTR(arr); arr = DatumGetArrayTypeP(proargmodes); /* ensure not toasted */ if (ARR_NDIM(arr) != 1 || ARR_DIMS(arr)[0] != numargs || + ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != CHAROID) elog(ERROR, "proargmodes is not a 1-D char array"); argmodes = (char *) ARR_DATA_PTR(arr); @@ -832,10 +836,11 @@ build_function_result_tupdesc_d(Datum proallargtypes, arr = DatumGetArrayTypeP(proargnames); /* ensure not toasted */ if (ARR_NDIM(arr) != 1 || ARR_DIMS(arr)[0] != numargs || + ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != TEXTOID) elog(ERROR, "proargnames is not a 1-D text array"); deconstruct_array(arr, TEXTOID, -1, false, 'i', - &argnames, &nargnames); + &argnames, NULL, &nargnames); Assert(nargnames == numargs); } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 79e162efc0..6b83f36321 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -10,7 +10,7 @@ * Written by Peter Eisentraut . * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.299 2005/11/04 23:50:30 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.300 2005/11/17 22:14:54 tgl Exp $ * *-------------------------------------------------------------------- */ @@ -877,6 +877,16 @@ static struct config_bool ConfigureNamesBool[] = true, NULL, NULL }, { + {"array_nulls", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, + gettext_noop("Enable input of NULL elements in arrays."), + gettext_noop("When turned on, unquoted NULL in an array input " + "value means a NULL value; " + "otherwise it is taken literally.") + }, + &Array_nulls, + true, NULL, NULL + }, + { {"default_with_oids", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, gettext_noop("Create new tables with OIDs by default."), NULL @@ -5383,14 +5393,13 @@ GUCArrayAdd(ArrayType *array, const char *name, const char *value) } } - isnull = false; a = array_set(array, 1, &index, datum, - -1 /* varlenarray */ , + false, + -1 /* varlena array */ , -1 /* TEXT's typlen */ , false /* TEXT's typbyval */ , - 'i' /* TEXT's typalign */ , - &isnull); + 'i' /* TEXT's typalign */ ); } else a = construct_array(&datum, 1, @@ -5456,14 +5465,13 @@ GUCArrayDelete(ArrayType *array, const char *name) /* else add it to the output array */ if (newarray) { - isnull = false; newarray = array_set(newarray, 1, &index, d, + false, -1 /* varlenarray */ , -1 /* TEXT's typlen */ , false /* TEXT's typbyval */ , - 'i' /* TEXT's typalign */ , - &isnull); + 'i' /* TEXT's typalign */ ); } else newarray = construct_array(&d, 1, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 773899e8b7..94503ddfbb 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -413,10 +413,11 @@ # - Previous Postgres Versions - #add_missing_from = off -#regex_flavor = advanced # advanced, extended, or basic -#sql_inheritance = on +#array_nulls = on #default_with_oids = off #escape_string_warning = off +#regex_flavor = advanced # advanced, extended, or basic +#sql_inheritance = on # - Other Platforms & Clients - diff --git a/src/include/c.h b/src/include/c.h index 2f21247b26..fb7361905d 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -12,7 +12,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/c.h,v 1.190 2005/10/15 02:49:41 momjian Exp $ + * $PostgreSQL: pgsql/src/include/c.h,v 1.191 2005/11/17 22:14:54 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -442,8 +442,8 @@ typedef struct varlena VarChar; /* var-length char, ie SQL varchar(n) */ typedef struct { int32 size; /* these fields must match ArrayType! */ - int ndim; - int flags; + int ndim; /* always 1 for int2vector */ + int32 dataoffset; /* always 0 for int2vector */ Oid elemtype; int dim1; int lbound1; @@ -453,8 +453,8 @@ typedef struct typedef struct { int32 size; /* these fields must match ArrayType! */ - int ndim; - int flags; + int ndim; /* always 1 for oidvector */ + int32 dataoffset; /* always 0 for oidvector */ Oid elemtype; int dim1; int lbound1; diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 94cadcd492..d2637e37eb 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -37,7 +37,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.306 2005/11/07 17:36:46 tgl Exp $ + * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.307 2005/11/17 22:14:54 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 200511071 +#define CATALOG_VERSION_NO 200511171 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 5b0af25c1c..824f7a3fba 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/pg_proc.h,v 1.388 2005/11/07 17:36:46 tgl Exp $ + * $PostgreSQL: pgsql/src/include/catalog/pg_proc.h,v 1.389 2005/11/17 22:14:54 tgl Exp $ * * NOTES * The script catalog/genbki.sh reads this file and generates .bki @@ -995,11 +995,11 @@ DATA(insert OID = 2091 ( array_lower PGNSP PGUID 12 f f t f i 2 23 "2277 23" DESCR("array lower dimension"); DATA(insert OID = 2092 ( array_upper PGNSP PGUID 12 f f t f i 2 23 "2277 23" _null_ _null_ _null_ array_upper - _null_ )); DESCR("array upper dimension"); -DATA(insert OID = 378 ( array_append PGNSP PGUID 12 f f t f i 2 2277 "2277 2283" _null_ _null_ _null_ array_push - _null_ )); +DATA(insert OID = 378 ( array_append PGNSP PGUID 12 f f f f i 2 2277 "2277 2283" _null_ _null_ _null_ array_push - _null_ )); DESCR("append element onto end of array"); -DATA(insert OID = 379 ( array_prepend PGNSP PGUID 12 f f t f i 2 2277 "2283 2277" _null_ _null_ _null_ array_push - _null_ )); +DATA(insert OID = 379 ( array_prepend PGNSP PGUID 12 f f f f i 2 2277 "2283 2277" _null_ _null_ _null_ array_push - _null_ )); DESCR("prepend element onto front of array"); -DATA(insert OID = 383 ( array_cat PGNSP PGUID 12 f f t f i 2 2277 "2277 2277" _null_ _null_ _null_ array_cat - _null_ )); +DATA(insert OID = 383 ( array_cat PGNSP PGUID 12 f f f f i 2 2277 "2277 2277" _null_ _null_ _null_ array_cat - _null_ )); DESCR("concatenate two arrays"); DATA(insert OID = 384 ( array_coerce PGNSP PGUID 12 f f t f s 1 2277 "2277" _null_ _null_ _null_ array_type_coerce - _null_ )); DESCR("coerce array to another array type"); diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index da4c6baa80..c668382ba3 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/utils/acl.h,v 1.86 2005/11/04 17:25:15 tgl Exp $ + * $PostgreSQL: pgsql/src/include/utils/acl.h,v 1.87 2005/11/17 22:14:55 tgl Exp $ * * NOTES * An ACL array is simply an array of AclItems, representing the union @@ -97,7 +97,7 @@ typedef ArrayType Acl; #define ACL_NUM(ACL) (ARR_DIMS(ACL)[0]) #define ACL_DAT(ACL) ((AclItem *) ARR_DATA_PTR(ACL)) -#define ACL_N_SIZE(N) (ARR_OVERHEAD(1) + ((N) * sizeof(AclItem))) +#define ACL_N_SIZE(N) (ARR_OVERHEAD_NONULLS(1) + ((N) * sizeof(AclItem))) #define ACL_SIZE(ACL) ARR_SIZE(ACL) /* @@ -107,7 +107,7 @@ typedef ArrayType IdList; #define IDLIST_NUM(IDL) (ARR_DIMS(IDL)[0]) #define IDLIST_DAT(IDL) ((Oid *) ARR_DATA_PTR(IDL)) -#define IDLIST_N_SIZE(N) (ARR_OVERHEAD(1) + ((N) * sizeof(Oid))) +#define IDLIST_N_SIZE(N) (ARR_OVERHEAD_NONULLS(1) + ((N) * sizeof(Oid))) #define IDLIST_SIZE(IDL) ARR_SIZE(IDL) /* diff --git a/src/include/utils/array.h b/src/include/utils/array.h index 1e8be02606..d3653cff0d 100644 --- a/src/include/utils/array.h +++ b/src/include/utils/array.h @@ -1,16 +1,55 @@ /*------------------------------------------------------------------------- * * array.h - * Utilities for the new array code. Contains prototypes from the - * following files: - * utils/adt/arrayfuncs.c - * utils/adt/arrayutils.c + * Declarations for Postgres arrays. + * + * A standard varlena array has the following internal structure: + * - total number of bytes (also, TOAST info flags) + * - number of dimensions of the array + * - offset to stored data, or 0 if no nulls bitmap + * - element type OID + * - length of each array axis (C array of int) + * - lower boundary of each dimension (C array of int) + * - bitmap showing locations of nulls (OPTIONAL) + * - whatever is the stored data + * + * The and arrays each have ndim elements. + * + * The may be omitted if the array contains no NULL elements. + * If it is absent, the field is zero and the offset to the + * stored data must be computed on-the-fly. If the bitmap is present, + * is nonzero and is equal to the offset from the array start + * to the first data element (including any alignment padding). The bitmap + * follows the same conventions as tuple null bitmaps, ie, a 1 indicates + * a non-null entry and the LSB of each bitmap byte is used first. + * + * The actual data starts on a MAXALIGN boundary. Individual items in the + * array are aligned as specified by the array element type. They are + * stored in row-major order (last subscript varies most rapidly). + * + * NOTE: it is important that array elements of toastable datatypes NOT be + * toasted, since the tupletoaster won't know they are there. (We could + * support compressed toasted items; only out-of-line items are dangerous. + * However, it seems preferable to store such items uncompressed and allow + * the toaster to compress the whole array as one input.) + * + * + * The OIDVECTOR and INT2VECTOR datatypes are storage-compatible with + * generic arrays, but they support only one-dimensional arrays with no + * nulls (and no null bitmap). + * + * There are also some "fixed-length array" datatypes, such as NAME and + * POINT. These are simply a sequence of a fixed number of items each + * of a fixed-length datatype, with no overhead; the item size must be + * a multiple of its alignment requirement, because we do no padding. + * We support subscripting on these types, but array_in() and array_out() + * only work with varlena arrays. * * * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/utils/array.h,v 1.55 2005/10/15 02:49:46 momjian Exp $ + * $PostgreSQL: pgsql/src/include/utils/array.h,v 1.56 2005/11/17 22:14:55 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -30,8 +69,7 @@ typedef struct { int32 size; /* total array size (varlena requirement) */ int ndim; /* # of dimensions */ - int flags; /* implementation flags */ - /* flags field is currently unused, always zero. */ + int32 dataoffset; /* offset to data, or 0 if no bitmap */ Oid elemtype; /* element type OID */ } ArrayType; @@ -39,9 +77,10 @@ typedef struct ArrayBuildState { MemoryContext mcontext; /* where all the temp stuff is kept */ Datum *dvalues; /* array of accumulated Datums */ + bool *dnulls; /* array of is-null flags for Datums */ /* - * The allocated size of dvalues[] is always a multiple of + * The allocated size of dvalues[] and dnulls[] is always a multiple of * ARRAY_ELEMS_CHUNKSIZE */ #define ARRAY_ELEMS_CHUNKSIZE 64 @@ -98,30 +137,49 @@ typedef struct ArrayMapState * * Unlike C, the default lower bound is 1. */ -#define ARR_SIZE(a) (((ArrayType *) (a))->size) -#define ARR_NDIM(a) (((ArrayType *) (a))->ndim) -#define ARR_ELEMTYPE(a) (((ArrayType *) (a))->elemtype) +#define ARR_SIZE(a) ((a)->size) +#define ARR_NDIM(a) ((a)->ndim) +#define ARR_HASNULL(a) ((a)->dataoffset != 0) +#define ARR_ELEMTYPE(a) ((a)->elemtype) #define ARR_DIMS(a) \ ((int *) (((char *) (a)) + sizeof(ArrayType))) #define ARR_LBOUND(a) \ ((int *) (((char *) (a)) + sizeof(ArrayType) + \ - (sizeof(int) * ARR_NDIM(a)))) + sizeof(int) * ARR_NDIM(a))) + +#define ARR_NULLBITMAP(a) \ + (ARR_HASNULL(a) ? \ + (bits8 *) (((char *) (a)) + sizeof(ArrayType) + \ + 2 * sizeof(int) * ARR_NDIM(a)) \ + : (bits8 *) NULL) /* - * The total array header size for an array of dimension n (in bytes). + * The total array header size (in bytes) for an array with the specified + * number of dimensions and total number of items. */ -#define ARR_OVERHEAD(n) \ - (MAXALIGN(sizeof(ArrayType) + 2 * sizeof(int) * (n))) +#define ARR_OVERHEAD_NONULLS(ndims) \ + MAXALIGN(sizeof(ArrayType) + 2 * sizeof(int) * (ndims)) +#define ARR_OVERHEAD_WITHNULLS(ndims, nitems) \ + MAXALIGN(sizeof(ArrayType) + 2 * sizeof(int) * (ndims) + \ + ((nitems) + 7) / 8) + +#define ARR_DATA_OFFSET(a) \ + (ARR_HASNULL(a) ? (a)->dataoffset : ARR_OVERHEAD_NONULLS(ARR_NDIM(a))) /* * Returns a pointer to the actual array data. */ #define ARR_DATA_PTR(a) \ - (((char *) (a)) + ARR_OVERHEAD(ARR_NDIM(a))) + (((char *) (a)) + ARR_DATA_OFFSET(a)) /* + * GUC parameter + */ +extern bool Array_nulls; + +/* * prototypes for functions defined in arrayfuncs.c */ extern Datum array_in(PG_FUNCTION_ARGS); @@ -145,37 +203,40 @@ extern Datum array_larger(PG_FUNCTION_ARGS); extern Datum array_smaller(PG_FUNCTION_ARGS); extern Datum array_ref(ArrayType *array, int nSubscripts, int *indx, - int arraylen, int elmlen, bool elmbyval, char elmalign, + int arraytyplen, int elmlen, bool elmbyval, char elmalign, bool *isNull); extern ArrayType *array_set(ArrayType *array, int nSubscripts, int *indx, - Datum dataValue, - int arraylen, int elmlen, bool elmbyval, char elmalign, - bool *isNull); + Datum dataValue, bool isNull, + int arraytyplen, int elmlen, bool elmbyval, char elmalign); extern ArrayType *array_get_slice(ArrayType *array, int nSubscripts, int *upperIndx, int *lowerIndx, - int arraylen, int elmlen, bool elmbyval, char elmalign, - bool *isNull); + int arraytyplen, int elmlen, bool elmbyval, char elmalign); extern ArrayType *array_set_slice(ArrayType *array, int nSubscripts, int *upperIndx, int *lowerIndx, - ArrayType *srcArray, - int arraylen, int elmlen, bool elmbyval, char elmalign, - bool *isNull); + ArrayType *srcArray, bool isNull, + int arraytyplen, int elmlen, bool elmbyval, char elmalign); extern Datum array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, ArrayMapState *amstate); +extern void array_bitmap_copy(bits8 *destbitmap, int destoffset, + const bits8 *srcbitmap, int srcoffset, + int nitems); + extern ArrayType *construct_array(Datum *elems, int nelems, Oid elmtype, int elmlen, bool elmbyval, char elmalign); extern ArrayType *construct_md_array(Datum *elems, + bool *nulls, int ndims, int *dims, int *lbs, Oid elmtype, int elmlen, bool elmbyval, char elmalign); +extern ArrayType *construct_empty_array(Oid elmtype); extern void deconstruct_array(ArrayType *array, Oid elmtype, int elmlen, bool elmbyval, char elmalign, - Datum **elemsp, int *nelemsp); + Datum **elemsp, bool **nullsp, int *nelemsp); extern ArrayBuildState *accumArrayResult(ArrayBuildState *astate, Datum dvalue, bool disnull, Oid element_type, @@ -189,13 +250,13 @@ extern Datum makeMdArrayResult(ArrayBuildState *astate, int ndims, * prototypes for functions defined in arrayutils.c */ -extern int ArrayGetOffset(int n, int *dim, int *lb, int *indx); -extern int ArrayGetOffset0(int n, int *tup, int *scale); -extern int ArrayGetNItems(int ndims, int *dims); -extern void mda_get_range(int n, int *span, int *st, int *endp); -extern void mda_get_prod(int n, int *range, int *prod); -extern void mda_get_offset_values(int n, int *dist, int *prod, int *span); -extern int mda_next_tuple(int n, int *curr, int *span); +extern int ArrayGetOffset(int n, const int *dim, const int *lb, const int *indx); +extern int ArrayGetOffset0(int n, const int *tup, const int *scale); +extern int ArrayGetNItems(int ndim, const int *dims); +extern void mda_get_range(int n, int *span, const int *st, const int *endp); +extern void mda_get_prod(int n, const int *range, int *prod); +extern void mda_get_offset_values(int n, int *dist, const int *prod, const int *span); +extern int mda_next_tuple(int n, int *curr, const int *span); /* * prototypes for functions defined in array_userfuncs.c diff --git a/src/pl/plpgsql/src/pl_comp.c b/src/pl/plpgsql/src/pl_comp.c index 2c84899519..f899bb2526 100644 --- a/src/pl/plpgsql/src/pl_comp.c +++ b/src/pl/plpgsql/src/pl_comp.c @@ -3,7 +3,7 @@ * procedural language * * IDENTIFICATION - * $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_comp.c,v 1.94 2005/10/15 02:49:49 momjian Exp $ + * $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_comp.c,v 1.95 2005/11/17 22:14:55 tgl Exp $ * * This software is copyrighted by Jan Wieck - Hamburg. * @@ -787,6 +787,7 @@ fetchArgInfo(HeapTuple procTup, Oid **p_argtypes, char ***p_argnames, numargs = ARR_DIMS(arr)[0]; if (ARR_NDIM(arr) != 1 || numargs < 0 || + ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != OIDOID) elog(ERROR, "proallargtypes is not a 1-D Oid array"); Assert(numargs >= procStruct->pronargs); @@ -814,7 +815,7 @@ fetchArgInfo(HeapTuple procTup, Oid **p_argtypes, char ***p_argnames, { deconstruct_array(DatumGetArrayTypeP(proargnames), TEXTOID, -1, false, 'i', - &elems, &nelems); + &elems, NULL, &nelems); if (nelems != numargs) /* should not happen */ elog(ERROR, "proargnames must have the same number of elements as the function has arguments"); *p_argnames = (char **) palloc(sizeof(char *) * numargs); @@ -834,6 +835,7 @@ fetchArgInfo(HeapTuple procTup, Oid **p_argtypes, char ***p_argnames, arr = DatumGetArrayTypeP(proargmodes); /* ensure not toasted */ if (ARR_NDIM(arr) != 1 || ARR_DIMS(arr)[0] != numargs || + ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != CHAROID) elog(ERROR, "proargmodes is not a 1-D char array"); *p_argmodes = (char *) palloc(numargs * sizeof(char)); diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c index df82dd3dc1..608854cbb5 100644 --- a/src/pl/plpgsql/src/pl_exec.c +++ b/src/pl/plpgsql/src/pl_exec.c @@ -3,7 +3,7 @@ * procedural language * * IDENTIFICATION - * $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_exec.c,v 1.154 2005/10/24 15:10:22 tgl Exp $ + * $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_exec.c,v 1.155 2005/11/17 22:14:55 tgl Exp $ * * This software is copyrighted by Jan Wieck - Hamburg. * @@ -3331,11 +3331,7 @@ exec_assign_value(PLpgSQL_execstate * estate, if (arraytyplen > 0) /* fixed-length array? */ return; - oldarrayval = construct_md_array(NULL, 0, NULL, NULL, - arrayelemtypeid, - elemtyplen, - elemtypbyval, - elemtypalign); + oldarrayval = construct_empty_array(arrayelemtypeid); } else oldarrayval = (ArrayType *) DatumGetPointer(oldarraydatum); @@ -3354,18 +3350,11 @@ exec_assign_value(PLpgSQL_execstate * estate, nsubscripts, subscriptvals, coerced_value, + *isNull, arraytyplen, elemtyplen, elemtypbyval, - elemtypalign, - isNull); - - /* - * Assign it to the base variable. - */ - exec_assign_value(estate, target, - PointerGetDatum(newarrayval), - arraytypeid, isNull); + elemtypalign); /* * Avoid leaking the result of exec_simple_cast_value, if it @@ -3375,6 +3364,15 @@ exec_assign_value(PLpgSQL_execstate * estate, pfree(DatumGetPointer(coerced_value)); /* + * Assign the new array to the base variable. It's never + * NULL at this point. + */ + *isNull = false; + exec_assign_value(estate, target, + PointerGetDatum(newarrayval), + arraytypeid, isNull); + + /* * Avoid leaking the modified array value, too. */ pfree(newarrayval); diff --git a/src/test/regress/expected/arrays.out b/src/test/regress/expected/arrays.out index dcc9e9c1ab..da218f0047 100644 --- a/src/test/regress/expected/arrays.out +++ b/src/test/regress/expected/arrays.out @@ -63,9 +63,9 @@ SELECT a[1:3], FROM arrtest; a | b | c | d ------------+-----------------+-----------+--------------- - {1,2,3} | {{{0,0},{1,2}}} | | - {11,12,23} | | {foobar} | {{elt1,elt2}} - | | {foo,bar} | + {1,2,3} | {{{0,0},{1,2}}} | {} | {} + {11,12,23} | {} | {foobar} | {{elt1,elt2}} + {} | {} | {foo,bar} | {} (3 rows) SELECT array_dims(a) AS a,array_dims(b) AS b,array_dims(c) AS c @@ -111,9 +111,36 @@ SELECT a[1:3], FROM arrtest; a | b | c | d ------------+-----------------------+-------------------+---------- - {16,25,3} | {{{113,142},{1,147}}} | | - | | {foo,new_word} | - {16,25,23} | | {foobar,new_word} | {{elt2}} + {16,25,3} | {{{113,142},{1,147}}} | {} | {} + {} | {} | {foo,new_word} | {} + {16,25,23} | {} | {foobar,new_word} | {{elt2}} +(3 rows) + +INSERT INTO arrtest(a) VALUES('{1,null,3}'); +SELECT a FROM arrtest; + a +--------------- + {16,25,3,4,5} + {} + {16,25,23} + {1,NULL,3} +(4 rows) + +UPDATE arrtest SET a[4] = NULL WHERE a[2] IS NULL; +SELECT a FROM arrtest WHERE a[2] IS NULL; + a +----------------- + [4:4]={NULL} + {1,NULL,3,NULL} +(2 rows) + +DELETE FROM arrtest WHERE a[2] IS NULL AND b IS NULL; +SELECT a,b,c FROM arrtest; + a | b | c +---------------+-----------------------+------------------- + {16,25,3,4,5} | {{{113,142},{1,147}}} | {} + {16,25,23} | {{3,4},{4,5}} | {foobar,new_word} + [4:4]={NULL} | {3,4} | {foo,new_word} (3 rows) -- @@ -176,6 +203,19 @@ SELECT ARRAY(select f2 from arrtest_f order by f2) AS "ARRAY"; {1.15,1.15,1.18,1.21,1.24,1.26,1.26,1.3,1.32} (1 row) +-- with nulls +SELECT '{1,null,3}'::int[]; + int4 +------------ + {1,NULL,3} +(1 row) + +SELECT ARRAY[1,NULL,3]; + array +------------ + {1,NULL,3} +(1 row) + -- functions SELECT array_append(array[42], 6) AS "{42,6}"; {42,6} @@ -355,6 +395,55 @@ select 33 * any ('{1,2,3}'); ERROR: op ANY/ALL (array) requires operator to yield boolean select 33 * any (44); ERROR: op ANY/ALL (array) requires array on right side +-- nulls +select 33 = any (null::int[]); + ?column? +---------- + +(1 row) + +select null::int = any ('{1,2,3}'); + ?column? +---------- + +(1 row) + +select 33 = any ('{1,null,3}'); + ?column? +---------- + +(1 row) + +select 33 = any ('{1,null,33}'); + ?column? +---------- + t +(1 row) + +select 33 = all (null::int[]); + ?column? +---------- + +(1 row) + +select null::int = all ('{1,2,3}'); + ?column? +---------- + +(1 row) + +select 33 = all ('{1,null,3}'); + ?column? +---------- + f +(1 row) + +select 33 = all ('{33,null,33}'); + ?column? +---------- + +(1 row) + -- test indexes on arrays create temp table arr_tbl (f1 int[] unique); NOTICE: CREATE TABLE / UNIQUE will create implicit index "arr_tbl_f1_key" for table "arr_tbl" diff --git a/src/test/regress/expected/domain.out b/src/test/regress/expected/domain.out index d2766ee2a4..5309234ce2 100644 --- a/src/test/regress/expected/domain.out +++ b/src/test/regress/expected/domain.out @@ -91,7 +91,7 @@ select testint4arr[1], testtextarr[2:2] from domarrtest; testint4arr | testtextarr -------------+------------- 2 | {{c,d}} - | + | {} 2 | {{c,d}} 2 | {{c}} | {{d,e,f}} diff --git a/src/test/regress/sql/arrays.sql b/src/test/regress/sql/arrays.sql index bc4d1345fb..d0574beda0 100644 --- a/src/test/regress/sql/arrays.sql +++ b/src/test/regress/sql/arrays.sql @@ -83,6 +83,13 @@ SELECT a[1:3], d[1:1][2:2] FROM arrtest; +INSERT INTO arrtest(a) VALUES('{1,null,3}'); +SELECT a FROM arrtest; +UPDATE arrtest SET a[4] = NULL WHERE a[2] IS NULL; +SELECT a FROM arrtest WHERE a[2] IS NULL; +DELETE FROM arrtest WHERE a[2] IS NULL AND b IS NULL; +SELECT a,b,c FROM arrtest; + -- -- array expressions and operators -- @@ -128,6 +135,10 @@ SELECT ARRAY[[[[[['hello'],['world']]]]]]; SELECT ARRAY[ARRAY['hello'],ARRAY['world']]; SELECT ARRAY(select f2 from arrtest_f order by f2) AS "ARRAY"; +-- with nulls +SELECT '{1,null,3}'::int[]; +SELECT ARRAY[1,NULL,3]; + -- functions SELECT array_append(array[42], 6) AS "{42,6}"; SELECT array_prepend(6, array[42]) AS "{6,42}"; @@ -168,6 +179,15 @@ select 33.4 > all (array[1,2,3]); -- errors select 33 * any ('{1,2,3}'); select 33 * any (44); +-- nulls +select 33 = any (null::int[]); +select null::int = any ('{1,2,3}'); +select 33 = any ('{1,null,3}'); +select 33 = any ('{1,null,33}'); +select 33 = all (null::int[]); +select null::int = all ('{1,2,3}'); +select 33 = all ('{1,null,3}'); +select 33 = all ('{33,null,33}'); -- test indexes on arrays create temp table arr_tbl (f1 int[] unique); -- 2.11.0