src/backend/utils/mb/wchar.c

   1 /*
   2  * conversion functions between pg_wchar and multibyte streams.
   3  * Tatsuo Ishii
   4  * $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.74 2010/01/04 20:38:31 adunstan Exp $
   5  *
   6  */
   7 /* can be used in either frontend or backend */
   8 #ifdef FRONTEND
   9 #include "postgres_fe.h"
  10 #define Assert(condition)
  11 #else
  12 #include "postgres.h"
  13 #endif
  14
  15 #include "mb/pg_wchar.h"
  16
  17
  18 /*
  19  * conversion to pg_wchar is done by "table driven."
  20  * to add an encoding support, define mb2wchar_with_len(), mblen(), dsplen()
  21  * for the particular encoding. Note that if the encoding is only
  22  * supported in the client, you don't need to define
  23  * mb2wchar_with_len() function (SJIS is the case).
  24  *
  25  * These functions generally assume that their input is validly formed.
  26  * The "verifier" functions, further down in the file, have to be more
  27  * paranoid.  We expect that mblen() does not need to examine more than
  28  * the first byte of the character to discover the correct length.
  29  *
  30  * Note: for the display output of psql to work properly, the return values
  31  * of the dsplen functions must conform to the Unicode standard. In particular
  32  * the NUL character is zero width and control characters are generally
  33  * width -1. It is recommended that non-ASCII encodings refer their ASCII
  34  * subset to the ASCII routines to ensure consistency.
  35  */
  36
  37 /*
  38  * SQL/ASCII
  39  */
  40 static int
  41 pg_ascii2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
  42 {
  43         int                     cnt = 0;
  44
  45         while (len > 0 && *from)
  46         {
  47                 *to++ = *from++;
  48                 len--;
  49                 cnt++;
  50         }
  51         *to = 0;
  52         return cnt;
  53 }
  54
  55 static int
  56 pg_ascii_mblen(const unsigned char *s)
  57 {
  58         return 1;
  59 }
  60
  61 static int
  62 pg_ascii_dsplen(const unsigned char *s)
  63 {
  64         if (*s == '\0')
  65                 return 0;
  66         if (*s < 0x20 || *s == 0x7f)
  67                 return -1;
  68
  69         return 1;
  70 }
  71
  72 /*
  73  * EUC
  74  */
  75 static int
  76 pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
  77 {
  78         int                     cnt = 0;
  79
  80         while (len > 0 && *from)
  81         {
  82                 if (*from == SS2 && len >= 2)   /* JIS X 0201 (so called "1 byte
  83                                                                                  * KANA") */
  84                 {
  85                         from++;
  86                         *to = (SS2 << 8) | *from++;
  87                         len -= 2;
  88                 }
  89                 else if (*from == SS3 && len >= 3)              /* JIS X 0212 KANJI */
  90                 {
  91                         from++;
  92                         *to = (SS3 << 16) | (*from++ << 8);
  93                         *to |= *from++;
  94                         len -= 3;
  95                 }
  96                 else if (IS_HIGHBIT_SET(*from) && len >= 2)             /* JIS X 0208 KANJI */
  97                 {
  98                         *to = *from++ << 8;
  99                         *to |= *from++;
 100                         len -= 2;
 101                 }
 102                 else
 103                         /* must be ASCII */
 104                 {
 105                         *to = *from++;
 106                         len--;
 107                 }
 108                 to++;
 109                 cnt++;
 110         }
 111         *to = 0;
 112         return cnt;
 113 }
 114
 115 static inline int
 116 pg_euc_mblen(const unsigned char *s)
 117 {
 118         int                     len;
 119
 120         if (*s == SS2)
 121                 len = 2;
 122         else if (*s == SS3)
 123                 len = 3;
 124         else if (IS_HIGHBIT_SET(*s))
 125                 len = 2;
 126         else
 127                 len = 1;
 128         return len;
 129 }
 130
 131 static inline int
 132 pg_euc_dsplen(const unsigned char *s)
 133 {
 134         int                     len;
 135
 136         if (*s == SS2)
 137                 len = 2;
 138         else if (*s == SS3)
 139                 len = 2;
 140         else if (IS_HIGHBIT_SET(*s))
 141                 len = 2;
 142         else
 143                 len = pg_ascii_dsplen(s);
 144         return len;
 145 }
 146
 147 /*
 148  * EUC_JP
 149  */
 150 static int
 151 pg_eucjp2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
 152 {
 153         return pg_euc2wchar_with_len(from, to, len);
 154 }
 155
 156 static int
 157 pg_eucjp_mblen(const unsigned char *s)
 158 {
 159         return pg_euc_mblen(s);
 160 }
 161
 162 static int
 163 pg_eucjp_dsplen(const unsigned char *s)
 164 {
 165         int                     len;
 166
 167         if (*s == SS2)
 168                 len = 1;
 169         else if (*s == SS3)
 170                 len = 2;
 171         else if (IS_HIGHBIT_SET(*s))
 172                 len = 2;
 173         else
 174                 len = pg_ascii_dsplen(s);
 175         return len;
 176 }
 177
 178 /*
 179  * EUC_KR
 180  */
 181 static int
 182 pg_euckr2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
 183 {
 184         return pg_euc2wchar_with_len(from, to, len);
 185 }
 186
 187 static int
 188 pg_euckr_mblen(const unsigned char *s)
 189 {
 190         return pg_euc_mblen(s);
 191 }
 192
 193 static int
 194 pg_euckr_dsplen(const unsigned char *s)
 195 {
 196         return pg_euc_dsplen(s);
 197 }
 198
 199 /*
 200  * EUC_CN
 201  *
 202  */
 203 static int
 204 pg_euccn2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
 205 {
 206         int                     cnt = 0;
 207
 208         while (len > 0 && *from)
 209         {
 210                 if (*from == SS2 && len >= 3)   /* code set 2 (unused?) */
 211                 {
 212                         from++;
 213                         *to = (SS2 << 16) | (*from++ << 8);
 214                         *to |= *from++;
 215                         len -= 3;
 216                 }
 217                 else if (*from == SS3 && len >= 3)              /* code set 3 (unsed ?) */
 218                 {
 219                         from++;
 220                         *to = (SS3 << 16) | (*from++ << 8);
 221                         *to |= *from++;
 222                         len -= 3;
 223                 }
 224                 else if (IS_HIGHBIT_SET(*from) && len >= 2)             /* code set 1 */
 225                 {
 226                         *to = *from++ << 8;
 227                         *to |= *from++;
 228                         len -= 2;
 229                 }
 230                 else
 231                 {
 232                         *to = *from++;
 233                         len--;
 234                 }
 235                 to++;
 236                 cnt++;
 237         }
 238         *to = 0;
 239         return cnt;
 240 }
 241
 242 static int
 243 pg_euccn_mblen(const unsigned char *s)
 244 {
 245         int                     len;
 246
 247         if (IS_HIGHBIT_SET(*s))
 248                 len = 2;
 249         else
 250                 len = 1;
 251         return len;
 252 }
 253
 254 static int
 255 pg_euccn_dsplen(const unsigned char *s)
 256 {
 257         int                     len;
 258
 259         if (IS_HIGHBIT_SET(*s))
 260                 len = 2;
 261         else
 262                 len = pg_ascii_dsplen(s);
 263         return len;
 264 }
 265
 266 /*
 267  * EUC_TW
 268  *
 269  */
 270 static int
 271 pg_euctw2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
 272 {
 273         int                     cnt = 0;
 274
 275         while (len > 0 && *from)
 276         {
 277                 if (*from == SS2 && len >= 4)   /* code set 2 */
 278                 {
 279                         from++;
 280                         *to = (((uint32) SS2) << 24) | (*from++ << 16);
 281                         *to |= *from++ << 8;
 282                         *to |= *from++;
 283                         len -= 4;
 284                 }
 285                 else if (*from == SS3 && len >= 3)              /* code set 3 (unused?) */
 286                 {
 287                         from++;
 288                         *to = (SS3 << 16) | (*from++ << 8);
 289                         *to |= *from++;
 290                         len -= 3;
 291                 }
 292                 else if (IS_HIGHBIT_SET(*from) && len >= 2)             /* code set 2 */
 293                 {
 294                         *to = *from++ << 8;
 295                         *to |= *from++;
 296                         len -= 2;
 297                 }
 298                 else
 299                 {
 300                         *to = *from++;
 301                         len--;
 302                 }
 303                 to++;
 304                 cnt++;
 305         }
 306         *to = 0;
 307         return cnt;
 308 }
 309
 310 static int
 311 pg_euctw_mblen(const unsigned char *s)
 312 {
 313         int                     len;
 314
 315         if (*s == SS2)
 316                 len = 4;
 317         else if (*s == SS3)
 318                 len = 3;
 319         else if (IS_HIGHBIT_SET(*s))
 320                 len = 2;
 321         else
 322                 len = 1;
 323         return len;
 324 }
 325
 326 static int
 327 pg_euctw_dsplen(const unsigned char *s)
 328 {
 329         int                     len;
 330
 331         if (*s == SS2)
 332                 len = 2;
 333         else if (*s == SS3)
 334                 len = 2;
 335         else if (IS_HIGHBIT_SET(*s))
 336                 len = 2;
 337         else
 338                 len = pg_ascii_dsplen(s);
 339         return len;
 340 }
 341
 342 /*
 343  * JOHAB
 344  */
 345 static int
 346 pg_johab_mblen(const unsigned char *s)
 347 {
 348         return pg_euc_mblen(s);
 349 }
 350
 351 static int
 352 pg_johab_dsplen(const unsigned char *s)
 353 {
 354         return pg_euc_dsplen(s);
 355 }
 356
 357 /*
 358  * convert UTF8 string to pg_wchar (UCS-4)
 359  * caller must allocate enough space for "to", including a trailing zero!
 360  * len: length of from.
 361  * "from" not necessarily null terminated.
 362  */
 363 static int
 364 pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
 365 {
 366         int                     cnt = 0;
 367         uint32          c1,
 368                                 c2,
 369                                 c3,
 370                                 c4;
 371
 372         while (len > 0 && *from)
 373         {
 374                 if ((*from & 0x80) == 0)
 375                 {
 376                         *to = *from++;
 377                         len--;
 378                 }
 379                 else if ((*from & 0xe0) == 0xc0)
 380                 {
 381                         if (len < 2)
 382                                 break;                  /* drop trailing incomplete char */
 383                         c1 = *from++ & 0x1f;
 384                         c2 = *from++ & 0x3f;
 385                         *to = (c1 << 6) | c2;
 386                         len -= 2;
 387                 }
 388                 else if ((*from & 0xf0) == 0xe0)
 389                 {
 390                         if (len < 3)
 391                                 break;                  /* drop trailing incomplete char */
 392                         c1 = *from++ & 0x0f;
 393                         c2 = *from++ & 0x3f;
 394                         c3 = *from++ & 0x3f;
 395                         *to = (c1 << 12) | (c2 << 6) | c3;
 396                         len -= 3;
 397                 }
 398                 else if ((*from & 0xf8) == 0xf0)
 399                 {
 400                         if (len < 4)
 401                                 break;                  /* drop trailing incomplete char */
 402                         c1 = *from++ & 0x07;
 403                         c2 = *from++ & 0x3f;
 404                         c3 = *from++ & 0x3f;
 405                         c4 = *from++ & 0x3f;
 406                         *to = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
 407                         len -= 4;
 408                 }
 409                 else
 410                 {
 411                         /* treat a bogus char as length 1; not ours to raise error */
 412                         *to = *from++;
 413                         len--;
 414                 }
 415                 to++;
 416                 cnt++;
 417         }
 418         *to = 0;
 419         return cnt;
 420 }
 421
 422
 423 /*
 424  * Map a Unicode code point to UTF-8.  utf8string must have 4 bytes of
 425  * space allocated.
 426  */
 427 unsigned char *
 428 unicode_to_utf8(pg_wchar c, unsigned char *utf8string)
 429 {
 430         if (c <= 0x7F)
 431         {
 432                 utf8string[0] = c;
 433         }
 434         else if (c <= 0x7FF)
 435         {
 436                 utf8string[0] = 0xC0 | ((c >> 6) & 0x1F);
 437                 utf8string[1] = 0x80 | (c & 0x3F);
 438         }
 439         else if (c <= 0xFFFF)
 440         {
 441                 utf8string[0] = 0xE0 | ((c >> 12) & 0x0F);
 442                 utf8string[1] = 0x80 | ((c >> 6) & 0x3F);
 443                 utf8string[2] = 0x80 | (c & 0x3F);
 444         }
 445         else
 446         {
 447                 utf8string[0] = 0xF0 | ((c >> 18) & 0x07);
 448                 utf8string[1] = 0x80 | ((c >> 12) & 0x3F);
 449                 utf8string[2] = 0x80 | ((c >> 6) & 0x3F);
 450                 utf8string[3] = 0x80 | (c & 0x3F);
 451         }
 452
 453         return utf8string;
 454 }
 455
 456
 457 /*
 458  * Return the byte length of a UTF8 character pointed to by s
 459  *
 460  * Note: in the current implementation we do not support UTF8 sequences
 461  * of more than 4 bytes; hence do NOT return a value larger than 4.
 462  * We return "1" for any leading byte that is either flat-out illegal or
 463  * indicates a length larger than we support.
 464  *
 465  * pg_utf2wchar_with_len(), utf2ucs(), pg_utf8_islegal(), and perhaps
 466  * other places would need to be fixed to change this.
 467  */
 468 int
 469 pg_utf_mblen(const unsigned char *s)
 470 {
 471         int                     len;
 472
 473         if ((*s & 0x80) == 0)
 474                 len = 1;
 475         else if ((*s & 0xe0) == 0xc0)
 476                 len = 2;
 477         else if ((*s & 0xf0) == 0xe0)
 478                 len = 3;
 479         else if ((*s & 0xf8) == 0xf0)
 480                 len = 4;
 481 #ifdef NOT_USED
 482         else if ((*s & 0xfc) == 0xf8)
 483                 len = 5;
 484         else if ((*s & 0xfe) == 0xfc)
 485                 len = 6;
 486 #endif
 487         else
 488                 len = 1;
 489         return len;
 490 }
 491
 492 /*
 493  * This is an implementation of wcwidth() and wcswidth() as defined in
 494  * "The Single UNIX Specification, Version 2, The Open Group, 1997"
 495  * <http://www.UNIX-systems.org/online.html>
 496  *
 497  * Markus Kuhn -- 2001-09-08 -- public domain
 498  *
 499  * customised for PostgreSQL
 500  *
 501  * original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
 502  */
 503
 504 struct mbinterval
 505 {
 506         unsigned short first;
 507         unsigned short last;
 508 };
 509
 510 /* auxiliary function for binary search in interval table */
 511 static int
 512 mbbisearch(pg_wchar ucs, const struct mbinterval * table, int max)
 513 {
 514         int                     min = 0;
 515         int                     mid;
 516
 517         if (ucs < table[0].first || ucs > table[max].last)
 518                 return 0;
 519         while (max >= min)
 520         {
 521                 mid = (min + max) / 2;
 522                 if (ucs > table[mid].last)
 523                         min = mid + 1;
 524                 else if (ucs < table[mid].first)
 525                         max = mid - 1;
 526                 else
 527                         return 1;
 528         }
 529
 530         return 0;
 531 }
 532
 533
 534 /* The following functions define the column width of an ISO 10646
 535  * character as follows:
 536  *
 537  *        - The null character (U+0000) has a column width of 0.
 538  *
 539  *        - Other C0/C1 control characters and DEL will lead to a return
 540  *              value of -1.
 541  *
 542  *        - Non-spacing and enclosing combining characters (general
 543  *              category code Mn or Me in the Unicode database) have a
 544  *              column width of 0.
 545  *
 546  *        - Other format characters (general category code Cf in the Unicode
 547  *              database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
 548  *
 549  *        - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
 550  *              have a column width of 0.
 551  *
 552  *        - Spacing characters in the East Asian Wide (W) or East Asian
 553  *              FullWidth (F) category as defined in Unicode Technical
 554  *              Report #11 have a column width of 2.
 555  *
 556  *        - All remaining characters (including all printable
 557  *              ISO 8859-1 and WGL4 characters, Unicode control characters,
 558  *              etc.) have a column width of 1.
 559  *
 560  * This implementation assumes that wchar_t characters are encoded
 561  * in ISO 10646.
 562  */
 563
 564 static int
 565 ucs_wcwidth(pg_wchar ucs)
 566 {
 567         /* sorted list of non-overlapping intervals of non-spacing characters */
 568         static const struct mbinterval combining[] = {
 569                 {0x0300, 0x034E}, {0x0360, 0x0362}, {0x0483, 0x0486},
 570                 {0x0488, 0x0489}, {0x0591, 0x05A1}, {0x05A3, 0x05B9},
 571                 {0x05BB, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C2},
 572                 {0x05C4, 0x05C4}, {0x064B, 0x0655}, {0x0670, 0x0670},
 573                 {0x06D6, 0x06E4}, {0x06E7, 0x06E8}, {0x06EA, 0x06ED},
 574                 {0x070F, 0x070F}, {0x0711, 0x0711}, {0x0730, 0x074A},
 575                 {0x07A6, 0x07B0}, {0x0901, 0x0902}, {0x093C, 0x093C},
 576                 {0x0941, 0x0948}, {0x094D, 0x094D}, {0x0951, 0x0954},
 577                 {0x0962, 0x0963}, {0x0981, 0x0981}, {0x09BC, 0x09BC},
 578                 {0x09C1, 0x09C4}, {0x09CD, 0x09CD}, {0x09E2, 0x09E3},
 579                 {0x0A02, 0x0A02}, {0x0A3C, 0x0A3C}, {0x0A41, 0x0A42},
 580                 {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A70, 0x0A71},
 581                 {0x0A81, 0x0A82}, {0x0ABC, 0x0ABC}, {0x0AC1, 0x0AC5},
 582                 {0x0AC7, 0x0AC8}, {0x0ACD, 0x0ACD}, {0x0B01, 0x0B01},
 583                 {0x0B3C, 0x0B3C}, {0x0B3F, 0x0B3F}, {0x0B41, 0x0B43},
 584                 {0x0B4D, 0x0B4D}, {0x0B56, 0x0B56}, {0x0B82, 0x0B82},
 585                 {0x0BC0, 0x0BC0}, {0x0BCD, 0x0BCD}, {0x0C3E, 0x0C40},
 586                 {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56},
 587                 {0x0CBF, 0x0CBF}, {0x0CC6, 0x0CC6}, {0x0CCC, 0x0CCD},
 588                 {0x0D41, 0x0D43}, {0x0D4D, 0x0D4D}, {0x0DCA, 0x0DCA},
 589                 {0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0E31, 0x0E31},
 590                 {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, {0x0EB1, 0x0EB1},
 591                 {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, {0x0EC8, 0x0ECD},
 592                 {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37},
 593                 {0x0F39, 0x0F39}, {0x0F71, 0x0F7E}, {0x0F80, 0x0F84},
 594                 {0x0F86, 0x0F87}, {0x0F90, 0x0F97}, {0x0F99, 0x0FBC},
 595                 {0x0FC6, 0x0FC6}, {0x102D, 0x1030}, {0x1032, 0x1032},
 596                 {0x1036, 0x1037}, {0x1039, 0x1039}, {0x1058, 0x1059},
 597                 {0x1160, 0x11FF}, {0x17B7, 0x17BD}, {0x17C6, 0x17C6},
 598                 {0x17C9, 0x17D3}, {0x180B, 0x180E}, {0x18A9, 0x18A9},
 599                 {0x200B, 0x200F}, {0x202A, 0x202E}, {0x206A, 0x206F},
 600                 {0x20D0, 0x20E3}, {0x302A, 0x302F}, {0x3099, 0x309A},
 601                 {0xFB1E, 0xFB1E}, {0xFE20, 0xFE23}, {0xFEFF, 0xFEFF},
 602                 {0xFFF9, 0xFFFB}
 603         };
 604
 605         /* test for 8-bit control characters */
 606         if (ucs == 0)
 607                 return 0;
 608
 609         if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff)
 610                 return -1;
 611
 612         /* binary search in table of non-spacing characters */
 613         if (mbbisearch(ucs, combining,
 614                                    sizeof(combining) / sizeof(struct mbinterval) - 1))
 615                 return 0;
 616
 617         /*
 618          * if we arrive here, ucs is not a combining or C0/C1 control character
 619          */
 620
 621         return 1 +
 622                 (ucs >= 0x1100 &&
 623                  (ucs <= 0x115f ||              /* Hangul Jamo init. consonants */
 624                   (ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & ~0x0011) != 0x300a &&
 625                    ucs != 0x303f) ||    /* CJK ... Yi */
 626                   (ucs >= 0xac00 && ucs <= 0xd7a3) ||   /* Hangul Syllables */
 627                   (ucs >= 0xf900 && ucs <= 0xfaff) ||   /* CJK Compatibility
 628                                                                                                  * Ideographs */
 629                   (ucs >= 0xfe30 && ucs <= 0xfe6f) ||   /* CJK Compatibility Forms */
 630                   (ucs >= 0xff00 && ucs <= 0xff5f) ||   /* Fullwidth Forms */
 631                   (ucs >= 0xffe0 && ucs <= 0xffe6) ||
 632                   (ucs >= 0x20000 && ucs <= 0x2ffff)));
 633 }
 634
 635 static pg_wchar
 636 utf2ucs(const unsigned char *c)
 637 {
 638         /*
 639          * one char version of pg_utf2wchar_with_len. no control here, c must
 640          * point to a large enough string
 641          */
 642         if ((*c & 0x80) == 0)
 643                 return (pg_wchar) c[0];
 644         else if ((*c & 0xe0) == 0xc0)
 645                 return (pg_wchar) (((c[0] & 0x1f) << 6) |
 646                                                    (c[1] & 0x3f));
 647         else if ((*c & 0xf0) == 0xe0)
 648                 return (pg_wchar) (((c[0] & 0x0f) << 12) |
 649                                                    ((c[1] & 0x3f) << 6) |
 650                                                    (c[2] & 0x3f));
 651         else if ((*c & 0xf8) == 0xf0)
 652                 return (pg_wchar) (((c[0] & 0x07) << 18) |
 653                                                    ((c[1] & 0x3f) << 12) |
 654                                                    ((c[2] & 0x3f) << 6) |
 655                                                    (c[3] & 0x3f));
 656         else
 657                 /* that is an invalid code on purpose */
 658                 return 0xffffffff;
 659 }
 660
 661 static int
 662 pg_utf_dsplen(const unsigned char *s)
 663 {
 664         return ucs_wcwidth(utf2ucs(s));
 665 }
 666
 667 /*
 668  * convert mule internal code to pg_wchar
 669  * caller should allocate enough space for "to"
 670  * len: length of from.
 671  * "from" not necessarily null terminated.
 672  */
 673 static int
 674 pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
 675 {
 676         int                     cnt = 0;
 677
 678         while (len > 0 && *from)
 679         {
 680                 if (IS_LC1(*from) && len >= 2)
 681                 {
 682                         *to = *from++ << 16;
 683                         *to |= *from++;
 684                         len -= 2;
 685                 }
 686                 else if (IS_LCPRV1(*from) && len >= 3)
 687                 {
 688                         from++;
 689                         *to = *from++ << 16;
 690                         *to |= *from++;
 691                         len -= 3;
 692                 }
 693                 else if (IS_LC2(*from) && len >= 3)
 694                 {
 695                         *to = *from++ << 16;
 696                         *to |= *from++ << 8;
 697                         *to |= *from++;
 698                         len -= 3;
 699                 }
 700                 else if (IS_LCPRV2(*from) && len >= 4)
 701                 {
 702                         from++;
 703                         *to = *from++ << 16;
 704                         *to |= *from++ << 8;
 705                         *to |= *from++;
 706                         len -= 4;
 707                 }
 708                 else
 709                 {                                               /* assume ASCII */
 710                         *to = (unsigned char) *from++;
 711                         len--;
 712                 }
 713                 to++;
 714                 cnt++;
 715         }
 716         *to = 0;
 717         return cnt;
 718 }
 719
 720 int
 721 pg_mule_mblen(const unsigned char *s)
 722 {
 723         int                     len;
 724
 725         if (IS_LC1(*s))
 726                 len = 2;
 727         else if (IS_LCPRV1(*s))
 728                 len = 3;
 729         else if (IS_LC2(*s))
 730                 len = 3;
 731         else if (IS_LCPRV2(*s))
 732                 len = 4;
 733         else
 734                 len = 1;                                /* assume ASCII */
 735         return len;
 736 }
 737
 738 static int
 739 pg_mule_dsplen(const unsigned char *s)
 740 {
 741         int                     len;
 742
 743         if (IS_LC1(*s))
 744                 len = 1;
 745         else if (IS_LCPRV1(*s))
 746                 len = 1;
 747         else if (IS_LC2(*s))
 748                 len = 2;
 749         else if (IS_LCPRV2(*s))
 750                 len = 2;
 751         else
 752                 len = 1;                                /* assume ASCII */
 753
 754         return len;
 755 }
 756
 757 /*
 758  * ISO8859-1
 759  */
 760 static int
 761 pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
 762 {
 763         int                     cnt = 0;
 764
 765         while (len > 0 && *from)
 766         {
 767                 *to++ = *from++;
 768                 len--;
 769                 cnt++;
 770         }
 771         *to = 0;
 772         return cnt;
 773 }
 774
 775 static int
 776 pg_latin1_mblen(const unsigned char *s)
 777 {
 778         return 1;
 779 }
 780
 781 static int
 782 pg_latin1_dsplen(const unsigned char *s)
 783 {
 784         return pg_ascii_dsplen(s);
 785 }
 786
 787 /*
 788  * SJIS
 789  */
 790 static int
 791 pg_sjis_mblen(const unsigned char *s)
 792 {
 793         int                     len;
 794
 795         if (*s >= 0xa1 && *s <= 0xdf)
 796                 len = 1;                                /* 1 byte kana? */
 797         else if (IS_HIGHBIT_SET(*s))
 798                 len = 2;                                /* kanji? */
 799         else
 800                 len = 1;                                /* should be ASCII */
 801         return len;
 802 }
 803
 804 static int
 805 pg_sjis_dsplen(const unsigned char *s)
 806 {
 807         int                     len;
 808
 809         if (*s >= 0xa1 && *s <= 0xdf)
 810                 len = 1;                                /* 1 byte kana? */
 811         else if (IS_HIGHBIT_SET(*s))
 812                 len = 2;                                /* kanji? */
 813         else
 814                 len = pg_ascii_dsplen(s);               /* should be ASCII */
 815         return len;
 816 }
 817
 818 /*
 819  * Big5
 820  */
 821 static int
 822 pg_big5_mblen(const unsigned char *s)
 823 {
 824         int                     len;
 825
 826         if (IS_HIGHBIT_SET(*s))
 827                 len = 2;                                /* kanji? */
 828         else
 829                 len = 1;                                /* should be ASCII */
 830         return len;
 831 }
 832
 833 static int
 834 pg_big5_dsplen(const unsigned char *s)
 835 {
 836         int                     len;
 837
 838         if (IS_HIGHBIT_SET(*s))
 839                 len = 2;                                /* kanji? */
 840         else
 841                 len = pg_ascii_dsplen(s);               /* should be ASCII */
 842         return len;
 843 }
 844
 845 /*
 846  * GBK
 847  */
 848 static int
 849 pg_gbk_mblen(const unsigned char *s)
 850 {
 851         int                     len;
 852
 853         if (IS_HIGHBIT_SET(*s))
 854                 len = 2;                                /* kanji? */
 855         else
 856                 len = 1;                                /* should be ASCII */
 857         return len;
 858 }
 859
 860 static int
 861 pg_gbk_dsplen(const unsigned char *s)
 862 {
 863         int                     len;
 864
 865         if (IS_HIGHBIT_SET(*s))
 866                 len = 2;                                /* kanji? */
 867         else
 868                 len = pg_ascii_dsplen(s);               /* should be ASCII */
 869         return len;
 870 }
 871
 872 /*
 873  * UHC
 874  */
 875 static int
 876 pg_uhc_mblen(const unsigned char *s)
 877 {
 878         int                     len;
 879
 880         if (IS_HIGHBIT_SET(*s))
 881                 len = 2;                                /* 2byte? */
 882         else
 883                 len = 1;                                /* should be ASCII */
 884         return len;
 885 }
 886
 887 static int
 888 pg_uhc_dsplen(const unsigned char *s)
 889 {
 890         int                     len;
 891
 892         if (IS_HIGHBIT_SET(*s))
 893                 len = 2;                                /* 2byte? */
 894         else
 895                 len = pg_ascii_dsplen(s);               /* should be ASCII */
 896         return len;
 897 }
 898
 899 /*
 900  *      * GB18030
 901  *       * Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
 902  *        */
 903 static int
 904 pg_gb18030_mblen(const unsigned char *s)
 905 {
 906         int                     len;
 907
 908         if (!IS_HIGHBIT_SET(*s))
 909                 len = 1;                                /* ASCII */
 910         else
 911         {
 912                 if ((*(s + 1) >= 0x40 && *(s + 1) <= 0x7e) || (*(s + 1) >= 0x80 && *(s + 1) <= 0xfe))
 913                         len = 2;
 914                 else if (*(s + 1) >= 0x30 && *(s + 1) <= 0x39)
 915                         len = 4;
 916                 else
 917                         len = 2;
 918         }
 919         return len;
 920 }
 921
 922 static int
 923 pg_gb18030_dsplen(const unsigned char *s)
 924 {
 925         int                     len;
 926
 927         if (IS_HIGHBIT_SET(*s))
 928                 len = 2;
 929         else
 930                 len = pg_ascii_dsplen(s);               /* ASCII */
 931         return len;
 932 }
 933
 934 /*
 935  *-------------------------------------------------------------------
 936  * multibyte sequence validators
 937  *
 938  * These functions accept "s", a pointer to the first byte of a string,
 939  * and "len", the remaining length of the string.  If there is a validly
 940  * encoded character beginning at *s, return its length in bytes; else
 941  * return -1.
 942  *
 943  * The functions can assume that len > 0 and that *s != '\0', but they must
 944  * test for and reject zeroes in any additional bytes of a multibyte character.
 945  *
 946  * Note that this definition allows the function for a single-byte
 947  * encoding to be just "return 1".
 948  *-------------------------------------------------------------------
 949  */
 950
 951 static int
 952 pg_ascii_verifier(const unsigned char *s, int len)
 953 {
 954         return 1;
 955 }
 956
 957 #define IS_EUC_RANGE_VALID(c)   ((c) >= 0xa1 && (c) <= 0xfe)
 958
 959 static int
 960 pg_eucjp_verifier(const unsigned char *s, int len)
 961 {
 962         int                     l;
 963         unsigned char c1,
 964                                 c2;
 965
 966         c1 = *s++;
 967
 968         switch (c1)
 969         {
 970                 case SS2:                               /* JIS X 0201 */
 971                         l = 2;
 972                         if (l > len)
 973                                 return -1;
 974                         c2 = *s++;
 975                         if (c2 < 0xa1 || c2 > 0xdf)
 976                                 return -1;
 977                         break;
 978
 979                 case SS3:                               /* JIS X 0212 */
 980                         l = 3;
 981                         if (l > len)
 982                                 return -1;
 983                         c2 = *s++;
 984                         if (!IS_EUC_RANGE_VALID(c2))
 985                                 return -1;
 986                         c2 = *s++;
 987                         if (!IS_EUC_RANGE_VALID(c2))
 988                                 return -1;
 989                         break;
 990
 991                 default:
 992                         if (IS_HIGHBIT_SET(c1))         /* JIS X 0208? */
 993                         {
 994                                 l = 2;
 995                                 if (l > len)
 996                                         return -1;
 997                                 if (!IS_EUC_RANGE_VALID(c1))
 998                                         return -1;
 999                                 c2 = *s++;
1000                                 if (!IS_EUC_RANGE_VALID(c2))
1001                                         return -1;
1002                         }
1003                         else
1004                                 /* must be ASCII */
1005                         {
1006                                 l = 1;
1007                         }
1008                         break;
1009         }
1010
1011         return l;
1012 }
1013
1014 static int
1015 pg_euckr_verifier(const unsigned char *s, int len)
1016 {
1017         int                     l;
1018         unsigned char c1,
1019                                 c2;
1020
1021         c1 = *s++;
1022
1023         if (IS_HIGHBIT_SET(c1))
1024         {
1025                 l = 2;
1026                 if (l > len)
1027                         return -1;
1028                 if (!IS_EUC_RANGE_VALID(c1))
1029                         return -1;
1030                 c2 = *s++;
1031                 if (!IS_EUC_RANGE_VALID(c2))
1032                         return -1;
1033         }
1034         else
1035                 /* must be ASCII */
1036         {
1037                 l = 1;
1038         }
1039
1040         return l;
1041 }
1042
1043 /* EUC-CN byte sequences are exactly same as EUC-KR */
1044 #define pg_euccn_verifier       pg_euckr_verifier
1045
1046 static int
1047 pg_euctw_verifier(const unsigned char *s, int len)
1048 {
1049         int                     l;
1050         unsigned char c1,
1051                                 c2;
1052
1053         c1 = *s++;
1054
1055         switch (c1)
1056         {
1057                 case SS2:                               /* CNS 11643 Plane 1-7 */
1058                         l = 4;
1059                         if (l > len)
1060                                 return -1;
1061                         c2 = *s++;
1062                         if (c2 < 0xa1 || c2 > 0xa7)
1063                                 return -1;
1064                         c2 = *s++;
1065                         if (!IS_EUC_RANGE_VALID(c2))
1066                                 return -1;
1067                         c2 = *s++;
1068                         if (!IS_EUC_RANGE_VALID(c2))
1069                                 return -1;
1070                         break;
1071
1072                 case SS3:                               /* unused */
1073                         return -1;
1074
1075                 default:
1076                         if (IS_HIGHBIT_SET(c1))         /* CNS 11643 Plane 1 */
1077                         {
1078                                 l = 2;
1079                                 if (l > len)
1080                                         return -1;
1081                                 /* no further range check on c1? */
1082                                 c2 = *s++;
1083                                 if (!IS_EUC_RANGE_VALID(c2))
1084                                         return -1;
1085                         }
1086                         else
1087                                 /* must be ASCII */
1088                         {
1089                                 l = 1;
1090                         }
1091                         break;
1092         }
1093         return l;
1094 }
1095
1096 static int
1097 pg_johab_verifier(const unsigned char *s, int len)
1098 {
1099         int                     l,
1100                                 mbl;
1101         unsigned char c;
1102
1103         l = mbl = pg_johab_mblen(s);
1104
1105         if (len < l)
1106                 return -1;
1107
1108         if (!IS_HIGHBIT_SET(*s))
1109                 return mbl;
1110
1111         while (--l > 0)
1112         {
1113                 c = *++s;
1114                 if (!IS_EUC_RANGE_VALID(c))
1115                         return -1;
1116         }
1117         return mbl;
1118 }
1119
1120 static int
1121 pg_mule_verifier(const unsigned char *s, int len)
1122 {
1123         int                     l,
1124                                 mbl;
1125         unsigned char c;
1126
1127         l = mbl = pg_mule_mblen(s);
1128
1129         if (len < l)
1130                 return -1;
1131
1132         while (--l > 0)
1133         {
1134                 c = *++s;
1135                 if (!IS_HIGHBIT_SET(c))
1136                         return -1;
1137         }
1138         return mbl;
1139 }
1140
1141 static int
1142 pg_latin1_verifier(const unsigned char *s, int len)
1143 {
1144         return 1;
1145 }
1146
1147 static int
1148 pg_sjis_verifier(const unsigned char *s, int len)
1149 {
1150         int                     l,
1151                                 mbl;
1152         unsigned char c1,
1153                                 c2;
1154
1155         l = mbl = pg_sjis_mblen(s);
1156
1157         if (len < l)
1158                 return -1;
1159
1160         if (l == 1)                                     /* pg_sjis_mblen already verified it */
1161                 return mbl;
1162
1163         c1 = *s++;
1164         c2 = *s;
1165         if (!ISSJISHEAD(c1) || !ISSJISTAIL(c2))
1166                 return -1;
1167         return mbl;
1168 }
1169
1170 static int
1171 pg_big5_verifier(const unsigned char *s, int len)
1172 {
1173         int                     l,
1174                                 mbl;
1175
1176         l = mbl = pg_big5_mblen(s);
1177
1178         if (len < l)
1179                 return -1;
1180
1181         while (--l > 0)
1182         {
1183                 if (*++s == '\0')
1184                         return -1;
1185         }
1186
1187         return mbl;
1188 }
1189
1190 static int
1191 pg_gbk_verifier(const unsigned char *s, int len)
1192 {
1193         int                     l,
1194                                 mbl;
1195
1196         l = mbl = pg_gbk_mblen(s);
1197
1198         if (len < l)
1199                 return -1;
1200
1201         while (--l > 0)
1202         {
1203                 if (*++s == '\0')
1204                         return -1;
1205         }
1206
1207         return mbl;
1208 }
1209
1210 static int
1211 pg_uhc_verifier(const unsigned char *s, int len)
1212 {
1213         int                     l,
1214                                 mbl;
1215
1216         l = mbl = pg_uhc_mblen(s);
1217
1218         if (len < l)
1219                 return -1;
1220
1221         while (--l > 0)
1222         {
1223                 if (*++s == '\0')
1224                         return -1;
1225         }
1226
1227         return mbl;
1228 }
1229
1230 static int
1231 pg_gb18030_verifier(const unsigned char *s, int len)
1232 {
1233         int                     l,
1234                                 mbl;
1235
1236         l = mbl = pg_gb18030_mblen(s);
1237
1238         if (len < l)
1239                 return -1;
1240
1241         while (--l > 0)
1242         {
1243                 if (*++s == '\0')
1244                         return -1;
1245         }
1246
1247         return mbl;
1248 }
1249
1250 static int
1251 pg_utf8_verifier(const unsigned char *s, int len)
1252 {
1253         int                     l = pg_utf_mblen(s);
1254
1255         if (len < l)
1256                 return -1;
1257
1258         if (!pg_utf8_islegal(s, l))
1259                 return -1;
1260
1261         return l;
1262 }
1263
1264 /*
1265  * Check for validity of a single UTF-8 encoded character
1266  *
1267  * This directly implements the rules in RFC3629.  The bizarre-looking
1268  * restrictions on the second byte are meant to ensure that there isn't
1269  * more than one encoding of a given Unicode character point; that is,
1270  * you may not use a longer-than-necessary byte sequence with high order
1271  * zero bits to represent a character that would fit in fewer bytes.
1272  * To do otherwise is to create security hazards (eg, create an apparent
1273  * non-ASCII character that decodes to plain ASCII).
1274  *
1275  * length is assumed to have been obtained by pg_utf_mblen(), and the
1276  * caller must have checked that that many bytes are present in the buffer.
1277  */
1278 bool
1279 pg_utf8_islegal(const unsigned char *source, int length)
1280 {
1281         unsigned char a;
1282
1283         switch (length)
1284         {
1285                 default:
1286                         /* reject lengths 5 and 6 for now */
1287                         return false;
1288                 case 4:
1289                         a = source[3];
1290                         if (a < 0x80 || a > 0xBF)
1291                                 return false;
1292                         /* FALL THRU */
1293                 case 3:
1294                         a = source[2];
1295                         if (a < 0x80 || a > 0xBF)
1296                                 return false;
1297                         /* FALL THRU */
1298                 case 2:
1299                         a = source[1];
1300                         switch (*source)
1301                         {
1302                                 case 0xE0:
1303                                         if (a < 0xA0 || a > 0xBF)
1304                                                 return false;
1305                                         break;
1306                                 case 0xED:
1307                                         if (a < 0x80 || a > 0x9F)
1308                                                 return false;
1309                                         break;
1310                                 case 0xF0:
1311                                         if (a < 0x90 || a > 0xBF)
1312                                                 return false;
1313                                         break;
1314                                 case 0xF4:
1315                                         if (a < 0x80 || a > 0x8F)
1316                                                 return false;
1317                                         break;
1318                                 default:
1319                                         if (a < 0x80 || a > 0xBF)
1320                                                 return false;
1321                                         break;
1322                         }
1323                         /* FALL THRU */
1324                 case 1:
1325                         a = *source;
1326                         if (a >= 0x80 && a < 0xC2)
1327                                 return false;
1328                         if (a > 0xF4)
1329                                 return false;
1330                         break;
1331         }
1332         return true;
1333 }
1334
1335 /*
1336  *-------------------------------------------------------------------
1337  * encoding info table
1338  * XXX must be sorted by the same order as enum pg_enc (in mb/pg_wchar.h)
1339  *-------------------------------------------------------------------
1340  */
1341 pg_wchar_tbl pg_wchar_table[] = {
1342         {pg_ascii2wchar_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifier, 1},       /* PG_SQL_ASCII */
1343         {pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3},       /* PG_EUC_JP */
1344         {pg_euccn2wchar_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifier, 2},       /* PG_EUC_CN */
1345         {pg_euckr2wchar_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifier, 3},       /* PG_EUC_KR */
1346         {pg_euctw2wchar_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifier, 4},       /* PG_EUC_TW */
1347         {pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3},       /* PG_EUC_JIS_2004 */
1348         {pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifier, 4},      /* PG_UTF8 */
1349         {pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifier, 4},           /* PG_MULE_INTERNAL */
1350         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN1 */
1351         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN2 */
1352         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN3 */
1353         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN4 */
1354         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN5 */
1355         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN6 */
1356         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN7 */
1357         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN8 */
1358         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN9 */
1359         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_LATIN10 */
1360         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1256 */
1361         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1258 */
1362         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN866 */
1363         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN874 */
1364         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_KOI8R */
1365         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1251 */
1366         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1252 */
1367         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-5 */
1368         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-6 */
1369         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-7 */
1370         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* ISO-8859-8 */
1371         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1250 */
1372         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1253 */
1373         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1254 */
1374         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1255 */
1375         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_WIN1257 */
1376         {pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1},           /* PG_KOI8U */
1377         {0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2},        /* PG_SJIS */
1378         {0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifier, 2},        /* PG_BIG5 */
1379         {0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifier, 2},           /* PG_GBK */
1380         {0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifier, 2},           /* PG_UHC */
1381         {0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifier, 4},       /* PG_GB18030 */
1382         {0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifier, 3}, /* PG_JOHAB */
1383         {0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2}         /* PG_SHIFT_JIS_2004 */
1384 };
1385
1386 /* returns the byte length of a word for mule internal code */
1387 int
1388 pg_mic_mblen(const unsigned char *mbstr)
1389 {
1390         return pg_mule_mblen(mbstr);
1391 }
1392
1393 /*
1394  * Returns the byte length of a multibyte character.
1395  */
1396 int
1397 pg_encoding_mblen(int encoding, const char *mbstr)
1398 {
1399         Assert(PG_VALID_ENCODING(encoding));
1400
1401         return ((encoding >= 0 &&
1402                          encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
1403                 ((*pg_wchar_table[encoding].mblen) ((const unsigned char *) mbstr)) :
1404         ((*pg_wchar_table[PG_SQL_ASCII].mblen) ((const unsigned char *) mbstr)));
1405 }
1406
1407 /*
1408  * Returns the display length of a multibyte character.
1409  */
1410 int
1411 pg_encoding_dsplen(int encoding, const char *mbstr)
1412 {
1413         Assert(PG_VALID_ENCODING(encoding));
1414
1415         return ((encoding >= 0 &&
1416                          encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
1417            ((*pg_wchar_table[encoding].dsplen) ((const unsigned char *) mbstr)) :
1418         ((*pg_wchar_table[PG_SQL_ASCII].dsplen) ((const unsigned char *) mbstr)));
1419 }
1420
1421 /*
1422  * Verify the first multibyte character of the given string.
1423  * Return its byte length if good, -1 if bad.  (See comments above for
1424  * full details of the mbverify API.)
1425  */
1426 int
1427 pg_encoding_verifymb(int encoding, const char *mbstr, int len)
1428 {
1429         Assert(PG_VALID_ENCODING(encoding));
1430
1431         return ((encoding >= 0 &&
1432                          encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
1433                         ((*pg_wchar_table[encoding].mbverify) ((const unsigned char *) mbstr, len)) :
1434                         ((*pg_wchar_table[PG_SQL_ASCII].mbverify) ((const unsigned char *) mbstr, len)));
1435 }
1436
1437 /*
1438  * fetch maximum length of a given encoding
1439  */
1440 int
1441 pg_encoding_max_length(int encoding)
1442 {
1443         Assert(PG_VALID_ENCODING(encoding));
1444
1445         return pg_wchar_table[encoding].maxmblen;
1446 }
1447
1448 #ifndef FRONTEND
1449
1450 /*
1451  * fetch maximum length of the encoding for the current database
1452  */
1453 int
1454 pg_database_encoding_max_length(void)
1455 {
1456         return pg_wchar_table[GetDatabaseEncoding()].maxmblen;
1457 }
1458
1459 /*
1460  * Verify mbstr to make sure that it is validly encoded in the current
1461  * database encoding.  Otherwise same as pg_verify_mbstr().
1462  */
1463 bool
1464 pg_verifymbstr(const char *mbstr, int len, bool noError)
1465 {
1466         return
1467                 pg_verify_mbstr_len(GetDatabaseEncoding(), mbstr, len, noError) >= 0;
1468 }
1469
1470 /*
1471  * Verify mbstr to make sure that it is validly encoded in the specified
1472  * encoding.
1473  */
1474 bool
1475 pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
1476 {
1477         return pg_verify_mbstr_len(encoding, mbstr, len, noError) >= 0;
1478 }
1479
1480 /*
1481  * Verify mbstr to make sure that it is validly encoded in the specified
1482  * encoding.
1483  *
1484  * mbstr is not necessarily zero terminated; length of mbstr is
1485  * specified by len.
1486  *
1487  * If OK, return length of string in the encoding.
1488  * If a problem is found, return -1 when noError is
1489  * true; when noError is false, ereport() a descriptive message.
1490  */
1491 int
1492 pg_verify_mbstr_len(int encoding, const char *mbstr, int len, bool noError)
1493 {
1494         mbverifier      mbverify;
1495         int                     mb_len;
1496
1497         Assert(PG_VALID_ENCODING(encoding));
1498
1499         /*
1500          * In single-byte encodings, we need only reject nulls (\0).
1501          */
1502         if (pg_encoding_max_length(encoding) <= 1)
1503         {
1504                 const char *nullpos = memchr(mbstr, 0, len);
1505
1506                 if (nullpos == NULL)
1507                         return len;
1508                 if (noError)
1509                         return -1;
1510                 report_invalid_encoding(encoding, nullpos, 1);
1511         }
1512
1513         /* fetch function pointer just once */
1514         mbverify = pg_wchar_table[encoding].mbverify;
1515
1516         mb_len = 0;
1517
1518         while (len > 0)
1519         {
1520                 int                     l;
1521
1522                 /* fast path for ASCII-subset characters */
1523                 if (!IS_HIGHBIT_SET(*mbstr))
1524                 {
1525                         if (*mbstr != '\0')
1526                         {
1527                                 mb_len++;
1528                                 mbstr++;
1529                                 len--;
1530                                 continue;
1531                         }
1532                         if (noError)
1533                                 return -1;
1534                         report_invalid_encoding(encoding, mbstr, len);
1535                 }
1536
1537                 l = (*mbverify) ((const unsigned char *) mbstr, len);
1538
1539                 if (l < 0)
1540                 {
1541                         if (noError)
1542                                 return -1;
1543                         report_invalid_encoding(encoding, mbstr, len);
1544                 }
1545
1546                 mbstr += l;
1547                 len -= l;
1548                 mb_len++;
1549         }
1550         return mb_len;
1551 }
1552
1553 /*
1554  * check_encoding_conversion_args: check arguments of a conversion function
1555  *
1556  * "expected" arguments can be either an encoding ID or -1 to indicate that
1557  * the caller will check whether it accepts the ID.
1558  *
1559  * Note: the errors here are not really user-facing, so elog instead of
1560  * ereport seems sufficient.  Also, we trust that the "expected" encoding
1561  * arguments are valid encoding IDs, but we don't trust the actuals.
1562  */
1563 void
1564 check_encoding_conversion_args(int src_encoding,
1565                                                            int dest_encoding,
1566                                                            int len,
1567                                                            int expected_src_encoding,
1568                                                            int expected_dest_encoding)
1569 {
1570         if (!PG_VALID_ENCODING(src_encoding))
1571                 elog(ERROR, "invalid source encoding ID: %d", src_encoding);
1572         if (src_encoding != expected_src_encoding && expected_src_encoding >= 0)
1573                 elog(ERROR, "expected source encoding \"%s\", but got \"%s\"",
1574                          pg_enc2name_tbl[expected_src_encoding].name,
1575                          pg_enc2name_tbl[src_encoding].name);
1576         if (!PG_VALID_ENCODING(dest_encoding))
1577                 elog(ERROR, "invalid destination encoding ID: %d", dest_encoding);
1578         if (dest_encoding != expected_dest_encoding && expected_dest_encoding >= 0)
1579                 elog(ERROR, "expected destination encoding \"%s\", but got \"%s\"",
1580                          pg_enc2name_tbl[expected_dest_encoding].name,
1581                          pg_enc2name_tbl[dest_encoding].name);
1582         if (len < 0)
1583                 elog(ERROR, "encoding conversion length must not be negative");
1584 }
1585
1586 /*
1587  * report_invalid_encoding: complain about invalid multibyte character
1588  *
1589  * note: len is remaining length of string, not length of character;
1590  * len must be greater than zero, as we always examine the first byte.
1591  */
1592 void
1593 report_invalid_encoding(int encoding, const char *mbstr, int len)
1594 {
1595         int                     l = pg_encoding_mblen(encoding, mbstr);
1596         char            buf[8 * 2 + 1];
1597         char       *p = buf;
1598         int                     j,
1599                                 jlimit;
1600
1601         jlimit = Min(l, len);
1602         jlimit = Min(jlimit, 8);        /* prevent buffer overrun */
1603
1604         for (j = 0; j < jlimit; j++)
1605                 p += sprintf(p, "%02x", (unsigned char) mbstr[j]);
1606
1607         ereport(ERROR,
1608                         (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
1609                          errmsg("invalid byte sequence for encoding \"%s\": 0x%s",
1610                                         pg_enc2name_tbl[encoding].name,
1611                                         buf)));
1612 }
1613
1614 /*
1615  * report_untranslatable_char: complain about untranslatable character
1616  *
1617  * note: len is remaining length of string, not length of character;
1618  * len must be greater than zero, as we always examine the first byte.
1619  */
1620 void
1621 report_untranslatable_char(int src_encoding, int dest_encoding,
1622                                                    const char *mbstr, int len)
1623 {
1624         int                     l = pg_encoding_mblen(src_encoding, mbstr);
1625         char            buf[8 * 2 + 1];
1626         char       *p = buf;
1627         int                     j,
1628                                 jlimit;
1629
1630         jlimit = Min(l, len);
1631         jlimit = Min(jlimit, 8);        /* prevent buffer overrun */
1632
1633         for (j = 0; j < jlimit; j++)
1634                 p += sprintf(p, "%02x", (unsigned char) mbstr[j]);
1635
1636         ereport(ERROR,
1637                         (errcode(ERRCODE_UNTRANSLATABLE_CHARACTER),
1638           errmsg("character 0x%s of encoding \"%s\" has no equivalent in \"%s\"",
1639                          buf,
1640                          pg_enc2name_tbl[src_encoding].name,
1641                          pg_enc2name_tbl[dest_encoding].name)));
1642 }
1643
1644 #endif