From 2dbbf33f4a95cdcce66365bcdb47c885a8858d3c Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 18 Mar 2009 16:17:58 +0000 Subject: [PATCH] Add seven kanji characters defined in the Windows 950 codepage to our big5/win950 <-> UTF8 conversion tables. Per report by Roger Chang. --- src/backend/utils/mb/Unicode/UCS_to_big5.pl | 177 ++++++++++++++++++++++++++ src/backend/utils/mb/Unicode/UCS_to_most.pl | 3 +- src/backend/utils/mb/Unicode/big5_to_utf8.map | 11 +- src/backend/utils/mb/Unicode/utf8_to_big5.map | 9 +- 4 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 src/backend/utils/mb/Unicode/UCS_to_big5.pl diff --git a/src/backend/utils/mb/Unicode/UCS_to_big5.pl b/src/backend/utils/mb/Unicode/UCS_to_big5.pl new file mode 100644 index 0000000000..58886d67f1 --- /dev/null +++ b/src/backend/utils/mb/Unicode/UCS_to_big5.pl @@ -0,0 +1,177 @@ +#! /usr/bin/perl +# +# Copyright (c) 2001-2009, PostgreSQL Global Development Group +# +# $PostgreSQL: pgsql/src/backend/utils/mb/Unicode/UCS_to_big5.pl,v 1.1 2009/03/18 16:17:58 heikki Exp $ +# +# Generate UTF-8 <--> BIG5 conversion tables from +# map files provided by Unicode organization. +# Unfortunately it is prohibited by the organization +# to distribute the map files. So if you try to use this script, +# you have to obtain the map files from the organization's ftp site. +# ftp://www.unicode.org/Public/MAPPINGS/ +# +# Our "big5" comes from BIG5.TXT, with the addition of the characters +# in the range 0xf9d6-0xf9dc from CP950.TXT. +# +# BIG5.TXT format: +# BIG5 code in hex +# UCS-2 code in hex +# # and Unicode name (not used in this script) +# +# CP950.TXT format: +# CP950 code in hex +# UCS-2 code in hex +# # and Unicode name (not used in this script) + + +require "ucs2utf.pl"; + + +# +# first, generate UTF8 --> BIG5 table +# +$in_file = "BIG5.TXT"; + +open( FILE, $in_file ) || die( "cannot open $in_file" ); + +reset 'array'; + +while( ){ + chop; + if( /^#/ ){ + next; + } + ( $c, $u, $rest ) = split; + $ucs = hex($u); + $code = hex($c); + if( $code >= 0x80 && $ucs >= 0x0080){ + $utf = &ucs2utf($ucs); + if( $array{ $utf } ne "" ){ + printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs; + next; + } + $count++; + $array{ $utf } = $code; + } +} +close( FILE ); + +$in_file = "CP950.TXT"; + +open( FILE, $in_file ) || die( "cannot open $in_file" ); + +while( ){ + chop; + if( /^#/ ){ + next; + } + ( $c, $u, $rest ) = split; + $ucs = hex($u); + $code = hex($c); + + # Pick only the ETEN extended characters in the range 0xf9d6 - 0xf9dc + # from CP950.TXT + if( $code >= 0x80 && $ucs >= 0x0080 && + $code >= 0xf9d6 && $code <= 0xf9dc ){ + $utf = &ucs2utf($ucs); + if( $array{ $utf } ne "" ){ + printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs; + next; + } + $count++; + $array{ $utf } = $code; + } +} +close( FILE ); + +$file = lc("utf8_to_big5.map"); +open( FILE, "> $file" ) || die( "cannot open $file" ); +print FILE "static pg_utf_to_local ULmapBIG5[ $count ] = {\n"; + +for $index ( sort {$a <=> $b} keys( %array ) ){ + $code = $array{ $index }; + $count--; + if( $count == 0 ){ + printf FILE " {0x%04x, 0x%04x}\n", $index, $code; + } else { + printf FILE " {0x%04x, 0x%04x},\n", $index, $code; + } +} + +print FILE "};\n"; +close(FILE); + +# +# then generate BIG5 --> UTF8 table +# +$in_file = "BIG5.TXT"; + +open( FILE, $in_file ) || die( "cannot open $in_file" ); + +reset 'array'; + +while( ){ + chop; + if( /^#/ ){ + next; + } + ( $c, $u, $rest ) = split; + $ucs = hex($u); + $code = hex($c); + if( $code >= 0x80 && $ucs >= 0x0080){ + $utf = &ucs2utf($ucs); + if( $array{ $utf } ne "" ){ + printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs; + next; + } + $count++; + $array{ $code } = $utf; + } +} +close( FILE ); + +$in_file = "CP950.TXT"; + +open( FILE, $in_file ) || die( "cannot open $in_file" ); + +while( ){ + chop; + if( /^#/ ){ + next; + } + ( $c, $u, $rest ) = split; + $ucs = hex($u); + $code = hex($c); + + # Pick only the ETEN extended characters in the range 0xf9d6 - 0xf9dc + # from CP950.TXT + if( $code >= 0x80 && $ucs >= 0x0080 && + $code >= 0xf9d6 && $code <= 0xf9dc ){ + $utf = &ucs2utf($ucs); + if( $array{ $utf } ne "" ){ + printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs; + next; + } + $count++; + $array{ $code } = $utf; + } +} +close( FILE ); + +$file = lc("big5_to_utf8.map"); +open( FILE, "> $file" ) || die( "cannot open $file" ); +print FILE "static pg_local_to_utf LUmapBIG5[ $count ] = {\n"; +for $index ( sort {$a <=> $b} keys( %array ) ){ + $utf = $array{ $index }; + $count--; + if( $count == 0 ){ + printf FILE " {0x%04x, 0x%04x}\n", $index, $utf; + } else { + printf FILE " {0x%04x, 0x%04x},\n", $index, $utf; + } +} + +print FILE "};\n"; +close(FILE); + diff --git a/src/backend/utils/mb/Unicode/UCS_to_most.pl b/src/backend/utils/mb/Unicode/UCS_to_most.pl index 0e245d1071..aadfa38149 100644 --- a/src/backend/utils/mb/Unicode/UCS_to_most.pl +++ b/src/backend/utils/mb/Unicode/UCS_to_most.pl @@ -2,7 +2,7 @@ # # Copyright (c) 2001-2009, PostgreSQL Global Development Group # -# $PostgreSQL: pgsql/src/backend/utils/mb/Unicode/UCS_to_most.pl,v 1.7 2009/02/10 19:29:39 petere Exp $ +# $PostgreSQL: pgsql/src/backend/utils/mb/Unicode/UCS_to_most.pl,v 1.8 2009/03/18 16:17:28 heikki Exp $ # # Generate UTF-8 <--> character code conversion tables from # map files provided by Unicode organization. @@ -47,7 +47,6 @@ require "ucs2utf.pl"; 'GBK' => 'CP936.TXT', 'UHC' => 'CP949.TXT', 'JOHAB' => 'JOHAB.TXT', - 'BIG5' => 'BIG5.TXT', ); @charsets = keys(filename); diff --git a/src/backend/utils/mb/Unicode/big5_to_utf8.map b/src/backend/utils/mb/Unicode/big5_to_utf8.map index 3c98e39778..cf180b61cd 100644 --- a/src/backend/utils/mb/Unicode/big5_to_utf8.map +++ b/src/backend/utils/mb/Unicode/big5_to_utf8.map @@ -1,4 +1,4 @@ -static pg_local_to_utf LUmapBIG5[ 13710 ] = { +static pg_local_to_utf LUmapBIG5[ 13717 ] = { {0xa140, 0xe38080}, {0xa141, 0xefbc8c}, {0xa142, 0xe38081}, @@ -13708,5 +13708,12 @@ static pg_local_to_utf LUmapBIG5[ 13710 ] = { {0xf9d2, 0xe9baa4}, {0xf9d3, 0xe9bdbe}, {0xf9d4, 0xe9bd89}, - {0xf9d5, 0xe9be98} + {0xf9d5, 0xe9be98}, + {0xf9d6, 0xe7a281}, + {0xf9d7, 0xe98ab9}, + {0xf9d8, 0xe8a38f}, + {0xf9d9, 0xe5a2bb}, + {0xf9da, 0xe68192}, + {0xf9db, 0xe7b2a7}, + {0xf9dc, 0xe5abba} }; diff --git a/src/backend/utils/mb/Unicode/utf8_to_big5.map b/src/backend/utils/mb/Unicode/utf8_to_big5.map index 0eaeee77ff..85b3c24f1d 100644 --- a/src/backend/utils/mb/Unicode/utf8_to_big5.map +++ b/src/backend/utils/mb/Unicode/utf8_to_big5.map @@ -1,4 +1,4 @@ -static pg_utf_to_local ULmapBIG5[ 13704 ] = { +static pg_utf_to_local ULmapBIG5[ 13711 ] = { {0xc2a2, 0xa246}, {0xc2a3, 0xa247}, {0xc2a5, 0xa244}, @@ -2140,6 +2140,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = { {0xe5a2b1, 0xe54c}, {0xe5a2b3, 0xbc58}, {0xe5a2ba, 0xe94d}, + {0xe5a2bb, 0xf9d9}, {0xe5a2bc, 0xe94f}, {0xe5a2bd, 0xe94a}, {0xe5a2be, 0xbec1}, @@ -2508,6 +2509,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = { {0xe5abb7, 0xe557}, {0xe5abb8, 0xe55a}, {0xe5abb9, 0xe55c}, + {0xe5abba, 0xf9dc}, {0xe5abbb, 0xbc5f}, {0xe5abbd, 0xe556}, {0xe5abbf, 0xe554}, @@ -3370,6 +3372,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = { {0xe6818c, 0xcec9}, {0xe6818d, 0xabe9}, {0xe68190, 0xaea3}, + {0xe68192, 0xf9da}, {0xe68193, 0xcec5}, {0xe68194, 0xcec1}, {0xe68195, 0xaea4}, @@ -7482,6 +7485,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = { {0xe7a1be, 0xe2f0}, {0xe7a1bf, 0xb851}, {0xe7a280, 0xdef0}, + {0xe7a281, 0xf9d6}, {0xe7a283, 0xdeed}, {0xe7a284, 0xdee8}, {0xe7a285, 0xdeea}, @@ -8142,6 +8146,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = { {0xe7b2a2, 0xdae7}, {0xe7b2a3, 0xd6e1}, {0xe7b2a5, 0xb5b0}, + {0xe7b2a7, 0xf9db}, {0xe7b2a8, 0xdae9}, {0xe7b2af, 0xdf56}, {0xe7b2b1, 0xb864}, @@ -10208,6 +10213,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = { {0xe8a38c, 0xdff8}, {0xe8a38d, 0xdff3}, {0xe8a38e, 0xdff4}, + {0xe8a38f, 0xf9d8}, {0xe8a390, 0xdff9}, {0xe8a392, 0xb8cf}, {0xe8a394, 0xb8c7}, @@ -11806,6 +11812,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = { {0xe98ab5, 0xe8a1}, {0xe98ab6, 0xe867}, {0xe98ab7, 0xbe50}, + {0xe98ab9, 0xf9d7}, {0xe98abb, 0xbe4f}, {0xe98abc, 0xbe56}, {0xe98b80, 0xe865}, -- 2.11.0