** E-Mail: furukawa@tcp-ip.or.jp
** \e$B$^$G8fO"Mm$r$*4j$$$7$^$9!#\e(B
***********************************************************************/
-/* $Id: nkf.c,v 1.105 2006/08/23 17:30:32 naruse Exp $ */
+/* $Id: nkf.c,v 1.106 2006/09/14 19:30:03 naruse Exp $ */
#define NKF_VERSION "2.0.8"
-#define NKF_RELEASE_DATE "2006-08-24"
+#define NKF_RELEASE_DATE "2006-09-15"
#include "config.h"
#include "utf8tbl.h"
#define UTF8 12
#define UTF8_INPUT 13
-#define UTF16BE_INPUT 14
-#define UTF16LE_INPUT 15
+#define UTF16_INPUT 1015
+#define UTF32_INPUT 1017
+
+/* byte order */
+
+#define ENDIAN_BIG 1234
+#define ENDIAN_LITTLE 4321
+#define ENDIAN_2143 2143
+#define ENDIAN_3412 3412
#define WISH_TRUE 15
/* no NEC special, NEC-selected IBM extended and IBM extended characters */
static int no_cp932ext_f = FALSE;
/* ignore ZERO WIDTH NO-BREAK SPACE */
-static int ignore_zwnbsp_f = TRUE;
static int no_best_fit_chars_f = FALSE;
+static int input_endian = ENDIAN_BIG;
static nkf_char unicode_subchar = '?'; /* the regular substitution character */
static void nkf_each_char_to_hex(void (*f)(nkf_char c2,nkf_char c1), nkf_char c);
static void encode_fallback_html(nkf_char c);
static nkf_char w2e_conv(nkf_char c2,nkf_char c1,nkf_char c0,nkf_char *p2,nkf_char *p1);
static nkf_char w_iconv(nkf_char c2,nkf_char c1,nkf_char c0);
static nkf_char w_iconv16(nkf_char c2,nkf_char c1,nkf_char c0);
+static nkf_char w_iconv32(nkf_char c2,nkf_char c1,nkf_char c0);
static nkf_char unicode_to_jis_common(nkf_char c2,nkf_char c1,nkf_char c0,nkf_char *p2,nkf_char *p1);
static nkf_char w_iconv_common(nkf_char c1,nkf_char c0,const unsigned short *const *pp,nkf_char psize,nkf_char *p2,nkf_char *p1);
-static void w16w_conv(nkf_char val, nkf_char *p2, nkf_char *p1, nkf_char *p0);
+static void w16w_conv(nkf_char val, nkf_char *p2, nkf_char *p1, nkf_char *p0);\r
static nkf_char ww16_conv(nkf_char c2, nkf_char c1, nkf_char c0);
static nkf_char w16e_conv(nkf_char val,nkf_char *p2,nkf_char *p1);
+static void w_status(struct input_code *, nkf_char);
#endif
#ifdef UTF8_OUTPUT_ENABLE
-static int unicode_bom_f= 0; /* Output Unicode BOM */
-static int w_oconv16_LE = 0; /* utf-16 little endian */
+static int output_bom_f = FALSE;
+static int output_endian = ENDIAN_BIG;
static nkf_char e2w_conv(nkf_char c2,nkf_char c1);
static void w_oconv(nkf_char c2,nkf_char c1);
static void w_oconv16(nkf_char c2,nkf_char c1);
+static void w_oconv32(nkf_char c2,nkf_char c1);
#endif
static void e_oconv(nkf_char c2,nkf_char c1);
static nkf_char e2s_conv(nkf_char c2, nkf_char c1, nkf_char *p2, nkf_char *p1);
static unsigned char stdobuf[IOBUF_SIZE];
#endif
static unsigned char hold_buf[HOLD_SIZE*2];
-static int hold_count;
+static int hold_count = 0;
/* MIME preprocessor fifo */
#endif
#define PREFIX_EUCG3 NKF_INT32_C(0x8F00)
#define CLASS_MASK NKF_INT32_C(0xFF000000)
-#define CLASS_UTF16 NKF_INT32_C(0x01000000)
+#define CLASS_UNICODE NKF_INT32_C(0x01000000)
#define VALUE_MASK NKF_INT32_C(0x00FFFFFF)
#define UNICODE_MAX NKF_INT32_C(0x0010FFFF)
-#define is_unicode_capsule(c) ((c & CLASS_MASK) == CLASS_UTF16)
+#define is_unicode_capsule(c) ((c & CLASS_MASK) == CLASS_UNICODE)
#define is_unicode_bmp(c) ((c & VALUE_MASK) <= NKF_INT32_C(0xFFFF))
#ifdef NUMCHAR_OPTION
static void e_status(struct input_code *, nkf_char);
static void s_status(struct input_code *, nkf_char);
-#ifdef UTF8_INPUT_ENABLE
-static void w_status(struct input_code *, nkf_char);
-static void w16_status(struct input_code *, nkf_char);
-static int utf16_mode = UTF16BE_INPUT;
-#endif
-
struct input_code input_code_list[] = {
{"EUC-JP", 0, 0, 0, {0, 0, 0}, e_status, e_iconv, 0},
{"Shift_JIS", 0, 0, 0, {0, 0, 0}, s_status, s_iconv, 0},
#ifdef UTF8_INPUT_ENABLE
{"UTF-8", 0, 0, 0, {0, 0, 0}, w_status, w_iconv, 0},
- {"UTF-16", 0, 0, 0, {0, 0, 0}, w16_status, w_iconv16, 0},
#endif
{0}
};
input_f = UTF8_INPUT;
nfc_f = TRUE;
#endif
- }else if(strcmp(codeset, "UTF-16") == 0){
- input_f = UTF16BE_INPUT;
- utf16_mode = UTF16BE_INPUT;
- }else if(strcmp(codeset, "UTF-16BE") == 0 ||
+ }else if(strcmp(codeset, "UTF-16") == 0 ||
+ strcmp(codeset, "UTF-16BE") == 0 ||
strcmp(codeset, "UTF-16BE-BOM") == 0){
- input_f = UTF16BE_INPUT;
- utf16_mode = UTF16BE_INPUT;
+ input_f = UTF16_INPUT;
+ input_endian = ENDIAN_BIG;
}else if(strcmp(codeset, "UTF-16LE") == 0 ||
strcmp(codeset, "UTF-16LE-BOM") == 0){
- input_f = UTF16LE_INPUT;
- utf16_mode = UTF16LE_INPUT;
+ input_f = UTF16_INPUT;
+ input_endian = ENDIAN_LITTLE;
#endif
}
continue;
output_conv = w_oconv;
}else if(strcmp(codeset, "UTF-8N") == 0){
output_conv = w_oconv;
- unicode_bom_f=1;
}else if(strcmp(codeset, "UTF-8-BOM") == 0){
output_conv = w_oconv;
- unicode_bom_f=2;
+ output_bom_f = TRUE;
}else if(strcmp(codeset, "UTF-16BE") == 0){
- output_conv = w_oconv16;
- unicode_bom_f=1;
+ output_conv = w_oconv16;
}else if(strcmp(codeset, "UTF-16") == 0 ||
strcmp(codeset, "UTF-16BE-BOM") == 0){
- output_conv = w_oconv16;
- unicode_bom_f=2;
+ output_conv = w_oconv16;
+ output_bom_f = TRUE;
}else if(strcmp(codeset, "UTF-16LE") == 0){
- output_conv = w_oconv16;
- w_oconv16_LE = 1;
- unicode_bom_f=1;
+ output_conv = w_oconv16;
+ output_endian = ENDIAN_LITTLE;
}else if(strcmp(codeset, "UTF-16LE-BOM") == 0){
- output_conv = w_oconv16;
- w_oconv16_LE = 1;
- unicode_bom_f=2;
+ output_conv = w_oconv16;
+ output_endian = ENDIAN_LITTLE;
+ output_bom_f = TRUE;
+ }else if(strcmp(codeset, "UTF-32") == 0 ||
+ strcmp(codeset, "UTF-32BE") == 0){
+ output_conv = w_oconv32;
+ }else if(strcmp(codeset, "UTF-32BE-BOM") == 0){
+ output_conv = w_oconv32;
+ output_bom_f = TRUE;
+ }else if(strcmp(codeset, "UTF-32LE") == 0){
+ output_conv = w_oconv32;
+ output_endian = ENDIAN_LITTLE;
+ }else if(strcmp(codeset, "UTF-32LE-BOM") == 0){
+ output_conv = w_oconv32;
+ output_endian = ENDIAN_LITTLE;
+ output_bom_f = TRUE;
#endif
}
continue;
#endif
#ifdef UTF8_OUTPUT_ENABLE
case 'w': /* UTF-8 output */
- if ('1'== cp[0] && '6'==cp[1]) {
- output_conv = w_oconv16; cp+=2;
+ if (cp[0] == '8') {
+ output_conv = w_oconv; cp++;
+ if (cp[0] == '0'){
+ cp++;
+ } else {
+ output_bom_f = TRUE;
+ }
+ } else {
+ if ('1'== cp[0] && '6'==cp[1]) {
+ output_conv = w_oconv16; cp+=2;
+ } else if ('3'== cp[0] && '2'==cp[1]) {
+ output_conv = w_oconv32; cp+=2;
+ } else {
+ output_conv = w_oconv;
+ continue;
+ }
if (cp[0]=='L') {
- unicode_bom_f=2; cp++;
- w_oconv16_LE = 1;
- if (cp[0] == '0'){
- unicode_bom_f=1; cp++;
- }
+ cp++;
+ output_endian = ENDIAN_LITTLE;
} else if (cp[0] == 'B') {
- unicode_bom_f=2; cp++;
- if (cp[0] == '0'){
- unicode_bom_f=1; cp++;
- }
- }
- } else if (cp[0] == '8') {
- output_conv = w_oconv; cp++;
- unicode_bom_f=2;
+ cp++;
+ } else {
+ continue;
+ }
if (cp[0] == '0'){
- unicode_bom_f=1; cp++;
+ cp++;
+ } else {
+ output_bom_f = TRUE;
}
- } else
- output_conv = w_oconv;
+ }
continue;
#endif
#ifdef UTF8_INPUT_ENABLE
- case 'W': /* UTF-8 input */
- if ('1'== cp[0] && '6'==cp[1]) {
- input_f = UTF16BE_INPUT;
- utf16_mode = UTF16BE_INPUT;
- cp += 2;
+ case 'W': /* UTF input */
+ if (cp[0] == '8') {
+ cp++;
+ input_f = UTF8_INPUT;
+ }else{
+ if ('1'== cp[0] && '6'==cp[1]) {
+ cp += 2;
+ input_f = UTF16_INPUT;
+ input_endian = ENDIAN_BIG;
+ } else if ('3'== cp[0] && '2'==cp[1]) {
+ cp += 2;
+ input_f = UTF32_INPUT;
+ input_endian = ENDIAN_BIG;
+ } else {
+ input_f = UTF8_INPUT;
+ continue;
+ }
if (cp[0]=='L') {
cp++;
- input_f = UTF16LE_INPUT;
- utf16_mode = UTF16LE_INPUT;
+ input_endian = ENDIAN_LITTLE;
} else if (cp[0] == 'B') {
cp++;
- input_f = UTF16BE_INPUT;
- utf16_mode = UTF16BE_INPUT;
}
- } else if (cp[0] == '8') {
- cp++;
- input_f = UTF8_INPUT;
- } else
- input_f = UTF8_INPUT;
+ }
continue;
#endif
/* Input code assumption */
}
#ifdef UTF8_INPUT_ENABLE
-void w16_status(struct input_code *ptr, nkf_char c)
-{
- switch (ptr->stat){
- case -1:
- break;
- case 0:
- if (ptr->_file_stat == 0){
- if (c == 0xfe || c == 0xff){
- ptr->stat = c;
- status_push_ch(ptr, c);
- ptr->_file_stat = 1;
- }else{
- status_disable(ptr);
- ptr->_file_stat = -1;
- }
- }else if (ptr->_file_stat > 0){
- ptr->stat = 1;
- status_push_ch(ptr, c);
- }else if (ptr->_file_stat < 0){
- status_disable(ptr);
- }
- break;
-
- case 1:
- if (c == EOF){
- status_disable(ptr);
- ptr->_file_stat = -1;
- }else{
- status_push_ch(ptr, c);
- status_clear(ptr);
- }
- break;
-
- case 0xfe:
- case 0xff:
- if (ptr->stat != c && (c == 0xfe || c == 0xff)){
- status_push_ch(ptr, c);
- status_clear(ptr);
- }else{
- status_disable(ptr);
- ptr->_file_stat = -1;
- }
- break;
- }
-}
-
void w_status(struct input_code *ptr, nkf_char c)
{
switch (ptr->stat){
}else if (0xe0 <= c && c <= 0xef){
ptr->stat = 2;
status_push_ch(ptr, c);
+ }else if (0xf0 <= c && c <= 0xf4){
+ ptr->stat = 3;
+ status_push_ch(ptr, c);
}else{
status_disable(ptr);
}
status_disable(ptr);
}
break;
+ case 3:
+ if (0x80 <= c && c <= 0xbf){
+ if (ptr->index < ptr->stat){
+ status_push_ch(ptr, c);
+ } else {
+ status_clear(ptr);
+ }
+ }else{
+ status_disable(ptr);
+ }
+ break;
}
}
#endif
#ifdef UTF8_INPUT_ENABLE
} else if (input_f == UTF8_INPUT) {
set_iconv(-TRUE, w_iconv);
- } else if (input_f == UTF16BE_INPUT) {
- set_iconv(-TRUE, w_iconv16);
- } else if (input_f == UTF16LE_INPUT) {
+ } else if (input_f == UTF16_INPUT) {
set_iconv(-TRUE, w_iconv16);
+ } else if (input_f == UTF32_INPUT) {
+ set_iconv(-TRUE, w_iconv32);
#endif
} else {
set_iconv(FALSE, e_iconv);
}
/*
+ * Check and Ignore BOM
+ */
+void check_bom(FILE *f)\r
+{\r
+ int c2;\r
+ switch(c2 = (*i_getc)(f)){
+ case 0x00:
+ if((c2 = (*i_getc)(f)) == 0x00){
+ if((c2 = (*i_getc)(f)) == 0xFE){
+ if((c2 = (*i_getc)(f)) == 0xFF){
+ if(!input_f){
+ set_iconv(TRUE, w_iconv32);
+ }
+ input_endian = ENDIAN_BIG;
+ return;
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0xFE,f);
+ }else if(c2 == 0xFF){
+ if((c2 = (*i_getc)(f)) == 0xFE){
+ if(!input_f){
+ set_iconv(TRUE, w_iconv32);
+ }
+ input_endian = ENDIAN_2143;
+ return;
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0xFF,f);
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0x00,f);
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0x00,f);
+ break;
+ case 0xEF:
+ if((c2 = (*i_getc)(f)) == 0xBB){
+ if((c2 = (*i_getc)(f)) == 0xBF){
+ if(!input_f){
+ set_iconv(TRUE, w_iconv);
+ }
+ return;
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0xBB,f);
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0xEF,f);
+ break;
+ case 0xFE:
+ if((c2 = (*i_getc)(f)) == 0xFF){
+ if((c2 = (*i_getc)(f)) == 0x00){
+ if((c2 = (*i_getc)(f)) == 0x00){
+ if(!input_f){
+ set_iconv(TRUE, w_iconv32);
+ }
+ input_endian = ENDIAN_3412;
+ return;
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0x00,f);
+ }else (*i_ungetc)(c2,f);
+ if(!input_f){
+ set_iconv(TRUE, w_iconv16);
+ }
+ input_endian = ENDIAN_BIG;
+ return;
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0xFE,f);
+ break;
+ case 0xFF:
+ if((c2 = (*i_getc)(f)) == 0xFE){
+ if((c2 = (*i_getc)(f)) == 0x00){
+ if((c2 = (*i_getc)(f)) == 0x00){
+ if(!input_f){
+ set_iconv(TRUE, w_iconv32);
+ }
+ input_endian = ENDIAN_LITTLE;
+ return;
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0x00,f);
+ }else (*i_ungetc)(c2,f);
+ if(!input_f){
+ set_iconv(TRUE, w_iconv16);
+ }
+ input_endian = ENDIAN_LITTLE;
+ return;
+ }else (*i_ungetc)(c2,f);
+ (*i_ungetc)(0xFF,f);
+ break;
+ default:
+ (*i_ungetc)(c2,f);
+ break;
+ }
+}
+\r
+/*
Conversion main loop. Code detection only.
*/
nkf_char kanji_convert(FILE *f)
{
- nkf_char c1,
- c2, c3;
+ nkf_char c3, c2=0, c1, c0=0;
int is_8bit = FALSE;
- module_connection();
- c2 = 0;
-
if(input_f == SJIS_INPUT || input_f == EUC_INPUT
#ifdef UTF8_INPUT_ENABLE
- || input_f == UTF8_INPUT || input_f == UTF16BE_INPUT || input_f == UTF16LE_INPUT
+ || input_f == UTF8_INPUT || input_f == UTF16_INPUT
#endif
){
is_8bit = TRUE;
}
-
input_mode = ASCII;
output_mode = ASCII;
shift_mode = FALSE;
#define SEND ; /* output c1 and c2, get next */
#define LAST break /* end of loop, go closing */
+ module_connection();
+ check_bom(f);
+
while ((c1 = (*i_getc)(f)) != EOF) {
#ifdef INPUT_CODE_FIX
if (!input_f)
SEND;
} else {
/* first byte */
- if (
#ifdef UTF8_INPUT_ENABLE
- iconv == w_iconv16
-#else
- 0
-#endif
- ) {
- c2 = c1;
- c1 = (*i_getc)(f);
+ if (iconv == w_iconv16) {
+ if (input_endian == ENDIAN_BIG) {
+ c2 = c1;
+ if ((c1 = (*i_getc)(f)) != EOF) {
+ if (0xD8 <= c2 && c2 <= 0xDB) {
+ if ((c0 = (*i_getc)(f)) != EOF) {
+ c0 <<= 8;
+ if ((c3 = (*i_getc)(f)) != EOF) {
+ c0 |= c3;
+ } else c1 = EOF;
+ } else c1 = EOF;
+ }
+ }
+ } else {
+ if ((c2 = (*i_getc)(f)) != EOF) {
+ if (0xD8 <= c2 && c2 <= 0xDB) {
+ if ((c3 = (*i_getc)(f)) != EOF) {
+ c3 <<= 8;
+ if ((c0 = (*i_getc)(f)) != EOF) {
+ c0 |= c3;
+ } else c1 = EOF;
+ } else c1 = EOF;
+ }
+ } else c1 = EOF;
+ }
+ SEND;
+ } else if(iconv == w_iconv32){
+ int c3 = c1;
+ if((c2 = (*i_getc)(f)) != EOF &&
+ (c1 = (*i_getc)(f)) != EOF &&
+ (c0 = (*i_getc)(f)) != EOF){
+ switch(input_endian){
+ case ENDIAN_BIG:
+ c1 = (c2&0xFF)<<16 | (c1&0xFF)<<8 | (c0&0xFF);
+ break;
+ case ENDIAN_LITTLE:
+ c1 = (c3&0xFF) | (c2&0xFF)<<8 | (c1&0xFF)<<16;
+ break;
+ case ENDIAN_2143:
+ c1 = (c3&0xFF)<<16 | (c1&0xFF) | (c0&0xFF)<<8;
+ break;
+ case ENDIAN_3412:
+ c1 = (c3&0xFF)<<8 | (c2&0xFF) | (c0&0xFF)<<16;
+ break;
+ }
+ c2 = 0;
+ }else{
+ c1 = EOF;
+ }
SEND;
+ } else
+#endif
#ifdef NUMCHAR_OPTION
- } else if (is_unicode_capsule(c1)){
+ if (is_unicode_capsule(c1)){
SEND;
+ } else
#endif
- } else if (c1 > DEL) {
+ if (c1 > DEL) {
/* 8 bit code */
if (!estab_f && !iso8859_f) {
/* not established yet */
/* send: */
switch(input_mode){
case ASCII:
- if ((*iconv)(c2, c1, 0) < 0){ /* can be EUC/SJIS */
- nkf_char c0 = (*i_getc)(f);
- if (c0 != EOF){
+ switch ((*iconv)(c2, c1, c0)) { /* can be EUC / SJIS / UTF-8 / UTF-16 */
+ case -2:
+ /* 4 bytes UTF-8 */
+ if ((c0 = (*i_getc)(f)) != EOF) {
+ code_status(c0);
+ c0 <<= 8;
+ if ((c3 = (*i_getc)(f)) != EOF) {
+ code_status(c3);
+ (*iconv)(c2, c1, c0|c3);
+ }
+ }
+ break;
+ case -1:
+ /* 3 bytes EUC or UTF-8 */
+ if ((c0 = (*i_getc)(f)) != EOF) {
code_status(c0);
(*iconv)(c2, c1, c0);
}
+ break;
}
break;
case X0208:
}
c2 = 0;
+ c0 = 0;
continue;
/* goto next_word */
}
nkf_char
h_conv(FILE *f, nkf_char c2, nkf_char c1)
{
- nkf_char wc,c3;
+ nkf_char ret, c3, c0;
+ int hold_index;
/** it must NOT be in the kanji shifte sequence */
** Kanji codes by oconv and leave estab_f unchanged.
**/
- c3=c1;
- wc = 0;
- while (wc < hold_count){
- c2 = hold_buf[wc++];
+ ret = c1;
+ hold_index = 0;
+ while (hold_index < hold_count){
+ c2 = hold_buf[hold_index++];
if (c2 <= DEL
#ifdef NUMCHAR_OPTION
|| is_unicode_capsule(c2)
(*iconv)(X0201, c2, 0);
continue;
}
- if (wc < hold_count){
- c1 = hold_buf[wc++];
+ if (hold_index < hold_count){
+ c1 = hold_buf[hold_index++];
}else{
c1 = (*i_getc)(f);
if (c1 == EOF){
}
code_status(c1);
}
- if ((*iconv)(c2, c1, 0) < 0){
- nkf_char c0;
- if (wc < hold_count){
- c0 = hold_buf[wc++];
- }else{
- c0 = (*i_getc)(f);
- if (c0 == EOF){
- c3 = EOF;
- break;
- }
+ c0 = 0;
+ switch ((*iconv)(c2, c1, 0)) { /* can be EUC/SJIS/UTF-8 */
+ case -2:
+ /* 4 bytes UTF-8 */
+ if (hold_index < hold_count){
+ c0 = hold_buf[hold_index++];
+ } else if ((c0 = (*i_getc)(f)) == EOF) {
+ ret = EOF;
+ break;
+ } else {
+ code_status(c0);
+ c0 <<= 8;
+ if (hold_index < hold_count){
+ c3 = hold_buf[hold_index++];
+ } else if ((c3 = (*i_getc)(f)) == EOF) {
+ c0 = ret = EOF;
+ break;
+ } else {
+ code_status(c3);
+ (*iconv)(c2, c1, c0|c3);
+ }
+ }
+ break;
+ case -1:
+ /* 3 bytes EUC or UTF-8 */
+ if (hold_index < hold_count){
+ c0 = hold_buf[hold_index++];
+ } else if ((c0 = (*i_getc)(f)) == EOF) {
+ ret = EOF;
+ break;
+ } else {
code_status(c0);
}
(*iconv)(c2, c1, c0);
- }
+ break;
+ }
+ if (c0 == EOF) break;
}
- return c3;
+ return ret;
}
-
-
-nkf_char
-push_hold_buf(nkf_char c2)
+nkf_char push_hold_buf(nkf_char c2)
{
if (hold_count >= HOLD_SIZE*2)
return (EOF);
#ifdef NUMCHAR_OPTION
if (ret > 0){
if (p2) *p2 = 0;
- if (p1) *p1 = CLASS_UTF16 | ww16_conv(c2, c1, c0);
+ if (p1) *p1 = CLASS_UNICODE | ww16_conv(c2, c1, c0);
ret = 0;
}
#endif
nkf_char w_iconv(nkf_char c2, nkf_char c1, nkf_char c0)
{
nkf_char ret = 0;
+ static const int w_iconv_utf8_1st_byte[] =
+ { /* 0xC0 - 0xFF */
+ 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
+ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
+ 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 33,
+ 40, 41, 41, 41, 42, 43, 43, 43, 50, 50, 50, 50, 60, 60, 70, 70};
- /* throw away ZERO WIDTH NO-BREAK SPACE (U+FEFF) */
- if(ignore_zwnbsp_f){
- ignore_zwnbsp_f = FALSE;
- if(c2 == 0xef && c1 == 0xbb && c0 == 0xbf)
- return 0;
- }
-
- if (c2 == 0) /* 0x00-0x7f */
- c1 &= 0x7F; /* 1byte */
- else if (c0 == 0){
- if ((c2 & 0xe0) == 0xc0){ /* 0xc0-0xdf */
- /* 2ytes */
- if((c2 & 0xFE) == 0xC0 || c1 < 0x80 || 0xBF < c1) return 0;
- }else if ((c2 & 0xf0) == 0xe0) /* 0xe0-0xef */
- return -1; /* 3bytes */
-#ifdef __COMMENT__
- else if (0xf0 <= c2)
- return 0; /* 4,5,6bytes */
- else if ((c2 & 0xc0) == 0x80) /* 0x80-0xbf */
- return 0; /* trail byte */
-#endif
- else return 0;
- }else{
- /* must be 3bytes */
- if(c2 == 0xE0){
- if(c1 < 0xA0 || 0xBF < c1 || c0 < 0x80 || 0xBF < c0)
+ if (c2 < 0 || 0xff < c2) {
+ }else if (c2 == 0) { /* 0 : 1 byte*/
+ c0 = 0;
+ } else if ((c2 & 0xc0) == 0x80) { /* 0x80-0xbf : trail byte */
+ return 0;
+ } else{
+ switch (w_iconv_utf8_1st_byte[c2 - 0xC0]) {
+ case 21:
+ if (c1 < 0x80 || 0xBF < c1) return 0;
+ break;
+ case 30:
+ if (c0 == 0) return -1;
+ if (c1 < 0xA0 || 0xBF < c1 || (c0 & 0xc0) != 0x80)
+ return 0;
+ break;
+ case 31:
+ case 33:
+ if (c0 == 0) return -1;
+ if ((c1 & 0xc0) != 0x80 || (c0 & 0xc0) != 0x80)
+ return 0;
+ break;
+ case 32:
+ if (c0 == 0) return -1;
+ if (c1 < 0x80 || 0x9F < c1 || (c0 & 0xc0) != 0x80)
+ return 0;
+ break;
+ case 40:
+ if (c0 == 0) return -2;
+ if (c1 < 0x90 || 0xBF < c1 || (c0 & 0xc0c0) != 0x8080)
return 0;
- }else if(c2 == 0xED){
- if(c1 < 0x80 || 0x9F < c1 || c0 < 0x80 || 0xBF < c0)
+ break;
+ case 41:
+ if (c0 == 0) return -2;
+ if (c1 < 0x80 || 0xBF < c1 || (c0 & 0xc0c0) != 0x8080)
return 0;
- }else if((c2 & 0xf0) == 0xe0){
- if(c1 < 0x80 || 0xBF < c1 || c0 < 0x80 || 0xBF < c0)
+ break;
+ case 42:
+ if (c0 == 0) return -2;
+ if (c1 < 0x80 || 0x8F < c1 || (c0 & 0xc0c0) != 0x8080)
return 0;
- }else return 0;
+ break;
+ default:
+ return 0;
+ break;
+ }
}
if (c2 == 0 || c2 == EOF){
+ } else if ((c2 & 0xf8) == 0xf0) { /* 4 bytes */
+ c1 = CLASS_UNICODE | ww16_conv(c2, c1, c0);
+ c2 = 0;
} else {
ret = w2e_conv(c2, c1, c0, &c2, &c1);
}
*p2 = 0xe0 | (val >> 12);
*p1 = 0x80 | ((val >> 6) & 0x3f);
*p0 = 0x80 | (val & 0x3f);
+ } else if (val <= NKF_INT32_C(0x10FFFF)) {
+ *p2 = 0xe0 | (val >> 16);
+ *p1 = 0x80 | ((val >> 12) & 0x3f);
+ *p0 = 0x8080 | ((val << 2) & 0x3f00)| (val & 0x3f);
} else {
*p2 = 0;
*p1 = 0;
nkf_char ww16_conv(nkf_char c2, nkf_char c1, nkf_char c0)
{
nkf_char val;
- if (c2 >= 0xf0){
+ if (c2 >= 0xf8) {
val = -1;
+ } else if (c2 >= 0xf0){
+ /* c2: 1st, c1: 2nd, c0: 3rd/4th */
+ val = (c2 & 0x0f) << 18;
+ val |= (c1 & 0x3f) << 12;
+ val |= (c0 & 0x3f00) >> 2;
+ val |= (c0 & 0x3f);
}else if (c2 >= 0xe0){
val = (c2 & 0x0f) << 12;
val |= (c1 & 0x3f) << 6;
#ifdef NUMCHAR_OPTION
if (ret > 0){
*p2 = 0;
- *p1 = CLASS_UTF16 | val;
+ *p1 = CLASS_UNICODE | val;
ret = 0;
}
#endif
#ifdef UTF8_INPUT_ENABLE
nkf_char w_iconv16(nkf_char c2, nkf_char c1, nkf_char c0)
{
- nkf_char ret;
-
- /* throw away ZERO WIDTH NO-BREAK SPACE (U+FEFF) */
- if(ignore_zwnbsp_f){
- ignore_zwnbsp_f = FALSE;
- if (c2==0376 && c1==0377){
- utf16_mode = UTF16BE_INPUT;
- return 0;
- }else if(c2==0377 && c1==0376){
- utf16_mode = UTF16LE_INPUT;
- return 0;
- }
- }
- if (c2 != EOF && utf16_mode == UTF16LE_INPUT) {
- nkf_char tmp;
- tmp=c1; c1=c2; c2=tmp;
- }
+ nkf_char ret = 0;
if ((c2==0 && c1 < 0x80) || c2==EOF) {
(*oconv)(c2, c1);
return 0;
- }else if((c2>>3)==27){ /* surrogate pair */
+ }else if (0xD8 <= c2 && c2 <= 0xDB) {
+ if (c0 < NKF_INT32_C(0xDC00) || NKF_INT32_C(0xDFFF) < c0)
+ return -2;
+ c1 = CLASS_UNICODE | ((c2 << 18) + (c1 << 10) + c0 - NKF_INT32_C(0x35FDC00));
+ c2 = 0;
+ }else if ((c2>>3) == 27) { /* unpaired surrogate */
+ /*
+ return 2;
+ */
return 1;
}else ret = w16e_conv(((c2 & 0xff)<<8) + c1, &c2, &c1);
if (ret) return ret;
return 0;
}
+nkf_char w_iconv32(nkf_char c2, nkf_char c1, nkf_char c0)
+{
+ int ret = 0;
+
+ if ((c2 == 0 && c1 < 0x80) || c2==EOF) {
+ } else if (is_unicode_bmp(c1)) {
+ ret = w16e_conv(c1, &c2, &c1);
+ } else {
+ c2 = 0;
+ c1 = CLASS_UNICODE | c1;
+ }
+ if (ret) return ret;
+ (*oconv)(c2, c1);
+ return 0;
+}
+
nkf_char unicode_to_jis_common(nkf_char c2, nkf_char c1, nkf_char c0, nkf_char *p2, nkf_char *p1)
{
#if 0
{
nkf_char c0;
nkf_char val;
- if (c2 == EOF) {
- (*o_putc)(EOF);
- return;
- }
- if (unicode_bom_f==2) {
- (*o_putc)('\357');
+ if (output_bom_f) {
+ output_bom_f = FALSE;
+ (*o_putc)('\357');
(*o_putc)('\273');
(*o_putc)('\277');
- unicode_bom_f=1;
+ }
+
+ if (c2 == EOF) {
+ (*o_putc)(EOF);
+ return;
}
#ifdef NUMCHAR_OPTION
(*o_putc)(0x80 | ((val >> 6) & 0x3f));
(*o_putc)(0x80 | (val & 0x3f));
} else if (val <= NKF_INT32_C(0x10FFFF)) {
- (*o_putc)(0xE0 | ( val>>18));
+ (*o_putc)(0xF0 | ( val>>18));
(*o_putc)(0x80 | ((val>>12) & 0x3f));
(*o_putc)(0x80 | ((val>> 6) & 0x3f));
(*o_putc)(0x80 | ( val & 0x3f));
void w_oconv16(nkf_char c2, nkf_char c1)
{
- if (c2 == EOF) {
- (*o_putc)(EOF);
- return;
- }
-
- if (unicode_bom_f==2) {
- if (w_oconv16_LE){
+ if (output_bom_f) {
+ output_bom_f = FALSE;
+ if (output_endian == ENDIAN_LITTLE){
(*o_putc)((unsigned char)'\377');
(*o_putc)('\376');
}else{
(*o_putc)('\376');
(*o_putc)((unsigned char)'\377');
}
- unicode_bom_f=1;
+ }
+
+ if (c2 == EOF) {
+ (*o_putc)(EOF);
+ return;
}
if (c2 == ISO8859_1) {
if (c1 <= UNICODE_MAX) {
c2 = (c1 >> 10) + NKF_INT32_C(0xD7C0); /* high surrogate */
c1 = (c1 & 0x3FF) + NKF_INT32_C(0xDC00); /* low surrogate */
- if (w_oconv16_LE){
+ if (output_endian == ENDIAN_LITTLE){
(*o_putc)(c2 & 0xff);
(*o_putc)((c2 >> 8) & 0xff);
(*o_putc)(c1 & 0xff);
c2 = (val >> 8) & 0xff;
c1 = val & 0xff;
}
- if (w_oconv16_LE){
+ if (output_endian == ENDIAN_LITTLE){
(*o_putc)(c1);
(*o_putc)(c2);
}else{
}
}
+void w_oconv32(nkf_char c2, nkf_char c1)
+{
+ if (output_bom_f) {
+ output_bom_f = FALSE;
+ if (output_endian == ENDIAN_LITTLE){
+ (*o_putc)((unsigned char)'\377');
+ (*o_putc)('\376');
+ (*o_putc)('\000');
+ (*o_putc)('\000');
+ }else{
+ (*o_putc)('\000');
+ (*o_putc)('\000');
+ (*o_putc)('\376');
+ (*o_putc)((unsigned char)'\377');
+ }
+ }
+
+ if (c2 == EOF) {
+ (*o_putc)(EOF);
+ return;
+ }
+
+ if (c2 == ISO8859_1) {
+ c1 |= 0x80;
+#ifdef NUMCHAR_OPTION
+ } else if (c2 == 0 && is_unicode_capsule(c1)) {
+ c1 &= VALUE_MASK;
+#endif
+ } else if (c2) {
+ c1 = e2w_conv(c2, c1);
+ }
+ if (output_endian == ENDIAN_LITTLE){
+ (*o_putc)( c1 & NKF_INT32_C(0x000000FF));
+ (*o_putc)((c1 & NKF_INT32_C(0x0000FF00)) >> 8);
+ (*o_putc)((c1 & NKF_INT32_C(0x00FF0000)) >> 16);
+ (*o_putc)('\000');
+ }else{
+ (*o_putc)('\000');
+ (*o_putc)((c1 & NKF_INT32_C(0x00FF0000)) >> 16);
+ (*o_putc)((c1 & NKF_INT32_C(0x0000FF00)) >> 8);
+ (*o_putc)( c1 & NKF_INT32_C(0x000000FF));
+ }
+}
#endif
void e_oconv(nkf_char c2, nkf_char c1)
return;
} else if (c1 == 0x74 && (output_conv == w_oconv || output_conv == w_oconv16)) {
c2 = 0;
- c1 = CLASS_UTF16 | 0x3094;
+ c1 = CLASS_UNICODE | 0x3094;
(*o_hira_conv)(c2,c1);
return;
}
}
}
if (hira_f & 2) {
- if (c2 == 0 && c1 == (CLASS_UTF16 | 0x3094)) {
+ if (c2 == 0 && c1 == (CLASS_UNICODE | 0x3094)) {
c2 = 0x25;
c1 = 0x74;
} else if (c2 == 0x24 && 0x20 < c1 && c1 < 0x74) {
}
}
if (c != -1){
- return CLASS_UTF16 | c;
+ return CLASS_UNICODE | c;
}
while (i > 0){
(*u)(buf[i], f);
#endif
#ifdef UTF8_INPUT_ENABLE
no_cp932ext_f = FALSE;
- ignore_zwnbsp_f = TRUE;
no_best_fit_chars_f = FALSE;
encode_fallback = NULL;
unicode_subchar = '?';
+ input_endian = ENDIAN_BIG;
#endif
#ifdef UTF8_OUTPUT_ENABLE
- unicode_bom_f = 0;
- w_oconv16_LE = 0;
+ output_bom_f = FALSE;
+ output_endian = ENDIAN_BIG;
#endif
#ifdef UNICODE_NORMALIZATION
nfc_f = FALSE;
prefix_table[i] = 0;
}
}
-#ifdef UTF8_INPUT_ENABLE
- utf16_mode = UTF16BE_INPUT;
-#endif
+ hold_count = 0;
mimeout_buf_count = 0;
mimeout_mode = 0;
base64_count = 0;
OSDN Git Service
