** E-Mail: furukawa@tcp-ip.or.jp
** \e$B$^$G8fO"Mm$r$*4j$$$7$^$9!#\e(B
***********************************************************************/
-/* $Id: nkf.c,v 1.80 2005/11/06 20:11:27 naruse Exp $ */
+/* $Id: nkf.c,v 1.90 2006/01/23 18:48:14 naruse Exp $ */
#define NKF_VERSION "2.0.5"
-#define NKF_RELEASE_DATE "2005-11-07"
+#define NKF_RELEASE_DATE "2006-01-24"
#include "config.h"
#define COPY_RIGHT \
- "Copyright (C) 1987, FUJITSU LTD. (I.Ichikawa),2000 S. Kono, COW, 2002-2005 Kono, Furukawa, Naruse"
+ "Copyright (C) 1987, FUJITSU LTD. (I.Ichikawa),2000 S. Kono, COW, 2002-2006 Kono, Furukawa, Naruse"
/*
#define X0201 2
#define ISO8859_1 8
#define NO_X0201 3
-#define X0212 16
+#define X0212 0x2844
+#define X0213_1 0x2850
+#define X0213_2 0x2850
/* Input Assumption */
#define sizeof_utf8_to_euc_C2 64
#define sizeof_utf8_to_euc_E5B8 64
#define sizeof_utf8_to_euc_2bytes 112
-#define sizeof_utf8_to_euc_3bytes 112
+#define sizeof_utf8_to_euc_3bytes 16
#endif
/* MIME preprocessor */
STATIC int s_iconv PROTO((int c2,int c1,int c0));
STATIC int s2e_conv PROTO((int c2, int c1, int *p2, int *p1));
STATIC int e_iconv PROTO((int c2,int c1,int c0));
+#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
+/* Microsoft UCS Mapping Compatible
+ * 0: Shift_JIS, eucJP-ascii
+ * 1: eucJP-ms
+ * 2: CP932, CP51932
+ */
+#define EUCJPMS 1
+#define MS_CODEPAGE 2
+STATIC int ms_ucs_map_f = 0;
+#endif
#ifdef UTF8_INPUT_ENABLE
+/* don't convert characters when the mapping is not defined in the standard */
+STATIC int strict_mapping_f = TRUE;
+/* disable NEC special, NEC-selected IBM extended and IBM extended characters */
+STATIC int disable_cp932ext_f = FALSE;
+/* ignore ZERO WIDTH NO-BREAK SPACE */
+STATIC int ignore_zwnbsp_f = TRUE;
+/* don't convert characters that can't secure round trip convertion */
+STATIC int unicode_round_trip_f = FALSE;
+STATIC int unicode_subchar = '?'; /* the regular substitution character */
STATIC void encode_fallback_html PROTO((int c));
STATIC void encode_fallback_xml PROTO((int c));
STATIC void encode_fallback_java PROTO((int c));
STATIC int w2e_conv PROTO((int c2,int c1,int c0,int *p2,int *p1));
STATIC int w_iconv PROTO((int c2,int c1,int c0));
STATIC int w_iconv16 PROTO((int c2,int c1,int c0));
+STATIC int unicode_to_jis_common PROTO((int c2,int c1,int c0,int *p2,int *p1));
STATIC int w_iconv_common PROTO((int c1,int c0,const unsigned short *const *pp,int psize,int *p2,int *p1));
STATIC int ww16_conv PROTO((int c2, int c1, int c0));
STATIC int w16e_conv PROTO((unsigned short val,int *p2,int *p1));
#endif
+#if defined(UTF8_OUTPUT_ENABLE) && defined(UTF8_INPUT_ENABLE)
+STATIC int internal_unicode_f = FALSE; /* Internal Unicode Processing */
+#endif
#ifdef UTF8_OUTPUT_ENABLE
+STATIC int unicode_bom_f= 0; /* Output Unicode BOM */
+STATIC int w_oconv16_LE = 0; /* utf-16 little endian */
STATIC int e2w_conv PROTO((int c2,int c1));
STATIC void w_oconv PROTO((int c2,int c1));
STATIC void w_oconv16 PROTO((int c2,int c1));
STATIC int x0201_f = NO_X0201; /* Assume NO JISX0201 */
#endif
STATIC int iso2022jp_f = FALSE; /* convert ISO-2022-JP */
-#if defined(UTF8_OUTPUT_ENABLE) && defined(UTF8_INPUT_ENABLE)
-STATIC int internal_unicode_f = FALSE; /* Internal Unicode Processing */
-#endif
-#ifdef UTF8_OUTPUT_ENABLE
-STATIC int unicode_bom_f= 0; /* Output Unicode BOM */
-STATIC int w_oconv16_LE = 0; /* utf-16 little endian */
-STATIC int ms_ucs_map_f = FALSE; /* Microsoft UCS Mapping Compatible */
-STATIC int unicode_subchar = '?'; /* the regular substitution character */
-#endif
#ifdef UNICODE_NORMALIZATION
STATIC int nfc_f = FALSE;
#endif
#ifdef SHIFTJIS_CP932
-/* invert IBM extended characters to others and controls some UCS mapping */
+/* invert IBM extended characters to others
+ and controls some UCS mapping for Microsoft Code Page */
STATIC int cp51932_f = TRUE;
#define CP932_TABLE_BEGIN (0xfa)
#define CP932_TABLE_END (0xfc)
STATIC int x0212_shift PROTO((int c));
STATIC int x0212_unshift PROTO((int c));
#endif
+STATIC int x0213_f = FALSE;
STATIC unsigned char prefix_table[256];
const char *name;
const char *alias;
} long_option[] = {
+ {"ic=", ""},
+ {"oc=", ""},
{"base64","jMB"},
{"euc","e"},
{"euc-input","E"},
#ifdef UTF8_INPUT_ENABLE
{"utf8-input", "W"},
{"utf16-input", "W16"},
+ {"disable-cp932ext", ""},
+ {"strict-mapping", ""},
+ {"enable-round-trip",""},
#endif
#ifdef UNICODE_NORMALIZATION
{"utf8mac-input", ""},
int i, j;
unsigned char *p = NULL;
unsigned char *cp_back = NULL;
+ unsigned char codeset[32];
if (option_mode==1)
return;
cp_back = cp;
cp = (unsigned char *)long_option[i].alias;
}else{
+ if (strcmp(long_option[i].name, "ic=") == 0){
+ for (i=0; i < 16 && SPACE < p[i] && p[i] < DEL; i++){
+ codeset[i] = nkf_toupper(p[i]);
+ }
+ codeset[i] = 0;
+ if(strcmp(codeset, "ISO-2022-JP") == 0){
+ input_f = JIS_INPUT;
+ }else if(strcmp(codeset, "SHIFT_JIS") == 0){
+ input_f = SJIS_INPUT;
+ if (x0201_f==NO_X0201) x0201_f=TRUE;
+ }else if(strcmp(codeset, "CP932") == 0){
+ input_f = SJIS_INPUT;
+ x0201_f = FALSE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = TRUE;
+ cp932inv_f = TRUE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 2;
+#endif
+ }else if(strcmp(codeset, "EUCJP") == 0 ||
+ strcmp(codeset, "EUC-JP") == 0){
+ input_f = JIS_INPUT;
+ }else if(strcmp(codeset, "CP51932") == 0){
+ input_f = JIS_INPUT;
+ x0201_f = FALSE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = TRUE;
+ cp932inv_f = TRUE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 2;
+#endif
+ }else if(strcmp(codeset, "EUC-JP-MS") == 0 ||
+ strcmp(codeset, "EUCJP-MS") == 0){
+ input_f = JIS_INPUT;
+ x0201_f = FALSE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = FALSE;
+ cp932inv_f = TRUE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 1;
+#endif
+ }else if(strcmp(codeset, "EUC-JP-ASCII") == 0 ||
+ strcmp(codeset, "EUCJP-ASCII") == 0){
+ input_f = JIS_INPUT;
+ x0201_f = FALSE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = FALSE;
+ cp932inv_f = TRUE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 0;
+#endif
+ }else if(strcmp(codeset, "SHIFT_JISX0213") == 0){
+ input_f = SJIS_INPUT;
+ x0213_f = TRUE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = FALSE;
+ cp932inv_f = FALSE;
+#endif
+ if (x0201_f==NO_X0201) x0201_f=TRUE;
+ }else if(strcmp(codeset, "EUC-JISX0213") == 0){
+ input_f = JIS_INPUT;
+ x0201_f = FALSE;
+ x0213_f = TRUE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = FALSE;
+ cp932inv_f = FALSE;
+#endif
+#ifdef UTF8_INPUT_ENABLE
+ }else if(strcmp(codeset, "UTF-8") == 0 ||
+ strcmp(codeset, "UTF-8N") == 0 ||
+ strcmp(codeset, "UTF-8-BOM") == 0){
+ input_f = UTF8_INPUT;
+#ifdef UNICODE_NORMALIZATION
+ }else if(strcmp(codeset, "UTF8-MAC") == 0 ||
+ strcmp(codeset, "UTF-8-MAC") == 0){
+ input_f = UTF8_INPUT;
+ nfc_f = TRUE;
+#endif
+ }else if(strcmp(codeset, "UTF-16") == 0){
+ input_f = UTF16BE_INPUT;
+ utf16_mode = UTF16BE_INPUT;
+ }else if(strcmp(codeset, "UTF-16BE") == 0 ||
+ strcmp(codeset, "UTF-16BE-BOM") == 0){
+ input_f = UTF16BE_INPUT;
+ utf16_mode = UTF16BE_INPUT;
+ }else if(strcmp(codeset, "UTF-16LE") == 0 ||
+ strcmp(codeset, "UTF-16LE-BOM") == 0){
+ input_f = UTF16LE_INPUT;
+ utf16_mode = UTF16LE_INPUT;
+#endif
+ }
+ continue;
+ }
+ if (strcmp(long_option[i].name, "oc=") == 0){
+ for (i=0; i < 16 && SPACE < p[i] && p[i] < DEL; i++){
+ codeset[i] = nkf_toupper(p[i]);
+ }
+ codeset[i] = 0;
+ if(strcmp(codeset, "ISO-2022-JP") == 0){
+ output_conv = j_oconv;
+ }else if(strcmp(codeset, "SHIFT_JIS") == 0){
+ output_conv = s_oconv;
+ }else if(strcmp(codeset, "CP932") == 0){
+ output_conv = s_oconv;
+ x0201_f = FALSE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = TRUE;
+ cp932inv_f = TRUE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 2;
+#endif
+ }else if(strcmp(codeset, "EUCJP") == 0 ||
+ strcmp(codeset, "EUC-JP") == 0){
+ output_conv = e_oconv;
+ }else if(strcmp(codeset, "CP51932") == 0){
+ output_conv = e_oconv;
+ x0201_f = FALSE;
+#ifdef SHIFTJIS_CP932
+ cp51932_f = TRUE;
+ cp932inv_f = TRUE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 2;
+#endif
+ }else if(strcmp(codeset, "EUC-JP-MS") == 0 ||
+ strcmp(codeset, "EUCJP-MS") == 0){
+ output_conv = e_oconv;
+ x0201_f = FALSE;
+#ifdef X0212_ENABLE
+ x0212_f = TRUE;
+#endif
+#ifdef SHIFTJIS_CP932
+ cp51932_f = FALSE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 1;
+#endif
+ }else if(strcmp(codeset, "EUC-JP-ASCII") == 0 ||
+ strcmp(codeset, "EUCJP-ASCII") == 0){
+ output_conv = e_oconv;
+ x0201_f = FALSE;
+#ifdef X0212_ENABLE
+ x0212_f = TRUE;
+#endif
+#ifdef SHIFTJIS_CP932
+ cp51932_f = FALSE;
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+ ms_ucs_map_f = 0;
+#endif
+ }else if(strcmp(codeset, "SHIFT_JISX0213") == 0){
+ output_conv = s_oconv;
+ x0213_f = TRUE;
+ }else if(strcmp(codeset, "EUC-JISX0213") == 0){
+ output_conv = e_oconv;
+#ifdef X0212_ENABLE
+ x0212_f = TRUE;
+#endif
+ x0213_f = TRUE;
+#ifdef UTF8_OUTPUT_ENABLE
+ }else if(strcmp(codeset, "UTF-8") == 0){
+ output_conv = w_oconv;
+ }else if(strcmp(codeset, "UTF-8N") == 0){
+ output_conv = w_oconv;
+ unicode_bom_f=1;
+ }else if(strcmp(codeset, "UTF-8-BOM") == 0){
+ output_conv = w_oconv;
+ unicode_bom_f=2;
+ }else if(strcmp(codeset, "UTF-16BE") == 0){
+ output_conv = w_oconv16;
+ unicode_bom_f=1;
+ }else if(strcmp(codeset, "UTF-16") == 0 ||
+ strcmp(codeset, "UTF-16BE-BOM") == 0){
+ output_conv = w_oconv16;
+ unicode_bom_f=2;
+ }else if(strcmp(codeset, "UTF-16LE") == 0){
+ output_conv = w_oconv16;
+ w_oconv16_LE = 1;
+ unicode_bom_f=1;
+ }else if(strcmp(codeset, "UTF-16LE-BOM") == 0){
+ output_conv = w_oconv16;
+ w_oconv16_LE = 1;
+ unicode_bom_f=2;
+#endif
+ }
+ continue;
+ }
#ifdef OVERWRITE
if (strcmp(long_option[i].name, "overwrite") == 0){
file_out = TRUE;
cp932inv_f = TRUE;
#endif
#ifdef UTF8_OUTPUT_ENABLE
- ms_ucs_map_f = TRUE;
+ ms_ucs_map_f = 2;
#endif
continue;
}
cp932inv_f = FALSE;
#endif
#ifdef UTF8_OUTPUT_ENABLE
- ms_ucs_map_f = FALSE;
+ ms_ucs_map_f = 0;
#endif
continue;
}
internal_unicode_f = TRUE;
continue;
}
+ if (strcmp(long_option[i].name, "disable-cp932ext") == 0){
+ disable_cp932ext_f = TRUE;
+ continue;
+ }
+ if (strcmp(long_option[i].name, "enable-round-trip") == 0){
+ unicode_round_trip_f = TRUE;
+ continue;
+ }
if (strcmp(long_option[i].name, "fb-skip") == 0){
encode_fallback = NULL;
continue;
#endif
#ifdef UTF8_OUTPUT_ENABLE
if (strcmp(long_option[i].name, "ms-ucs-map") == 0){
- ms_ucs_map_f = TRUE;
+ ms_ucs_map_f = 1;
continue;
}
#endif
#endif /* SHIFTJIS_CP932 */
#ifndef X0212_ENABLE
status_disable(ptr);
- break;
#endif
+ break;
}
}
module_connection();
c2 = 0;
+ if(input_f == SJIS_INPUT
+#ifdef UTF8_INPUT_ENABLE
+ || input_f == UTF8_INPUT || input_f == UTF16BE_INPUT
+#endif
+ ){
+ is_8bit = TRUE;
+ }
+
input_mode = ASCII;
output_mode = ASCII;
return ((hold_count >= HOLD_SIZE*2) ? EOF : hold_count);
}
+const int shift_jisx0213_s1a3_table[5][2] ={ { 1, 8}, { 3, 4}, { 5,12}, {13,14}, {15, 0} };
+
int s2e_conv(c2, c1, p2, p1)
int c2, c1;
int *p2, *p1;
}
#endif /* SHIFTJIS_CP932 */
#ifdef X0212_ENABLE
- if (x0212_f && 0xfa <= c2 && c2 <= 0xfc){
+ if (!x0213_f && x0212_f && 0xfa <= c2 && c2 <= 0xfc){
extern const unsigned short shiftjis_x0212[3][189];
val = shiftjis_x0212[c2 - 0xfa][c1 - 0x40];
if (val){
}
#endif
if(c2 >= 0x80){
- c2 = c2 + c2 - ((c2 <= 0x9f) ? SJ0162 : SJ6394);
- if (c1 < 0x9f)
- c1 = c1 - ((c1 > DEL) ? SPACE : 0x1f);
+ if(x0213_f && c2 >= 0xF0){
+ if(c2 <= 0xF3 || (c2 == 0xF4 && c1 < 0x9F)){ /* k=1, 3<=k<=5, k=8, 12<=k<=15 */
+ c2 = 0x8F20 + shift_jisx0213_s1a3_table[c2 - 0xF0][0x9E < c1];
+ }else{ /* 78<=k<=94 */
+ c2 = 0x8F00 | (c2 * 2 - 0x17B);
+ if (0x9E < c1) c2++;
+ }
+ }else{
+ c2 = c2 + c2 - ((c2 <= 0x9F) ? SJ0162 : SJ6394);
+ if (0x9E < c1) c2++;
+ }
+ if (c1 < 0x9F)
+ c1 = c1 - ((c1 > DEL) ? SPACE : 0x1F);
else {
- c1 = c1 - 0x7e;
- c2++;
+ c1 = c1 - 0x7E;
}
}
int c2, c1, c0;
int *p2, *p1;
{
- extern const unsigned short *const utf8_to_euc_2bytes[];
- extern const unsigned short *const *const utf8_to_euc_3bytes[];
int ret = 0;
- if (0xc0 <= c2 && c2 <= 0xef) {
- const unsigned short *const *pp;
-
- if (0xe0 <= c2) {
- if (c0 == 0) return -1;
- pp = utf8_to_euc_3bytes[c2 - 0x80];
- ret = w_iconv_common(c1, c0, pp, sizeof_utf8_to_euc_C2, p2, p1);
- } else {
- ret = w_iconv_common(c2, c1, utf8_to_euc_2bytes, sizeof_utf8_to_euc_2bytes, p2, p1);
- }
+ if (!c1){
+ *p2 = 0;
+ *p1 = c2;
+ }else if (0xc0 <= c2 && c2 <= 0xef) {
+ ret = unicode_to_jis_common(c2, c1, c0, p2, p1);
#ifdef NUMCHAR_OPTION
- if (ret){
+ if (ret > 0){
if (p2) *p2 = 0;
if (p1) *p1 = CLASS_UTF16 | ww16_conv(c2, c1, c0);
ret = 0;
}
#endif
- return ret;
- } else if (c2 == X0201) {
- c1 &= 0x7f;
}
- if (p2) *p2 = c2;
- if (p1) *p1 = c1;
return ret;
}
{
int ret = 0;
- if (c0 == 0){
- if (c2 == 0) /* 0x00-0x7f */
- ; /* 1byte */
- else if ((c2 & 0xe0) == 0xc0) /* 0xc0-0xdf */
- ; /* 2ytes */
- else if ((c2 & 0xf0) == 0xe0) /* 0xe0-0xef */
+ /* throw away ZERO WIDTH NO-BREAK SPACE (U+FEFF) */
+ if(ignore_zwnbsp_f){
+ ignore_zwnbsp_f = FALSE;
+ if(c2 == 0xef && c1 == 0xbb && c0 == 0xbf)
+ return 0;
+ }
+
+ if (c2 == 0) /* 0x00-0x7f */
+ c1 &= 0x7F; /* 1byte */
+ else if (c0 == 0){
+ if ((c2 & 0xe0) == 0xc0){ /* 0xc0-0xdf */
+ /* 2ytes */
+ if((c2 & 0xFE) == 0xC0 || c1 < 0x80 || 0xBF < c1) return 0;
+ }else if ((c2 & 0xf0) == 0xe0) /* 0xe0-0xef */
return -1; /* 3bytes */
#ifdef __COMMENT__
else if (0xf0 <= c2)
return 0; /* trail byte */
#endif
else return 0;
- }
- if (c2 == EOF);
- else if (c2 == 0xef && c1 == 0xbb && c0 == 0xbf) {
- return 0; /* throw BOM */
-#if defined(UTF8_OUTPUT_ENABLE) && defined(UTF8_INPUT_ENABLE)
+ }else{
+ /* must be 3bytes */
+ if(c2 == 0xE0){
+ if(c1 < 0xA0 || 0xBF < c1 || c0 < 0x80 || 0xBF < c0)
+ return 0;
+ }else if(c2 == 0xED){
+ if(c1 < 0x80 || 0x9F < c1 || c0 < 0x80 || 0xBF < c0)
+ return 0;
+ }else if((c2 & 0xf0) == 0xe0){
+ if(c1 < 0x80 || 0xBF < c1 || c0 < 0x80 || 0xBF < c0)
+ return 0;
+ }else return 0;
+ }
+ if (c2 == 0 || c2 == EOF){
+#ifdef UTF8_OUTPUT_ENABLE
} else if (internal_unicode_f && (output_conv == w_oconv || output_conv == w_oconv16)){
unsigned short val = 0;
if(c2 == 0){
}
return ret;
}
+#endif
+#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
void
w16w_conv(val, p2, p1, p0)
unsigned short val;
*p0 = 0x80 | (val & 0x3f);
}
}
+#endif
+#ifdef UTF8_INPUT_ENABLE
int
ww16_conv(c2, c1, c0)
int c2, c1, c0;
{
unsigned short val;
- if (c2 >= 0xe0){
+ if (c2 >= 0xf0){
+ val = -1;
+ }else if (c2 >= 0xe0){
val = (c2 & 0x0f) << 12;
val |= (c1 & 0x3f) << 6;
val |= (c0 & 0x3f);
unsigned short val;
int *p2, *p1;
{
- extern const unsigned short *const utf8_to_euc_2bytes[];
- extern const unsigned short *const *const utf8_to_euc_3bytes[];
int c2, c1, c0;
- const unsigned short *const *pp;
- int psize;
int ret = 0;
- w16w_conv(val, &c2, &c1, &c0);
- if (c1){
- if (c0){
- pp = utf8_to_euc_3bytes[c2 - 0x80];
- psize = sizeof_utf8_to_euc_C2;
- ret = w_iconv_common(c1, c0, pp, psize, p2, p1);
- }else{
- pp = utf8_to_euc_2bytes;
- psize = sizeof_utf8_to_euc_2bytes;
- ret = w_iconv_common(c2, c1, pp, psize, p2, p1);
- }
+ if (val < 0x80){
+ *p2 = 0;
+ *p1 = val;
+ }else{
+ if(!ms_ucs_map_f){
+ /* eucJP-ascii */
+ switch(val){
+ case 0x203E:
+ *p2 = 0x21;
+ *p1 = 0x31;
+ return ret;
+ break;
+ case 0xFF5E:
+ *p2 = 0x8F22;
+ *p1 = 0x37;
+ return ret;
+ break;
+ }
+ }
+ w16w_conv(val, &c2, &c1, &c0);
+ ret = unicode_to_jis_common(c2, c1, c0, p2, p1);
#ifdef NUMCHAR_OPTION
- if (ret){
- *p2 = 0;
- *p1 = CLASS_UTF16 | val;
- ret = 0;
- }
+ if (ret > 0){
+ *p2 = 0;
+ *p1 = CLASS_UTF16 | val;
+ ret = 0;
+ }
#endif
- }else{
- *p2 = 0;
- *p1 = c2;
}
return ret;
}
+#endif
+#ifdef UTF8_INPUT_ENABLE
int
w_iconv16(c2, c1, c0)
int c2, c1,c0;
{
int ret = 0;
- if (c2==0376 && c1==0377){
- utf16_mode = UTF16BE_INPUT;
- return 0;
- } else if (c2==0377 && c1==0376){
- utf16_mode = UTF16LE_INPUT;
- return 0;
+ /* throw away ZERO WIDTH NO-BREAK SPACE (U+FEFF) */
+ if(ignore_zwnbsp_f){
+ ignore_zwnbsp_f = FALSE;
+ if (c2==0376 && c1==0377){
+ utf16_mode = UTF16BE_INPUT;
+ return 0;
+ }else if(c2==0377 && c1==0376){
+ utf16_mode = UTF16LE_INPUT;
+ return 0;
+ }
}
if (c2 != EOF && utf16_mode == UTF16LE_INPUT) {
int tmp;
if ((c2==0 && c1 < 0x80) || c2==EOF) {
(*oconv)(c2, c1);
return 0;
- }
-#if defined(UTF8_OUTPUT_ENABLE) && defined(UTF8_INPUT_ENABLE)
- if (internal_unicode_f && (output_conv == w_oconv || output_conv == w_oconv16));
+ }else if((c2>>3)==27){ /* surrogate pair */
+ return 1;
+#ifdef UTF8_OUTPUT_ENABLE
+ }else if (internal_unicode_f && (output_conv == w_oconv || output_conv == w_oconv16)){
#endif
- else ret = w16e_conv(((c2<<8)&0xff00) + c1, &c2, &c1);
+ }else ret = w16e_conv(((c2<<8)&0xff00) + c1, &c2, &c1);
if (ret) return ret;
(*oconv)(c2, c1);
return 0;
}
int
+unicode_to_jis_common(c2, c1, c0, p2, p1)
+ int c2, c1, c0;
+ int *p2, *p1;
+{
+ extern const unsigned short *const utf8_to_euc_2bytes[];
+ extern const unsigned short *const *const utf8_to_euc_3bytes[];
+ int ret = 0;
+
+ if(c2 < 0xe0){
+ if (ms_ucs_map_f == 2){
+ /* CP932/CP51932: U+00A6 (BROKEN BAR) -> not 0x8fa2c3, but 0x7c */
+ if(c2 == 0xC2){
+ switch(c1){
+ case 0xA5:
+ if (p2) *p2 = 0;
+ if (p1) *p1 = 0x5C;
+ return 0;
+ case 0xA6:
+ if (p2) *p2 = 0;
+ if (p1) *p1 = 0x7C;
+ return 0;
+ }
+ }
+ }else if(strict_mapping_f){
+ switch(c2){
+ case 0xC2:
+ switch(c1){
+ case 0xAB: case 0xAD: case 0xB2: case 0xB3:
+ case 0xB5: case 0xB7: case 0xB9: case 0xBB:
+ return 1;
+ }
+ break;
+ case 0xC3:
+ switch(c1){
+ case 0x90:
+ return 1;
+ }
+ break;
+ }
+ }
+ ret = w_iconv_common(c2, c1, utf8_to_euc_2bytes, sizeof_utf8_to_euc_2bytes, p2, p1);
+ if(!ret && !ms_ucs_map_f
+#ifdef X0212_ENABLE
+ && !x0212_f
+#endif
+ ){
+ if(*p2 == 0 && *p1 < 0x80){
+ return 1;
+ }else if(*p2 > 0xFF){
+ int s2, s1;
+ if (e2s_conv(*p2, *p1, &s2, &s1) == 0){
+ s2e_conv(s2, s1, p2, p1);
+ if(*p2 == 0 && *p1 < 0x80)
+ return 1;
+ }else return 1;
+ }
+ }
+ }else if(c0){
+ if(unicode_round_trip_f){
+ switch(c2){
+ case 0xE2:
+ switch(c1){
+ case 0x80:
+ if(c0 == 0x95) return 1;
+ break;
+ case 0x88:
+ if(c0 == 0xA5) return 1;
+ break;
+ }
+ break;
+ case 0xEF:
+ switch(c1){
+ case 0xBB:
+ if(c0 == 0xBF) return 1;
+ break;
+ case 0xBC:
+ if(c0 == 0x8D) return 1;
+ break;
+ case 0xBF:
+ if(0xA0 <= c0 && c0 <= 0xA5) return 1;
+ break;
+ }
+ break;
+ }
+ }
+ if(!ms_ucs_map_f){
+ /* eucJP-ascii */
+ if(c2 == 0xE2 && c1 == 0x80 && c0 == 0xBE){
+ if (p2) *p2 = 0x21;
+ if (p1) *p1 = 0x31;
+ return ret;
+ }else if(c2 == 0xEF && c1 == 0xBD && c0 == 0x9E){
+ if (p2) *p2 = 0x8F22;
+ if (p1) *p1 = 0x37;
+ return ret;
+ }
+ }
+ if(!strict_mapping_f);
+ else if(ms_ucs_map_f == 2){
+ /* Microsoft Code Page */
+ switch(c2){
+ case 0xE2:
+ switch(c1){
+ case 0x80:
+ switch(c0){
+ case 0x94: case 0x96: case 0xBE:
+ return 1;
+ }
+ break;
+ case 0x88:
+ if(c0 == 0x92)
+ return 1;
+ break;
+ }
+ break;
+ case 0xE3:
+ switch(c1){
+ case 0x80:
+ if(c0 == 0x9C)
+ return 1;
+ break;
+ }
+ break;
+ }
+ }else{
+ /* eucJP-open */
+ if(c2 == 0xE3 && c1 == 0x82 && c0 == 0x94)
+ return 1;
+ }
+ ret = w_iconv_common(c1, c0, utf8_to_euc_3bytes[c2 - 0xE0], sizeof_utf8_to_euc_C2, p2, p1);
+ }else return -1;
+ return ret;
+}
+
+int
w_iconv_common(c1, c0, pp, psize, p2, p1)
int c1,c0;
const unsigned short *const *pp;
const unsigned short *p;
unsigned short val;
- /* CP932/CP51932: U+00A6 (BROKEN BAR) -> not 0x8fa2c3, but 0x7c */
- if (cp51932_f && c1 == 0xC2 && c0 == 0xA6){
- if (p2) *p2 = 0;
- if (p1) *p1 = 0x7C;
- return 0;
- }
-
if (pp == 0) return 1;
c1 -= 0x80;
if (c0 < 0 || sizeof_utf8_to_euc_E5B8 <= c0) return 1;
val = p[c0];
if (val == 0) return 1;
+ if (disable_cp932ext_f && (
+ (val>>8) == 0x2D || /* disable NEC special characters */
+ val > 0xF300 /* disable NEC special characters */
+ )) return 1;
c2 = val >> 8;
if (val & 0x8000){
return 0;
}
-#endif
-
-#ifdef UTF8_OUTPUT_ENABLE
void
nkf_each_char_to_hex(f, c)
void (*f)PROTO((int c2,int c1));
}
return;
}
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
int
e2w_conv(c2, c1)
int c2, c1;
p = euc_to_utf8_1byte;
#ifdef X0212_ENABLE
} else if (c2 >> 8 == 0x8f){
+ if(!ms_ucs_map_f && c2 == 0x8F22 && c1 == 0x43){
+ return 0xA6;
+ }
extern const unsigned short *const x0212_to_utf8_2bytes[];
c2 = (c2&0x7f) - 0x21;
if (0<=c2 && c2<sizeof_euc_to_utf8_2bytes)
(*o_putc)(c1 | 0x080);
} else {
output_mode = UTF8;
+#ifdef UTF8_INPUT_ENABLE
if (internal_unicode_f && (iconv == w_iconv || iconv == w_iconv16))
val = ((c2<<8)&0xff00) + c1;
- else val = e2w_conv(c2, c1);
+ else
+#endif
+ val = e2w_conv(c2, c1);
if (val){
w16w_conv(val, &c2, &c1, &c0);
(*o_putc)(c2);
unicode_bom_f=1;
}
+#ifdef UTF8_INPUT_ENABLE
if (internal_unicode_f && (iconv == w_iconv || iconv == w_iconv16)){
- } else if (c2 == ISO8859_1) {
+ } else
+#endif
+ if (c2 == ISO8859_1) {
c2 = 0;
c1 |= 0x80;
#ifdef NUMCHAR_OPTION
e2s_conv(c2, c1, p2, p1)
int c2, c1, *p2, *p1;
{
-#ifdef X0212_ENABLE
- int val = 0;
- const unsigned short *ptr;
int ndx;
- extern const unsigned short *const x0212_shiftjis[];
if ((c2 & 0xff00) == 0x8f00){
- ndx = c2 & 0x7f;
- if (0x21 <= ndx && ndx <= 0x7e){
- ptr = x0212_shiftjis[ndx - 0x21];
- if (ptr){
- val = ptr[(c1 & 0x7f) - 0x21];
- }
- if (val){
- c2 = val >> 8;
- c1 = val & 0xff;
- if (p2) *p2 = c2;
- if (p1) *p1 = c1;
- return 0;
- }
- }
- c2 = x0212_shift(c2);
- }
+ ndx = c2 & 0xff;
+ if (x0213_f){
+ if((0x21 <= ndx && ndx <= 0x2F)){
+ if (p2) *p2 = ((ndx - 1) >> 1) + 0xec - ndx / 8 * 3;
+ if (p1) *p1 = c1 + ((ndx & 1) ? ((c1 < 0x60) ? 0x1f : 0x20) : 0x7e);
+ return 0;
+ }else if(0x6E <= ndx && ndx <= 0x7E){
+ if (p2) *p2 = ((ndx - 1) >> 1) + 0xbe;
+ if (p1) *p1 = c1 + ((ndx & 1) ? ((c1 < 0x60) ? 0x1f : 0x20) : 0x7e);
+ return 0;
+ }
+ return 1;
+ }
+#ifdef X0212_ENABLE
+ else if(0x21 <= ndx && ndx <= 0x7e){
+ int val = 0;
+ const unsigned short *ptr;
+ extern const unsigned short *const x0212_shiftjis[];
+ ndx = c2 & 0x7f;
+ ptr = x0212_shiftjis[ndx - 0x21];
+ if (ptr){
+ val = ptr[(c1 & 0x7f) - 0x21];
+ }
+ if (val){
+ c2 = val >> 8;
+ c1 = val & 0xff;
+ if (p2) *p2 = c2;
+ if (p1) *p1 = c1;
+ return 0;
+ }
+ c2 = x0212_shift(c2);
+ }
#endif /* X0212_ENABLE */
- if ((c2 & 0xff00) == 0x8f00){
- return 1;
}
+ if(0x7F < c2) return 1;
if (p2) *p2 = ((c2 - 1) >> 1) + ((c2 <= 0x5e) ? 0x71 : 0xb1);
if (p1) *p1 = c1 + ((c2 & 1) ? ((c1 < 0x60) ? 0x1f : 0x20) : 0x7e);
return 0;
(*o_putc)(EOF);
#ifdef X0212_ENABLE
} else if ((c2 & 0xff00) >> 8 == 0x8f){
- if (output_mode!=X0212) {
- output_mode = X0212;
- (*o_putc)(ESC);
- (*o_putc)('$');
- (*o_putc)('(');
- (*o_putc)('D');
+ if(x0213_f){
+ if(output_mode!=X0213_2){
+ output_mode = X0213_2;
+ }
+ }else{
+ if(output_mode!=X0212){
+ output_mode = X0212;
+ }
}
+ (*o_putc)(ESC);
+ (*o_putc)('$');
+ (*o_putc)('(');
+ (*o_putc)(output_mode & 0x7F);
(*o_putc)(c2 & 0x7f);
(*o_putc)(c1);
#endif
}
(*o_putc)(c1);
} else {
- if (output_mode != X0208) {
+ if(x0213_f){
+ if (output_mode!=X0213_1) {
+ output_mode = X0213_1;
+ (*o_putc)(ESC);
+ (*o_putc)('$');
+ (*o_putc)('(');
+ (*o_putc)(output_mode & 0x7F);
+ }
+ }else if (output_mode != X0208) {
output_mode = X0208;
(*o_putc)(ESC);
(*o_putc)('$');
(const unsigned char *)"\075?ISO-8859-1?B?",
(const unsigned char *)"\075?ISO-2022-JP?B?",
(const unsigned char *)"\075?ISO-2022-JP?Q?",
-#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
+#if defined(UTF8_INPUT_ENABLE)
(const unsigned char *)"\075?UTF-8?B?",
(const unsigned char *)"\075?UTF-8?Q?",
#endif
/* \e$B3:Ev$9$k%3!<%I$NM%@hEY$r>e$2$k$?$a$NL\0u\e(B */
int (*mime_priority_func[])PROTO((int c2, int c1, int c0)) = {
e_iconv, s_iconv, 0, 0, 0, 0,
-#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
+#if defined(UTF8_INPUT_ENABLE)
w_iconv, w_iconv,
#endif
0,
const int mime_encode[] = {
JAPANESE_EUC, SHIFT_JIS,ISO8859_1, ISO8859_1, X0208, X0201,
-#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
+#if defined(UTF8_INPUT_ENABLE)
UTF8, UTF8,
#endif
ASCII,
const int mime_encode_method[] = {
'B', 'B','Q', 'B', 'B', 'Q',
-#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
+#if defined(UTF8_INPUT_ENABLE)
'B', 'Q',
#endif
'Q',
if (mime_decode_mode == 'Q') {
if ((c1 = (*i_mgetc)(f)) == EOF) return (EOF);
restart_mime_q:
- if (c1=='_') return ' ';
+ if (c1=='_' && mimebuf_f != FIXED_MIME) return ' ';
if (c1<=' ' || DEL<=c1) {
mime_decode_mode = exit_mode; /* prepare for quit */
return c1;
}
- if (c1!='=' && c1!='?') {
+ if (c1!='=' && (c1!='?' || mimebuf_f == FIXED_MIME)) {
return c1;
}
{
switch(mimeout_mode) {
case 'Q':
- if(c==SPACE){
- (*o_mputc)('_');
- base64_count++;
- } else if (c==CR||c==NL) {
+ if (c==CR||c==NL) {
(*o_mputc)(c);
base64_count = 0;
- } else if(c<SPACE||c=='='||c=='?'||c=='_'||DEL<=c) {
+ } else if(!nkf_isalnum(c)) {
(*o_mputc)('=');
(*o_mputc)(itoh4(((c>>4)&0xf)));
(*o_mputc)(itoh4((c&0xf)));
x0201_f = NO_X0201;
#endif
iso2022jp_f = FALSE;
+#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
+ ms_ucs_map_f = 0;
+#endif
#if defined(UTF8_OUTPUT_ENABLE) && defined(UTF8_INPUT_ENABLE)
internal_unicode_f = FALSE;
#endif
+#ifdef UTF8_INPUT_ENABLE
+ strict_mapping_f = TRUE;
+ disable_cp932ext_f = FALSE;
+ ignore_zwnbsp_f = TRUE;
+ unicode_round_trip_f = FALSE;
+ encode_fallback = NULL;
+ unicode_subchar = '?';
+#endif
#ifdef UTF8_OUTPUT_ENABLE
unicode_bom_f = 0;
w_oconv16_LE = 0;
- ms_ucs_map_f = FALSE;
- encode_fallback = NULL;
- unicode_subchar = '?';
#endif
#ifdef UNICODE_NORMALIZATION
nfc_f = FALSE;
#endif
#ifdef X0212_ENABLE
x0212_f = FALSE;
+ x0213_f = FALSE;
#endif
{
int i;
fprintf(stderr,"I Convert non ISO-2022-JP charactor to GETA\n");
fprintf(stderr,"-L[uwm] line mode u:LF w:CRLF m:CR (DEFAULT noconversion)\n");
fprintf(stderr,"long name options\n");
+ fprintf(stderr," --ic=<input codeset> --oc=<output codeset> set the input or output codeset\n");
fprintf(stderr," --fj,--unix,--mac,--windows convert for the system\n");
fprintf(stderr," --jis,--euc,--sjis,--utf8,--utf16,--mime,--base64 convert for the code\n");
fprintf(stderr," --hiragana, --katakana Hiragana/Katakana Conversion\n");
- fprintf(stderr," --x0212 Convert JISX0212\n");
- fprintf(stderr," --cp932, --no-cp932 CP932 compatibility\n");
fprintf(stderr," --prefix= Insert escape before troublesome characters of Shift_JIS\n");
#ifdef INPUT_OPTION
fprintf(stderr," --cap-input, --url-input Convert hex after ':' or '%%'\n");
#ifdef NUMCHAR_OPTION
fprintf(stderr," --numchar-input Convert Unicode Character Reference\n");
#endif
-#ifdef UNICODE_NORMALIZATION
- fprintf(stderr," --utf8mac-input UTF-8-MAC input\n");
-#endif
#ifdef UTF8_INPUT_ENABLE
fprintf(stderr," --fb-{skip, html, xml, perl, java, subchar}\n");
fprintf(stderr," set the way nkf handles unassigned characters\n");
#endif
-#ifdef UTF8_OUTPUT_ENABLE
- fprintf(stderr," --ms-ucs-map Microsoft UCS Mapping Compatible\n");
-#endif
#ifdef OVERWRITE
fprintf(stderr," --overwrite Overwrite original listed files by filtered result\n");
#endif