*
* 3. This notice may not be removed or altered from any source distribution.
*/
-#define NKF_VERSION "2.0.9"
-#define NKF_RELEASE_DATE "2009-02-20"
+#define NKF_VERSION "2.1.1"
+#define NKF_RELEASE_DATE "2010-01-05"
#define COPY_RIGHT \
"Copyright (C) 1987, FUJITSU LTD. (I.Ichikawa).\n" \
- "Copyright (C) 1996-2009, The nkf Project."
+ "Copyright (C) 1996-2010, The nkf Project."
#include "config.h"
#include "nkf.h"
} encoding_name_to_id_table[] = {
{"US-ASCII", ASCII},
{"ASCII", ASCII},
+ {"646", ASCII},
+ {"ROMAN8", ASCII},
{"ISO-2022-JP", ISO_2022_JP},
{"ISO2022JP-CP932", CP50220},
{"CP50220", CP50220},
{"ISO-2022-JP-2004", ISO_2022_JP_2004},
{"SHIFT_JIS", SHIFT_JIS},
{"SJIS", SHIFT_JIS},
+ {"PCK", SHIFT_JIS},
{"WINDOWS-31J", WINDOWS_31J},
{"CSWINDOWS31J", WINDOWS_31J},
{"CP932", WINDOWS_31J},
&& (c != '(') && (c != ')') && (c != '.') && (c != 0x22)))
#define is_ibmext_in_sjis(c2) (CP932_TABLE_BEGIN <= c2 && c2 <= CP932_TABLE_END)
-#define nkf_byte_jisx0201_katakana_p(c) (SP <= c && c < (0xE0&0x7F))
+#define nkf_byte_jisx0201_katakana_p(c) (SP <= c && c <= 0x5F)
#define HOLD_SIZE 1024
#if defined(INT_IS_SHORT)
{"Shift_JIS", 0, 0, 0, {0, 0, 0}, s_status, s_iconv, 0},
#ifdef UTF8_INPUT_ENABLE
{"UTF-8", 0, 0, 0, {0, 0, 0}, w_status, w_iconv, 0},
+ {"UTF-16", 0, 0, 0, {0, 0, 0}, NULL, w_iconv16, 0},
+ {"UTF-32", 0, 0, 0, {0, 0, 0}, NULL, w_iconv32, 0},
#endif
{0}
};
int shift = 20;
c &= VALUE_MASK;
while(shift >= 0){
- if(c >= 1<<shift){
+ if(c >= NKF_INT32_C(1)<<shift){
while(shift >= 0){
(*f)(0, bin2hex(c>>shift));
shift -= 4;
case CP50220:
case CP50221:
case CP50222:
+ x0201_f = TRUE;
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
case SHIFT_JIS:
break;
case WINDOWS_31J:
+ x0201_f = TRUE;
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
case EUCJP_NKF:
break;
case CP51932:
+ x0201_f = TRUE;
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
#endif
break;
case CP50221:
+ x0201_f = TRUE;
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
ms_ucs_map_f = UCS_MAP_CP932;
#endif
break;
+ case ISO_2022_JP:
+#ifdef SHIFTJIS_CP932
+ if (cp932inv_f == TRUE) cp932inv_f = FALSE;
+#endif
+ break;
case ISO_2022_JP_1:
x0212_f = TRUE;
#ifdef SHIFTJIS_CP932
case SHIFT_JIS:
break;
case WINDOWS_31J:
+ x0201_f = TRUE;
#ifdef UTF8_OUTPUT_ENABLE
ms_ucs_map_f = UCS_MAP_CP932;
#endif
#endif
break;
case CP51932:
+ x0201_f = TRUE;
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
output_endian = ENDIAN_LITTLE;
output_bom_f = TRUE;
break;
+ case UTF_32:
case UTF_32BE_BOM:
output_bom_f = TRUE;
break;
*p3 = 0x80 | ( val & 0x3f);
*p4 = 0;
} else if (nkf_char_unicode_value_p(val)) {
- *p1 = 0xe0 | (val >> 16);
+ *p1 = 0xf0 | (val >> 18);
*p2 = 0x80 | ((val >> 12) & 0x3f);
*p3 = 0x80 | ((val >> 6) & 0x3f);
*p4 = 0x80 | ( val & 0x3f);
static nkf_char
w_iconv16(nkf_char c2, nkf_char c1, nkf_char c0)
{
- return 0;
+ (*oconv)(c2, c1);
+ return 16; /* different from w_iconv32 */
}
static nkf_char
w_iconv32(nkf_char c2, nkf_char c1, nkf_char c0)
{
- return 0;
+ (*oconv)(c2, c1);
+ return 32; /* different from w_iconv16 */
}
static size_t
}
static int
-h_conv(FILE *f, int c1, int c2)
+h_conv(FILE *f, nkf_char c1, nkf_char c2)
{
- int ret, c4, c3;
+ int ret;
int hold_index;
-
+ nkf_char c3, c4;
/** it must NOT be in the kanji shifte sequence */
/** it must NOT be written in JIS7 */
} else if ((c3 = (*i_getc)(f)) == EOF) {
ret = EOF;
break;
- } else {
- code_status(c3);
- if (hold_index < hold_count){
- c4 = hold_buf[hold_index++];
- } else if ((c4 = (*i_getc)(f)) == EOF) {
- c3 = ret = EOF;
- break;
- } else {
- code_status(c4);
- (*iconv)(c1, c2, (c3<<8)|c4);
- }
}
+ code_status(c3);
+ if (hold_index < hold_count){
+ c4 = hold_buf[hold_index++];
+ } else if ((c4 = (*i_getc)(f)) == EOF) {
+ c3 = ret = EOF;
+ break;
+ }
+ code_status(c4);
+ (*iconv)(c1, c2, (c3<<8)|c4);
break;
case -1:
/* 3 bytes EUC or UTF-8 */
(const unsigned char *)"\075?ISO-8859-1?Q?",
(const unsigned char *)"\075?ISO-8859-1?B?",
(const unsigned char *)"\075?ISO-2022-JP?B?",
+ (const unsigned char *)"\075?ISO-2022-JP?B?",
(const unsigned char *)"\075?ISO-2022-JP?Q?",
#if defined(UTF8_INPUT_ENABLE)
(const unsigned char *)"\075?UTF-8?B?",
};
static const nkf_char mime_encode[] = {
- EUC_JP, SHIFT_JIS, ISO_8859_1, ISO_8859_1, JIS_X_0208, JIS_X_0201_1976_K,
+ EUC_JP, SHIFT_JIS, ISO_8859_1, ISO_8859_1, JIS_X_0208, JIS_X_0201_1976_K, JIS_X_0201_1976_K,
#if defined(UTF8_INPUT_ENABLE)
UTF_8, UTF_8,
#endif
};
static const nkf_char mime_encode_method[] = {
- 'B', 'B','Q', 'B', 'B', 'Q',
+ 'B', 'B','Q', 'B', 'B', 'B', 'Q',
#if defined(UTF8_INPUT_ENABLE)
'B', 'Q',
#endif
}
if (c1=='='&&c2<SP) { /* this is soft wrap */
while((c1 = (*i_mgetc)(f)) <=SP) {
- if ((c1 = (*i_mgetc)(f)) == EOF) return (EOF);
+ if (c1 == EOF) return (EOF);
}
mime_decode_mode = 'Q'; /* still in MIME */
goto restart_mime_q;
base64_count = 1;
}
} else {
- if (base64_count + mimeout_state.count/3*4> 66) {
+ if (!(c2 == 0 && (c1 == CR || c1 == LF)) &&
+ base64_count + mimeout_state.count/3*4> 66) {
(*o_base64conv)(EOF,0);
OCONV_NEWLINE((*o_base64conv));
(*o_base64conv)(0,SP);
return;
}
}
- (*o_mputc)(c);
- base64_count++;
+ if (c != 0x1B) {
+ (*o_mputc)(c);
+ base64_count++;
+ return;
+ }
}
- return;
}
if (mimeout_mode <= 0) {
- if (c <= DEL && (output_mode==ASCII ||output_mode == ISO_8859_1)) {
+ if (c <= DEL && (output_mode==ASCII || output_mode == ISO_8859_1 ||
+ output_mode == UTF_8)) {
if (nkf_isspace(c)) {
int flag = 0;
if (mimeout_mode == -1) {
}
}else{
/* mimeout_mode == 'B', 1, 2 */
- if ( c<=DEL && (output_mode==ASCII ||output_mode == ISO_8859_1)) {
+ if (c <= DEL && (output_mode==ASCII || output_mode == ISO_8859_1 ||
+ output_mode == UTF_8)) {
if (lastchar == CR || lastchar == LF){
if (nkf_isblank(c)) {
for (i=0;i<mimeout_state.count;i++) {
mimeout_addchar(mimeout_state.buf[i]);
}
mimeout_state.count = 0;
- } else if (SP<c && c<DEL) {
+ } else {
eof_mime();
for (i=0;i<mimeout_state.count;i++) {
(*o_mputc)(mimeout_state.buf[i]);
set_output_encoding(output_encoding);
oconv = nkf_enc_to_oconv(output_encoding);
o_putc = std_putc;
+ if (nkf_enc_unicode_p(output_encoding))
+ output_mode = UTF_8;
/* replace continucation module, from output side */
(c4 = (*i_getc)(f)) != EOF) {
nkf_iconv_utf_32(c1, c2, c3, c4);
}
- (*i_ungetc)(EOF, f);
+ goto finished;
}
else if (iconv == w_iconv16) {
while ((c1 = (*i_getc)(f)) != EOF &&
nkf_iconv_utf_16(c1, c2, c3, c4);
}
}
- (*i_ungetc)(EOF, f);
+ goto finished;
}
#endif
if (input_mode == JIS_X_0208 && DEL <= c1 && c1 < 0x92) {
/* CP5022x */
MORE;
+ }else if (input_codename && input_codename[0] == 'I' &&
+ 0xA1 <= c1 && c1 <= 0xDF) {
+ /* JIS X 0201 Katakana in 8bit JIS */
+ c2 = JIS_X_0201_1976_K;
+ c1 &= 0x7f;
+ SEND;
} else if (c1 > DEL) {
/* 8 bit code */
if (!estab_f && !iso8859_f) {
SKIP;
} else if (c1 == ESC && (!is_8bit || mime_decode_mode)) {
if ((c1 = (*i_getc)(f)) == EOF) {
- /* (*oconv)(0, ESC); don't send bogus code */
+ (*oconv)(0, ESC);
LAST;
}
else if (c1 == '&') {
} else if (c1 == ESC && iconv == s_iconv) {
/* ESC in Shift_JIS */
if ((c1 = (*i_getc)(f)) == EOF) {
- /* (*oconv)(0, ESC); don't send bogus code */
+ (*oconv)(0, ESC);
LAST;
} else if (c1 == '$') {
/* J-PHONE emoji */
/* goto next_word */
}
+finished:
/* epilogue */
(*iconv)(EOF, 0, 0);
if (!input_codename)
output_endian = ENDIAN_LITTLE;
} else if (cp[0] == 'B') {
cp++;
- } else {
- output_encoding = nkf_enc_from_index(enc_idx);
- continue;
}
if (cp[0] == '0'){
cp++;
bit:3 Convert HTML Entity
bit:4 Convert JIS X 0208 Katakana to JIS X 0201 Katakana
*/
- while ('0'<= *cp && *cp <='9') {
+ while ('0'<= *cp && *cp <='4') {
alpha_f |= 1 << (*cp++ - '0');
}
- if (alpha_f & ((1 << 2) | (1 << 3))) alpha_f |= 1;
- if (!alpha_f) alpha_f = 1;
+ alpha_f |= 1;
continue;
case 'x': /* Convert X0201 kana to X0208 or X0201 Conversion */
x0201_f = FALSE; /* No X0201->X0208 conversion */