* \e$B8=:_!"\e(Bnkf \e$B$O\e(B SorceForge \e$B$K$F%a%s%F%J%s%9$,B3$1$i$l$F$$$^$9!#\e(B
* http://sourceforge.jp/projects/nkf/
***********************************************************************/
-/* $Id: nkf.c,v 1.140 2007/10/02 08:37:46 naruse Exp $ */
+/* $Id: nkf.c,v 1.149 2007/11/18 12:05:18 naruse Exp $ */
#define NKF_VERSION "2.0.8"
-#define NKF_RELEASE_DATE "2007-10-02"
+#define NKF_RELEASE_DATE "2007-11-18"
#define COPY_RIGHT \
"Copyright (C) 1987, FUJITSU LTD. (I.Ichikawa),2000 S. Kono, COW\n" \
"Copyright (C) 2002-2007 Kono, Furukawa, Naruse, mastodon"
#include "config.h"
#include "utf8tbl.h"
+
+#ifndef MIME_DECODE_DEFAULT
+#define MIME_DECODE_DEFAULT STRICT_MIME
+#endif
+
#if (defined(__TURBOC__) || defined(_MSC_VER) || defined(LSI_C) || defined(__MINGW32__) || defined(__EMX__) || defined(__MSDOS__) || defined(__WINDOWS__) || defined(__DOS__) || defined(__OS2__)) && !defined(MSDOS)
#define MSDOS
#if (defined(__Win32__) || defined(_WIN32)) && !defined(__WIN32__)
('a'<=c&&c<='f') ? (c-'a'+10) : 0)
#define bin2hex(c) ("0123456789ABCDEF"[c&15])
#define is_eucg3(c2) (((unsigned short)c2 >> 8) == SS3)
+#define nkf_noescape_mime(c) ((c == CR) || (c == LF) || \
+ ((c > SP) && (c < DEL) && (c != '?') && (c != '=') && (c != '_') \
+ && (c != '(') && (c != ')') && (c != '.') && (c != 0x22)))
#define CP932_TABLE_BEGIN 0xFA
#define CP932_TABLE_END 0xFC
static void version(void);
#endif
static void options(unsigned char *c);
-#if defined(PERL_XS) || defined(WIN32DLL)
static void reinit(void);
-#endif
/* buffers */
static int hira_f = FALSE; /* hira/kata henkan */
static int input_f = FALSE; /* non fixed input code */
static int alpha_f = FALSE; /* convert JIx0208 alphbet to ASCII */
-static int mime_f = STRICT_MIME; /* convert MIME B base64 or Q */
+static int mime_f = MIME_DECODE_DEFAULT; /* convert MIME B base64 or Q */
static int mime_decode_f = FALSE; /* mime decode is explicitly on */
static int mimebuf_f = FALSE; /* MIME buffered input */
static int broken_f = FALSE; /* convert ESC-less broken JIS */
#ifdef CHECK_OPTION
static int noout_f = FALSE;
static void no_putc(nkf_char c);
-static nkf_char debug_f = FALSE;
+static int debug_f = FALSE;
static void debug(const char *str);
static nkf_char (*iconv_for_check)(nkf_char c2,nkf_char c1,nkf_char c0) = 0;
#endif
-static int guess_f = FALSE;
+static int guess_f = 0; /* 0: OFF, 1: ON, 2: VERBOSE */
#if !defined PERL_XS
static void print_guessed_code(char *filename);
#endif
{0}
};
-static int mimeout_mode = 0;
+static int mimeout_mode = 0; /* 0, -1, 'Q', 'B', 1, 2 */
static int base64_count = 0;
/* X0208 -> ASCII converter */
#endif
static int nlmode_f = 0; /* CR, LF, CRLF */
-static int input_nextline = 0; /* 0: unestablished, EOF: MIXED */
+static int input_newline = 0; /* 0: unestablished, EOF: MIXED */
static nkf_char prev_cr = 0; /* CR or 0 */
#ifdef EASYWIN /*Easy Win */
static int end_check;
for (argc--,argv++; (argc > 0) && **argv == '-'; argc--, argv++) {
cp = (unsigned char *)*argv;
options(cp);
+ if (guess_f) {
+#ifdef CHECK_OPTION
+ int debug_f_back = debug_f;
+#endif
+#ifdef EXEC_IO
+ int exec_f_back = exec_f;
+#endif
+#ifdef X0212_ENABLE
+ int x0212_f_back = x0212_f;
+#endif
+#ifdef X0212_ENABLE
+ int x0213_f_back = x0213_f;
+#endif
+ int guess_f_back = guess_f;
+ reinit();
+ guess_f = guess_f_back;
+ mime_f = FALSE;
+#ifdef CHECK_OPTION
+ debug_f = debug_f_back;
+#endif
+#ifdef EXEC_IO
+ exec_f = exec_f_back;
+#endif
+#ifdef X0212_ENABLE
+ x0212_f = x0212_f_back;
+#endif
+#ifdef X0213_ENABLE
+ x0213_f = x0213_f_back;
+#endif
+ }
#ifdef EXEC_IO
if (exec_f){
int fds[2], pid;
int is_argument_error = FALSE;
while (argc--) {
input_codename = NULL;
- input_nextline = NULL;
+ input_newline = 0;
#ifdef CHECK_OPTION
iconv_for_check = 0;
#endif
{"hiragana","h1"},
{"katakana","h2"},
{"katakana-hiragana","h3"},
+ {"guess=", ""},
{"guess", "g"},
{"cp932", ""},
{"no-cp932", ""},
}
p = 0;
}
- if (p == 0) return;
+ if (p == 0) {
+ fprintf(stderr, "unknown long option: --%s\n", cp);
+ return;
+ }
while(*cp && *cp != SP && cp++);
if (long_option[i].alias[0]){
cp_back = cp;
input_f = UTF32_INPUT;
input_endian = ENDIAN_LITTLE;
#endif
+ } else {
+ fprintf(stderr, "unknown input encoding: %s\n", codeset);
}
continue;
}
output_endian = ENDIAN_LITTLE;
output_bom_f = TRUE;
#endif
+ } else {
+ fprintf(stderr, "unknown output encoding: %s\n", codeset);
}
continue;
}
+ if (strcmp(long_option[i].name, "guess=") == 0){
+ if (p[0] == '1') {
+ guess_f = 2;
+ } else {
+ guess_f = 1;
+ }
+ continue;
+ }
#ifdef OVERWRITE
if (strcmp(long_option[i].name, "overwrite") == 0){
file_out_f = TRUE;
nlmode_f = 0; cp++;
}
continue;
- case 'g':
#ifndef PERL_XS
- guess_f = TRUE;
-#endif
+ case 'g':
+ if (*cp == '1') {
+ guess_f = 2;
+ cp++;
+ } else if (*cp == '0') {
+ guess_f = 1;
+ cp++;
+ } else {
+ guess_f = 1;
+ }
continue;
+#endif
case SP:
/* module muliple options in a string are allowed for Perl moudle */
while(*cp && *cp++!='-');
continue;
default:
+ fprintf(stderr, "unknown option: -%c\n", *(cp-1));
/* bogus option but ignored */
continue;
}
#define SCORE_L2 (1) /* \e$BBh\e(B2\e$B?e=`4A;z\e(B */
#define SCORE_KANA (SCORE_L2 << 1) /* \e$B$$$o$f$kH>3Q%+%J\e(B */
#define SCORE_DEPEND (SCORE_KANA << 1) /* \e$B5!<o0MB8J8;z\e(B */
-#ifdef SHIFTJIS_CP932
-#define SCORE_CP932 (SCORE_DEPEND << 1) /* CP932 \e$B$K$h$kFI$_49$(\e(B */
-#define SCORE_NO_EXIST (SCORE_CP932 << 1) /* \e$BB8:_$7$J$$J8;z\e(B */
-#else
-#define SCORE_NO_EXIST (SCORE_DEPEND << 1) /* \e$BB8:_$7$J$$J8;z\e(B */
-#endif
+#define SCORE_CP932 (SCORE_DEPEND << 1) /* CP932 \e$B$K$h$kFI$_49$(\e(B (IBM extended characters) */
+#define SCORE_X0212 (SCORE_CP932 << 1) /* JIS X 0212 */
+#define SCORE_NO_EXIST (SCORE_X0212 << 1) /* \e$BB8:_$7$J$$J8;z\e(B */
#define SCORE_iMIME (SCORE_NO_EXIST << 1) /* MIME \e$B$K$h$k;XDj\e(B */
#define SCORE_ERROR (SCORE_iMIME << 1) /* \e$B%(%i!<\e(B */
static const char score_table_F0[] = {
SCORE_L2, SCORE_L2, SCORE_L2, SCORE_L2,
SCORE_L2, SCORE_DEPEND, SCORE_NO_EXIST, SCORE_NO_EXIST,
- SCORE_DEPEND, SCORE_DEPEND, SCORE_DEPEND, SCORE_DEPEND,
- SCORE_DEPEND, SCORE_NO_EXIST, SCORE_NO_EXIST, SCORE_ERROR,
+ SCORE_DEPEND, SCORE_DEPEND, SCORE_CP932, SCORE_CP932,
+ SCORE_CP932, SCORE_NO_EXIST, SCORE_NO_EXIST, SCORE_ERROR,
};
void set_code_score(struct input_code *ptr, nkf_char score)
set_code_score(ptr, SCORE_ERROR);
}else if (c2 == SSO){
set_code_score(ptr, SCORE_KANA);
+ }else if (c2 == 0x8f){
+ set_code_score(ptr, SCORE_X0212);
#ifdef UTF8_OUTPUT_ENABLE
}else if (!e2w_conv(c2, c1)){
set_code_score(ptr, SCORE_NO_EXIST);
status_push_ch(ptr, c);
code_score(ptr);
status_clear(ptr);
- }else if ((0x81 <= c && c < 0xa0) || (0xe0 <= c && c <= 0xef)){
+ }else if ((0x81 <= c && c < 0xa0) || (0xe0 <= c && c <= 0xea)){
ptr->stat = 1;
status_push_ch(ptr, c);
+ }else if (0xed <= c && c <= 0xee){
+ ptr->stat = 3;
+ status_push_ch(ptr, c);
#ifdef SHIFTJIS_CP932
- }else if (cp51932_f
- && is_ibmext_in_sjis(c)){
+ }else if (is_ibmext_in_sjis(c)){
ptr->stat = 2;
status_push_ch(ptr, c);
#endif /* SHIFTJIS_CP932 */
#ifdef X0212_ENABLE
- }else if (x0212_f && 0xf0 <= c && c <= 0xfc){
+ }else if (0xf0 <= c && c <= 0xfc){
ptr->stat = 1;
status_push_ch(ptr, c);
#endif /* X0212_ENABLE */
break;
case 2:
#ifdef SHIFTJIS_CP932
+ if ((0x40 <= c && c <= 0x7e) || (0x80 <= c && c <= 0xfc)) {
+ status_push_ch(ptr, c);
+ if (s2e_conv(ptr->buf[0], ptr->buf[1], &ptr->buf[0], &ptr->buf[1]) == 0) {
+ set_code_score(ptr, SCORE_CP932);
+ status_clear(ptr);
+ break;
+ }
+ }
+#endif /* SHIFTJIS_CP932 */
+ status_disable(ptr);
+ break;
+ case 3:
if ((0x40 <= c && c <= 0x7e) || (0x80 <= c && c <= 0xfc)){
status_push_ch(ptr, c);
- if (s2e_conv(ptr->buf[0], ptr->buf[1], &ptr->buf[0], &ptr->buf[1]) == 0){
- set_code_score(ptr, SCORE_CP932);
- status_clear(ptr);
- break;
- }
+ s2e_conv(ptr->buf[0], ptr->buf[1], &ptr->buf[0], &ptr->buf[1]);
+ set_code_score(ptr, SCORE_CP932);
+ status_clear(ptr);
+ }else{
+ status_disable(ptr);
}
-#endif /* SHIFTJIS_CP932 */
-#ifndef X0212_ENABLE
- status_disable(ptr);
-#endif
break;
}
}
output_mode = ASCII;
(*o_putc)(c1);
} else if (c2 == ISO8859_1) {
- output_mode = ISO8859_1;
+ output_mode = UTF8;
(*o_putc)(c1 | 0x080);
} else {
output_mode = UTF8;
void nl_conv(nkf_char c2, nkf_char c1)
{
- if (guess_f && input_nextline != EOF) {
+ if (guess_f && input_newline != EOF) {
if (c2 == 0 && c1 == LF) {
- if (!input_nextline) input_nextline = prev_cr ? CRLF : LF;
- else if (input_nextline != (prev_cr ? CRLF : LF)) input_nextline = EOF;
- } else if (c2 == 0 && c1 == CR && input_nextline == LF) input_nextline = EOF;
+ if (!input_newline) input_newline = prev_cr ? CRLF : LF;
+ else if (input_newline != (prev_cr ? CRLF : LF)) input_newline = EOF;
+ } else if (c2 == 0 && c1 == CR && input_newline == LF) input_newline = EOF;
else if (!prev_cr);
- else if (!input_nextline) input_nextline = CR;
- else if (input_nextline != CR) input_nextline = EOF;
+ else if (!input_newline) input_newline = CR;
+ else if (input_newline != CR) input_newline = EOF;
}
if (prev_cr || c2 == 0 && c1 == LF) {
prev_cr = 0;
if (input_codename && !*input_codename) {
printf("BINARY\n");
} else {
- printf("%s%s\n",
- (input_codename ? input_codename : "ASCII"),
- input_nextline == CR ? " (CR)" :
- input_nextline == LF ? " (LF)" :
- input_nextline == CRLF ? " (CRLF)" :
- input_nextline == EOF ? " (MIXED NL)" :
- "");
+ struct input_code *p = find_inputcode_byfunc(iconv);
+ if (guess_f == 1) {
+ printf("%s\n", input_codename ? input_codename : "ASCII");
+ } else {
+ if (!input_codename) {
+ input_codename = "ASCII";
+ } else if (strcmp(input_codename, "Shift_JIS") == 0) {
+ if (p->score & (SCORE_DEPEND|SCORE_CP932))
+ input_codename = "CP932";
+ } else if (strcmp(input_codename, "EUC-JP") == 0) {
+ if (p->score & (SCORE_X0212))
+ input_codename = "EUCJP-MS";
+ else if (p->score & (SCORE_DEPEND|SCORE_CP932))
+ input_codename = "CP51932";
+ } else if (strcmp(input_codename, "ISO-2022-JP") == 0) {
+ if (p->score & (SCORE_KANA))
+ input_codename = "CP50221";
+ else if (p->score & (SCORE_DEPEND|SCORE_CP932))
+ input_codename = "CP50220";
+ }
+ printf("%s%s\n",
+ input_codename,
+ input_newline == CR ? " (CR)" :
+ input_newline == LF ? " (LF)" :
+ input_newline == CRLF ? " (CRLF)" :
+ input_newline == EOF ? " (MIXED NL)" :
+ "");
+ }
}
}
#endif /*WIN32DLL*/
if (c > '@') {
if (c < '[') {
i = c - 'A'; /* A..Z 0-25 */
+ } else if (c == '_') {
+ i = '?' /* 63 */ ; /* _ 63 */
} else {
i = c - 'G' /* - 'a' + 26 */ ; /* a..z 26-51 */
}
} else if (c > '/') {
i = c - '0' + '4' /* - '0' + 52 */ ; /* 0..9 52-61 */
- } else if (c == '+') {
- i = '>' /* 62 */ ; /* + 62 */
+ } else if (c == '+' || c == '-') {
+ i = '>' /* 62 */ ; /* + and - 62 */
} else {
i = '?' /* 63 */ ; /* / 63 */
}
#define MIMEOUT_BUF_LENGTH (60)
char mimeout_buf[MIMEOUT_BUF_LENGTH+1];
int mimeout_buf_count = 0;
-int mimeout_preserve_space = 0;
-#define itoh4(c) (c>=10?c+'A'-10:c+'0')
void open_mime(nkf_char mode)
{
}
}
mimeout_mode = mime_encode_method[i];
-
i = 0;
if (base64_count>45) {
if (mimeout_buf_count>0 && nkf_isblank(mimeout_buf[i])){
(*o_mputc)(LF);
(*o_mputc)(SP);
base64_count = 1;
- if (!mimeout_preserve_space && mimeout_buf_count>0
+ if (mimeout_buf_count>0
&& (mimeout_buf[i]==SP || mimeout_buf[i]==TAB
|| mimeout_buf[i]==CR || mimeout_buf[i]==LF)) {
i++;
}
}
- if (!mimeout_preserve_space) {
- for (;i<mimeout_buf_count;i++) {
- if (mimeout_buf[i]==SP || mimeout_buf[i]==TAB
- || mimeout_buf[i]==CR || mimeout_buf[i]==LF) {
- (*o_mputc)(mimeout_buf[i]);
- base64_count ++;
- } else {
- break;
- }
+ for (;i<mimeout_buf_count;i++) {
+ if (mimeout_buf[i]==SP || mimeout_buf[i]==TAB
+ || mimeout_buf[i]==CR || mimeout_buf[i]==LF) {
+ (*o_mputc)(mimeout_buf[i]);
+ base64_count ++;
+ } else {
+ break;
}
}
- mimeout_preserve_space = FALSE;
-
while(*p) {
(*o_mputc)(*p++);
base64_count ++;
base64_count += 2;
break;
}
- if (mimeout_mode) {
+ if (mimeout_mode > 0) {
if (mimeout_f!=FIXED_MIME) {
close_mime();
} else if (mimeout_mode != 'Q')
base64_count = 0;
} else if(!nkf_isalnum(c)) {
(*o_mputc)('=');
- (*o_mputc)(itoh4(((c>>4)&0xf)));
- (*o_mputc)(itoh4((c&0xf)));
+ (*o_mputc)(bin2hex(((c>>4)&0xf)));
+ (*o_mputc)(bin2hex((c&0xf)));
base64_count += 3;
} else {
(*o_mputc)(c);
void mime_prechar(nkf_char c2, nkf_char c1)
{
- if (mimeout_mode){
+ if (mimeout_mode > 0){
if (c2 == EOF){
if (base64_count + mimeout_buf_count/3*4> 73){
(*o_base64conv)(EOF,0);
(*o_base64conv)(0,LF);
(*o_base64conv)(0,SP);
+ base64_count = 1;
}
- } else if (c2){
- if (base64_count + mimeout_buf_count/3*4> 66){
+ } else {
+ if (base64_count + mimeout_buf_count/3*4> 66) {
(*o_base64conv)(EOF,0);
(*o_base64conv)(0,LF);
(*o_base64conv)(0,SP);
+ base64_count = 1;
+ mimeout_mode = -1;
}
- }/*else if (mime_lastchar2){
- if (c1 <=DEL && !nkf_isspace(c1)){
- (*o_base64conv)(0,SP);
- }
- }*/
- }/*else{
- if (c2 && mime_lastchar2 == 0
- && mime_lastchar1 && !nkf_isspace(mime_lastchar1)){
- (*o_base64conv)(0,SP);
}
- }*/
- /*mime_lastchar2 = c2;
- mime_lastchar1 = c1;*/
+ } else if (c2) {
+ if (c2 != EOF && base64_count + mimeout_buf_count/3*4> 60) {
+ mimeout_mode = (output_mode==ASCII ||output_mode == ISO8859_1) ? 'Q' : 'B';
+ open_mime(output_mode);
+ (*o_base64conv)(EOF,0);
+ (*o_base64conv)(0,LF);
+ (*o_base64conv)(0,SP);
+ base64_count = 1;
+ mimeout_mode = -1;
+ }
+ }
}
void mime_putc(nkf_char c)
/* mimeout_f != FIXED_MIME */
if (c == EOF) { /* c==EOF */
+ if (mimeout_mode == -1 && mimeout_buf_count > 1) open_mime(output_mode);
j = mimeout_buf_count;
mimeout_buf_count = 0;
i = 0;
- if (mimeout_mode) {
+ if (mimeout_mode > 0) {
if (!nkf_isblank(mimeout_buf[j-1])) {
for (;i<j;i++) {
if (nkf_isspace(mimeout_buf[i]) && base64_count < 71){
return;
}
+ if (mimeout_buf_count > 0){
+ lastchar = mimeout_buf[mimeout_buf_count - 1];
+ }else{
+ lastchar = -1;
+ }
+
if (mimeout_mode=='Q') {
if (c <= DEL && (output_mode==ASCII ||output_mode == ISO8859_1)) {
if (c == CR || c == LF) {
(*o_mputc)(SP);
base64_count++;
}
- }
- (*o_mputc)(c);
- base64_count++;
+ } else {
+ if (base64_count > 70) {
+ close_mime();
+ (*o_mputc)(LF);
+ (*o_mputc)(SP);
+ base64_count = 1;
+ open_mime(output_mode);
+ }
+ if (!nkf_noescape_mime(c)) {
+ mimeout_addchar(c);
+ return;
+ }
+ }
+ (*o_mputc)(c);
+ base64_count++;
}
return;
}
- if (mimeout_buf_count > 0){
- lastchar = mimeout_buf[mimeout_buf_count - 1];
- }else{
- lastchar = -1;
- }
-
- if (!mimeout_mode) {
+ if (mimeout_mode <= 0) {
if (c <= DEL && (output_mode==ASCII ||output_mode == ISO8859_1)) {
if (nkf_isspace(c)) {
+ int flag = 0;
+ if (mimeout_mode == -1) {
+ flag = 1;
+ }
if (c==CR || c==LF) {
- base64_count=0;
+ if (flag) {
+ open_mime(output_mode);
+ output_mode = 0;
+ } else {
+ base64_count = 0;
+ }
}
for (i=0;i<mimeout_buf_count;i++) {
(*o_mputc)(mimeout_buf[i]);
base64_count++;
}
}
- mimeout_buf[0] = (char)c;
- mimeout_buf_count = 1;
+ if (flag) {
+ eof_mime();
+ base64_count = 0;
+ mimeout_mode = 0;
+ }
+ mimeout_buf[0] = (char)c;
+ mimeout_buf_count = 1;
}else{
if (base64_count > 1
&& base64_count + mimeout_buf_count > 76
base64_count = 0;
mimeout_buf_count = 0;
}
+ mimeout_buf[mimeout_buf_count++] = (char)c;
+ return;
}
if (c==SP || c==TAB || c==CR || c==LF) {
for (i=0;i<mimeout_buf_count;i++) {
mimeout_buf_count = 0;
}
return;
- }
+ }
if (mimeout_buf_count>0 && SP<c && c!='=') {
mimeout_buf[mimeout_buf_count++] = (char)c;
if (mimeout_buf_count>MIMEOUT_BUF_LENGTH) {
}
-#if defined(PERL_XS) || defined(WIN32DLL)
void reinit(void)
{
{
hira_f = FALSE;
input_f = FALSE;
alpha_f = FALSE;
- mime_f = STRICT_MIME;
+ mime_f = MIME_DECODE_DEFAULT;
mime_decode_f = FALSE;
mimebuf_f = FALSE;
broken_f = FALSE;
noout_f = FALSE;
debug_f = FALSE;
#endif
- guess_f = FALSE;
+ guess_f = 0;
#ifdef EXEC_IO
exec_f = 0;
#endif
mime_decode_mode = FALSE;
file_out_f = FALSE;
nlmode_f = 0;
- input_nextline = 0;
+ input_newline = 0;
prev_cr = 0;
option_mode = 0;
broken_counter = 0;
reinitdll();
#endif /*WIN32DLL*/
}
-#endif
void no_connection(nkf_char c2, nkf_char c1)
{
OSDN Git Service
