3 * @author Perry Rapp, Creator, 2003-2004
4 * @date Created: 2003-10
5 * @date Edited: 2004-05-26 (Perry Rapp)
7 * @brief Implementation of utility unicode conversion routines
11 Copyright (c) 2003 Perry Rapp
12 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
13 The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 static char THIS_FILE[] = __FILE__;
26 // This is not in older platform sdk versions
27 #ifndef WC_NO_BEST_FIT_CHARS
28 #define WC_NO_BEST_FIT_CHARS 0x00000400
34 static bool f_osvi_fetched=false;
35 static OSVERSIONINFO f_osvi;
38 * @brief fetch current OS version into file level variable & set flag
43 memset(&f_osvi, 0, sizeof(f_osvi));
44 f_osvi.dwOSVersionInfoSize = sizeof(f_osvi);
45 GetVersionEx(&f_osvi);
46 f_osvi_fetched = true;
49 static LPCTSTR f_unicodesetNames[] = { _T("<NONE>"), _T("UCS-2LE"), _T("UCS-2BE"), _T("UTF-8") };
51 * @brief return string for enum value
53 CString GetUnicodesetName(UNICODESET unicoding)
55 if (unicoding>=0 && unicoding<sizeof(f_unicodesetNames)/sizeof(f_unicodesetNames[0]))
56 return f_unicodesetNames[unicoding];
62 * @brief Convert unicode codepoint to UTF-8 byte string
64 * utf8 must be a 7+ byte buffer
65 * returns length of byte string written
66 * Does not zero-terminate!
69 Ucs4_to_Utf8(UINT unich, unsigned char * utf8)
71 #pragma warning(disable: 4244) // possible loss of data due to type conversion
74 utf8[0] = (unsigned char)unich;
77 else if (unich <= 0x7ff)
79 utf8[0] = 0xc0 + (unich >> 6);
80 utf8[1] = 0x80 + (unich & 0x3f);
83 else if (unich <= 0xffff)
85 utf8[0] = 0xe0 + (unich >> 12);
86 utf8[1] = 0x80 + ((unich >> 6) & 0x3f);
87 utf8[2] = 0x80 + (unich & 0x3f);
90 else if (unich <= 0x1fffff)
92 utf8[0] = 0xf0 + (unich >> 18);
93 utf8[1] = 0x80 + ((unich >> 12) & 0x3f);
94 utf8[2] = 0x80 + ((unich >> 6) & 0x3f);
95 utf8[3] = 0x80 + (unich & 0x3f);
98 else if (unich <= 0x3ffffff)
100 utf8[0] = 0xf8 + (unich >> 24);
101 utf8[1] = 0x80 + ((unich >> 18) & 0x3f);
102 utf8[2] = 0x80 + ((unich >> 12) & 0x3f);
103 utf8[3] = 0x80 + ((unich >> 6) & 0x3f);
104 utf8[4] = 0x80 + (unich & 0x3f);
107 else if (unich <= 0x7fffffff)
109 utf8[0] = 0xfc + (unich >> 30);
110 utf8[1] = 0x80 + ((unich >> 24) & 0x3f);
111 utf8[2] = 0x80 + ((unich >> 18) & 0x3f);
112 utf8[3] = 0x80 + ((unich >> 12) & 0x3f);
113 utf8[4] = 0x80 + ((unich >> 6) & 0x3f);
114 utf8[5] = 0x80 + (unich & 0x3f);
119 // Invalid Unicode codepoint (high bit was set)
120 // TODO: What do we do ?
124 #pragma warning(default: 4244) // possible loss of data due to type conversion
128 * @brief return byte length of UTF-8 character from its initial character (-1 if invalid)
131 Utf8len_fromLeadByte(unsigned char ch)
133 if (ch < 0x80) return 1;
134 if (ch < 0xC0) return -1;
135 if (ch < 0xE0) return 2;
136 if (ch < 0xF0) return 3;
137 if (ch < 0xF8) return 4;
138 if (ch < 0xFC) return 5;
139 if (ch < 0xFE) return 6;
144 * @brief return #bytes required to represent Unicode codepoint as UTF-8
147 Utf8len_fromCodepoint(UINT ch)
149 if (ch <= 0x7F) return 1;
150 if (ch <= 0x7FF) return 2;
151 if (ch <= 0xFFFF) return 3;
152 if (ch <= 0x1FFFFF) return 4;
153 if (ch <= 0x3FFFFFF) return 5;
154 if (ch <= 0x7FFFFFFF) return 6;
159 * @brief How many bytes will it take to write string as UTF-8 ?
161 * @param size size argument as filemapping are not 0 terminated
163 * @bug Fails for files larger than 2gigs
166 Utf8len_of_string(LPCWSTR text, int size)
169 for (int i=0; i<size; ++i)
171 int chlen = Utf8len_fromCodepoint(text[i]);
172 if (chlen < 1) chlen = 1;
178 * @brief How many chars in this UTF-8 string ?
180 * @param size size argument as filemapping are not 0 terminated
182 * @bug Fails for files larger than 2gigs
185 stringlen_of_utf8(LPCSTR text, int size)
188 for (int i=0; i<size; )
190 int chlen = Utf8len_fromLeadByte(text[i]);
191 if (chlen < 1) chlen = 1;
199 * @brief Read UTF-8 character and return as Unicode
202 GetUtf8Char(unsigned char * str)
204 /* test short cases first, as probably much more common */
205 if (!(*str & 0x80 && *str & 0x40)) {
208 if (!(*str & 0x20)) {
209 UINT ch = ((str[0] & 0x1F) << 6)
213 if (!(*str & 0x10)) {
214 UINT ch = ((str[0] & 0x0f) << 12)
215 + ((str[1] & 0x3F) << 6)
219 if (!(*str & 0x08)) {
220 UINT ch = ((str[0] & 0x0F) << 18)
221 + ((str[1] & 0x3F) << 12)
222 + ((str[2] & 0x3F) << 6)
226 if (!(*str & 0x04)) {
227 UINT ch = ((str[0] & 0x0F) << 24)
228 + ((str[1] & 0x3F) << 18)
229 + ((str[2] & 0x3F) << 12)
230 + ((str[3] & 0x3F) << 6)
234 UINT ch = ((str[0] & 0x0F) << 30)
235 + ((str[1] & 0x3F) << 24)
236 + ((str[2] & 0x3F) << 18)
237 + ((str[3] & 0x3F) << 12)
238 + ((str[4] & 0x3F) << 6)
245 * @brief Write unicode codepoint u out as UTF-8 to lpd, and advance lpd
247 * Returns number of bytes written (or -1 for error, in which case it writes '?')
249 int to_utf8_advance(UINT u, unsigned char * &lpd)
251 #pragma warning(disable: 4244) // possible loss of data due to type conversion
259 *lpd++ = 0xC0 + (u >> 6);
260 *lpd++ = 0x80 + (u & 0x3F);
263 else if (u < 0x10000)
265 *lpd++ = 0xE0 + (u >> 12);
266 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
267 *lpd++ = 0x80 + (u & 0x3F);
270 else if (u < 0x200000)
272 *lpd++ = 0xF0 + (u >> 18);
273 *lpd++ = 0x80 + ((u >> 12) & 0x3F);
274 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
275 *lpd++ = 0x80 + (u & 0x3F);
278 else if (u < 0x4000000)
280 *lpd++ = 0xF8 + (u >> 24);
281 *lpd++ = 0x80 + ((u >> 18) & 0x3F);
282 *lpd++ = 0x80 + ((u >> 12) & 0x3F);
283 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
284 *lpd++ = 0x80 + (u & 0x3F);
287 else if (u < 0x80000000)
289 *lpd++ = 0xF8 + (u >> 30);
290 *lpd++ = 0x80 + ((u >> 24) & 0x3F);
291 *lpd++ = 0x80 + ((u >> 18) & 0x3F);
292 *lpd++ = 0x80 + ((u >> 12) & 0x3F);
293 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
294 *lpd++ = 0x80 + (u & 0x3F);
302 #pragma warning(default: 4244) // possible loss of data due to type conversion
306 * @brief convert character passed (Unicode codepoint) to a TCHAR (set lossy flag if imperfect conversion)
308 CString maketchar(UINT unich, bool & lossy)
310 static bool vercheck=false;
311 static UINT codepage = CP_ACP;
314 if (!f_osvi_fetched) fetch_verinfo();
315 // Need 2000 or better for CP_THREAD_ACP
316 if (f_osvi.dwMajorVersion>=5)
317 codepage = CP_THREAD_ACP;
321 return maketchar(unich, lossy, codepage);
325 * @brief convert character passed (Unicode codepoint) to a TCHAR (set lossy flag if imperfect conversion)
327 CString maketchar(UINT unich, bool & lossy, UINT codepage)
341 CString s = (TCHAR)unich;
344 wchar_t wch = (wchar_t)unich;
347 static bool vercheck=false;
348 static bool has_no_best_fit=false;
351 if (!f_osvi_fetched) fetch_verinfo();
352 // Need 2000 (5.x) or 98 (4.10)
353 has_no_best_fit = f_osvi.dwMajorVersion>=5 || (f_osvi.dwMajorVersion==4 && f_osvi.dwMinorVersion>=10);
356 // So far it isn't lossy, so try for lossless conversion
358 BOOL defaulted=FALSE;
359 DWORD flags = has_no_best_fit ? WC_NO_BEST_FIT_CHARS : 0;
360 if (WideCharToMultiByte(codepage, flags, &wch, 1, &outch, 1, NULL, &defaulted)
368 // already lossy, so make our best shot
369 DWORD flags = WC_COMPOSITECHECK+WC_DISCARDNS+WC_SEPCHARS+WC_DEFAULTCHAR;
371 int n = WideCharToMultiByte(codepage, flags, &wch, 1, outbuff, sizeof(outbuff)-1, NULL, NULL);
382 * @brief convert 8-bit character input to Unicode codepoint and return it
385 byteToUnicode (unsigned char ch)
387 static bool vercheck=false;
388 static UINT codepage = CP_ACP;
391 if (!f_osvi_fetched) fetch_verinfo();
392 // Need 2000 or better for CP_THREAD_ACP
393 if (f_osvi.dwMajorVersion>=5)
394 codepage = CP_THREAD_ACP;
397 return byteToUnicode(ch, codepage);
401 * @brief convert 8-bit character input to Unicode codepoint and return it
404 byteToUnicode (unsigned char ch, UINT codepage)
412 int n = MultiByteToWideChar(codepage, flags, (LPCSTR)&ch, 1, &wbuff, 1);
420 * @brief Get appropriate default codepage for this operating system
425 // default codepage is CP_THREAD_ACP if available, else CP_ACP
426 static bool vercheck=false;
427 static int defcodepage = CP_ACP;
430 if (!f_osvi_fetched) fetch_verinfo();
431 // Need 2000 or better for CP_THREAD_ACP
432 if (f_osvi.dwMajorVersion>=5)
433 defcodepage = CP_THREAD_ACP;
439 void getDefaultEncoding(UNICODESET * unicoding, int * codepage)
446 *codepage = getDefaultCodepage();
451 * @brief Write appropriate BOM (Unicode byte order marker)
452 * returns #bytes written
455 writeBom(LPVOID dest, UNICODESET unicoding)
457 unsigned char * lpd = reinterpret_cast<unsigned char *>(dest);
458 // write Unicode byte order marker (BOM)
459 if (unicoding == UCS2LE)
465 else if (unicoding == UCS2BE)
471 else if (unicoding == UTF8)
482 * @brief Write src string (TCHAR) to destination buffer in codeset specified.
484 * In ANSI build, source string is assumed to be in defcodepage.
485 * This is designed for use writing to memory-mapped file.
486 * NB: Destination is not zero-terminated!
489 convertToBuffer(const CString & src, LPVOID dest, UNICODESET unicoding, int codepage)
491 static int defcodepage = getDefaultCodepage();
493 unsigned char * lpd = reinterpret_cast<unsigned char *>(dest);
494 unsigned char * start = lpd;
497 if (unicoding == UCS2LE)
499 int nbytes = src.GetLength() * 2;
500 CopyMemory(lpd, src, nbytes);
503 else if (unicoding == UCS2BE)
505 for (int i=0; i<src.GetLength(); ++i)
508 *lpd++ = (unsigned char)(u >> 8);
509 *lpd++ = (unsigned char)(u & 0xFF);
513 else if (unicoding == UTF8)
515 for (int i=0; i<src.GetLength(); ++i)
518 ucr::to_utf8_advance(u, lpd);
524 ASSERT(unicoding == NONE); // there aren't any other values in UNICODESET
525 // Write wchars to chars
527 // Take a swag at the maximum length :(
528 int maxlen = src.GetLength() * 3;
530 int nbytes = WideCharToMultiByte(codepage, flags, src, src.GetLength(), (char *)lpd, maxlen, NULL, &replaced);
531 // TODO: Ought to output replaced flag to caller
536 // ANSI build, TCHAR=char
538 if (unicoding == UCS2LE)
540 for (int i=0; i<src.GetLength(); ++i)
543 *lpd++ = (unsigned char)u;
548 else if (unicoding == UCS2BE)
550 for (int i=0; i<src.GetLength(); ++i)
554 *lpd++ = (unsigned char)u;
558 else if (unicoding == UTF8)
560 for (int i=0; i<src.GetLength(); ++i)
563 to_utf8_advance(u, lpd);
569 ASSERT(unicoding == NONE); // there aren't any other values in UNICODESET
571 if (codepage == defcodepage)
573 // trivial case, string is already in the correct codepage
574 CopyMemory(lpd, (LPCTSTR)src, src.GetLength());
575 return src.GetLength();
578 // Take a swag at the maximum length :(
579 int maxlen = src.GetLength() * 3;
580 int nbytes = CrossConvert(src, src.GetLength(), (LPSTR)lpd, maxlen, defcodepage, codepage, &lossy);
587 * @brief Extract character from pointer, handling UCS-2 codesets (doesn't handle UTF-8)
590 get_unicode_char(unsigned char * ptr, UNICODESET codeset, int codepage)
599 ch = (ptr[0] << 8) + ptr[1];
602 // TODO: How do we recognize valid codepage ?
603 // if not, use byteToUnicode(*ptr)
604 ch = byteToUnicode(*ptr, codepage);
610 * @brief Convert series of bytes (8-bit chars) to TCHARs, using specified codepage
612 * TODO: This doesn't inform the caller whether translation was lossy
613 * In fact, this doesn't even know. Probably going to have to make
614 * two passes, the first with MB_ERR_INVALID_CHARS. Ugh. :(
616 CString maketstring(LPCSTR lpd, UINT len, int codepage, bool * lossy)
618 static int defcodepage = getDefaultCodepage();
620 if (!len) return _T("");
622 // NO ! codepage = 0 is the value of CP_ACP !
624 // codepage = defcodepage;
627 // Convert input to Unicode, using specified codepage
628 // TCHAR is wchar_t, so convert into CString (str)
632 LPWSTR wbuff = str.GetBuffer(wlen);
633 int n = MultiByteToWideChar(codepage, flags, (LPCSTR)lpd, len, wbuff, wlen-1);
636 str.ReleaseBuffer(n);
645 if (EqualCodepages(codepage, ucr::getDefaultCodepage()))
647 // trivial case, they want the bytes in the file interpreted in our current codepage
648 // Only caveat is that input (lpd) is not zero-terminated
650 LPTSTR strbuff = str.GetBuffer(len+1);
651 _tcsncpy(strbuff, lpd, len);
652 strbuff[len] = 0; // Cannot call str.SetAt(...) until after ReleaseBuffer call
657 CString str = CrossConvertToStringA(lpd, len, codepage, defcodepage, lossy);
663 * @brief (ANSI build only) Convert from one 8 bit codepage to another
667 CrossConvertToStringA(LPCSTR src, UINT srclen, int cpin, int cpout, bool * lossy)
671 int wlen = srclen*2+6;
672 int clen = wlen * 2 + 6;
673 LPSTR cbuff = str.GetBuffer(clen);
674 int nbytes = CrossConvert(src, srclen, cbuff, clen, cpin, cpout, lossy);
675 str.ReleaseBuffer(nbytes);
681 * @brief Convert from one 8-bit codepage to another
683 * destsize must be at least 2
686 CrossConvert(LPCSTR src, UINT srclen, LPSTR dest, UINT destsize, int cpin, int cpout, bool * lossy)
688 ASSERT(destsize > 1);
690 // Convert input to Unicode, using specified codepage
692 int wlen = srclen*2+6;
693 wchar_t * wbuff = new wchar_t[wlen];
694 int n = MultiByteToWideChar(cpin, flags, (LPCSTR)src, srclen, wbuff, wlen-1);
701 wbuff[n] = 0; // zero-terminate string
703 // Now convert to TCHAR (which means defcodepage)
704 flags = WC_NO_BEST_FIT_CHARS; // TODO: Think about this
706 int clen = wlen * 2 + 6;
707 BOOL defaulted=FALSE;
708 n = WideCharToMultiByte(cpout, flags, wbuff, n, dest, destsize-1, NULL, &defaulted);
712 *lossy = !!defaulted;
716 buffer::buffer(unsigned int needed)
720 ptr = (unsigned char *)calloc(size, 1);
726 void buffer::resize(unsigned int needed)
731 ptr = (unsigned char *)realloc(ptr, size);
736 * @brief Convert from one text encoding to another; return false if any lossing conversions
738 bool convert(UNICODESET unicoding1, int codepage1, const unsigned char * src, int srcbytes, UNICODESET unicoding2, int codepage2, buffer * dest)
740 if (unicoding1 == unicoding2 && (unicoding1 || EqualCodepages(codepage1, codepage2)))
743 dest->resize(srcbytes);
744 CopyMemory(dest->ptr, src, srcbytes);
745 dest->used = srcbytes;
748 if ((unicoding1 == UCS2LE && unicoding2 == UCS2BE)
749 || (unicoding1 == UCS2BE && unicoding2 == UCS2LE))
752 dest->resize(srcbytes);
753 for (int i=0; i<srcbytes; i += 2)
755 // Byte-swap into destination
756 dest->ptr[i] = src[i+1];
757 dest->ptr[i+1] = src[i];
759 dest->used = srcbytes;
762 if (unicoding1 != UCS2LE && unicoding2 != UCS2LE)
764 // Break problem into two simpler pieces by converting through UCS-2LE
765 buffer intermed(dest->size);
766 bool step1 = convert(unicoding1, codepage1, src, srcbytes, UCS2LE, 0, &intermed);
767 bool step2 = convert(UCS2LE, 0, intermed.ptr, intermed.used, unicoding2, codepage2, dest);
768 return step1 && step2;
770 if (unicoding1 == UCS2LE)
772 // From UCS-2LE to 8-bit (or UTF-8)
774 // WideCharToMultiByte: lpDefaultChar & lpUsedDefaultChar must be NULL when using UTF-8
776 int destcp = (unicoding2 == UTF8 ? CP_UTF8 : codepage2);
778 int bytes = WideCharToMultiByte(destcp, flags, (LPCWSTR)src, srcbytes/2, 0, 0, NULL, NULL);
781 bytes = WideCharToMultiByte(destcp, flags, (LPCWSTR)src, srcbytes/2, (char *)dest->ptr, dest->size, NULL, NULL);
787 // From 8-bit (or UTF-8) to UCS-2LE
788 int srccp = (unicoding1 == UTF8 ? CP_UTF8 : codepage1);
790 int wchars = MultiByteToWideChar(srccp, flags, (LPCSTR)src, srcbytes, 0, 0);
791 dest->resize(wchars*2);
792 wchars = MultiByteToWideChar(srccp, flags, (LPCSTR)src, srcbytes, (LPWSTR)dest->ptr, dest->size/2);
793 dest->used = wchars * 2;
801 * @brief Change any special codepage constants into real codepage numbers
803 static int NormalizeCodepage(int cp)
805 if (cp == CP_THREAD_ACP) // should only happen on Win2000+
808 if (GetLocaleInfo(GetThreadLocale(), LOCALE_IDEFAULTANSICODEPAGE, buff, sizeof(buff)/sizeof(buff[0])))
811 // a valid codepage is better than no codepage
814 if (cp == CP_ACP) cp = GetACP();
815 if (cp == CP_OEMCP) cp = GetOEMCP();
820 * @brief Compare two codepages for equality
822 bool EqualCodepages(int cp1, int cp2)
825 || (NormalizeCodepage(cp1) == NormalizeCodepage(cp2));