3 * @author Perry Rapp, Creator, 2003-2006
4 * @date Created: 2003-10
5 * @date Edited: 2006-02-20 (Perry Rapp)
7 * @brief Implementation of utility unicode conversion routines
11 Copyright (c) 2003 Perry Rapp
12 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
13 The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 #include <Poco/UnicodeConverter.h>
23 #include "UnicodeString.h"
24 #include "ExConverter.h"
26 using Poco::UnicodeConverter;
31 // store the default codepage as specified by user in options
32 static int f_nDefaultCodepage = GetACP();
35 # pragma warning(push) // Saves the current warning state.
36 # pragma warning(disable:4244) // Temporarily disables warning 4244: "conversion from 'int' to 'char', possible loss of data"
38 * @brief Convert unicode codepoint to UTF-8 byte string
40 * utf8 must be a 7+ byte buffer
41 * returns length of byte string written
42 * Does not zero-terminate!
44 int Ucs4_to_Utf8(unsigned unich, unsigned char * utf8)
48 utf8[0] = (unsigned char)unich;
51 else if (unich <= 0x7ff)
53 utf8[0] = 0xc0 + (unich >> 6);
54 utf8[1] = 0x80 + (unich & 0x3f);
57 else if (unich <= 0xffff)
59 utf8[0] = 0xe0 + (unich >> 12);
60 utf8[1] = 0x80 + ((unich >> 6) & 0x3f);
61 utf8[2] = 0x80 + (unich & 0x3f);
64 else if (unich <= 0x1fffff)
66 utf8[0] = 0xf0 + (unich >> 18);
67 utf8[1] = 0x80 + ((unich >> 12) & 0x3f);
68 utf8[2] = 0x80 + ((unich >> 6) & 0x3f);
69 utf8[3] = 0x80 + (unich & 0x3f);
72 else if (unich <= 0x3ffffff)
74 utf8[0] = 0xf8 + (unich >> 24);
75 utf8[1] = 0x80 + ((unich >> 18) & 0x3f);
76 utf8[2] = 0x80 + ((unich >> 12) & 0x3f);
77 utf8[3] = 0x80 + ((unich >> 6) & 0x3f);
78 utf8[4] = 0x80 + (unich & 0x3f);
81 else if (unich <= 0x7fffffff)
83 utf8[0] = 0xfc + (unich >> 30);
84 utf8[1] = 0x80 + ((unich >> 24) & 0x3f);
85 utf8[2] = 0x80 + ((unich >> 18) & 0x3f);
86 utf8[3] = 0x80 + ((unich >> 12) & 0x3f);
87 utf8[4] = 0x80 + ((unich >> 6) & 0x3f);
88 utf8[5] = 0x80 + (unich & 0x3f);
93 // Invalid Unicode codepoint (high bit was set)
94 // TODO: What do we do ?
99 # pragma warning(pop) // Restores the warning state.
102 * @brief Gets a length of UTF-8 character in bytes.
103 * @param [in] ch The character for which to get the length.
104 * @return Byte length of UTF-8 character, -1 if invalid.
106 int Utf8len_fromLeadByte(unsigned char ch)
108 if (ch < 0x80) return 1;
109 if (ch < 0xC0) return -1;
110 if (ch < 0xE0) return 2;
111 if (ch < 0xF0) return 3;
112 if (ch < 0xF8) return 4;
113 if (ch < 0xFC) return 5;
114 if (ch < 0xFE) return 6;
119 * @brief return #bytes required to represent Unicode codepoint as UTF-8
121 int Utf8len_fromCodepoint(unsigned ch)
123 if (ch <= 0x7F) return 1;
124 if (ch <= 0x7FF) return 2;
125 if (ch <= 0xFFFF) return 3;
126 if (ch <= 0x1FFFFF) return 4;
127 if (ch <= 0x3FFFFFF) return 5;
128 if (ch <= 0x7FFFFFFF) return 6;
133 * @brief How many chars in this UTF-8 string ?
135 * @param size size argument as filemapping are not 0 terminated
137 * @bug Fails for files larger than 2gigs
139 size_t stringlen_of_utf8(const char* text, size_t size)
142 for (size_t i = 0; i < size;)
144 int chlen = Utf8len_fromLeadByte(text[i]);
145 if (chlen < 1) chlen = 1;
153 * @brief Read UTF-8 character and return as Unicode
155 unsigned GetUtf8Char(unsigned char * str)
157 /* test short cases first, as probably much more common */
158 if (!(*str & 0x80 && *str & 0x40))
164 unsigned ch = ((str[0] & 0x1F) << 6)
170 unsigned ch = ((str[0] & 0x0f) << 12)
171 + ((str[1] & 0x3F) << 6)
177 unsigned ch = ((str[0] & 0x0F) << 18)
178 + ((str[1] & 0x3F) << 12)
179 + ((str[2] & 0x3F) << 6)
185 unsigned ch = ((str[0] & 0x0F) << 24)
186 + ((str[1] & 0x3F) << 18)
187 + ((str[2] & 0x3F) << 12)
188 + ((str[3] & 0x3F) << 6)
194 unsigned ch = ((str[0] & 0x0F) << 30)
195 + ((str[1] & 0x3F) << 24)
196 + ((str[2] & 0x3F) << 18)
197 + ((str[3] & 0x3F) << 12)
198 + ((str[4] & 0x3F) << 6)
205 # pragma warning(push) // Saves the current warning state.
206 # pragma warning(disable:4244) // Temporarily disables warning 4244: "conversion from 'int' to 'char', possible loss of data"
208 * @brief Write unicode codepoint u out as UTF-8 to lpd, and advance lpd
210 * Returns number of bytes written (or -1 for error, in which case it writes '?')
212 int to_utf8_advance(unsigned u, unsigned char * &lpd)
221 *lpd++ = 0xC0 + (u >> 6);
222 *lpd++ = 0x80 + (u & 0x3F);
225 else if (u < 0x10000)
227 *lpd++ = 0xE0 + (u >> 12);
228 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
229 *lpd++ = 0x80 + (u & 0x3F);
232 else if (u < 0x200000)
234 *lpd++ = 0xF0 + (u >> 18);
235 *lpd++ = 0x80 + ((u >> 12) & 0x3F);
236 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
237 *lpd++ = 0x80 + (u & 0x3F);
240 else if (u < 0x4000000)
242 *lpd++ = 0xF8 + (u >> 24);
243 *lpd++ = 0x80 + ((u >> 18) & 0x3F);
244 *lpd++ = 0x80 + ((u >> 12) & 0x3F);
245 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
246 *lpd++ = 0x80 + (u & 0x3F);
249 else if (u < 0x80000000)
251 *lpd++ = 0xFC + (u >> 30);
252 *lpd++ = 0x80 + ((u >> 24) & 0x3F);
253 *lpd++ = 0x80 + ((u >> 18) & 0x3F);
254 *lpd++ = 0x80 + ((u >> 12) & 0x3F);
255 *lpd++ = 0x80 + ((u >> 6) & 0x3F);
256 *lpd++ = 0x80 + (u & 0x3F);
265 # pragma warning(pop) // Restores the warning state.
268 * @brief convert character passed (Unicode codepoint) to a TCHAR (set lossy flag if imperfect conversion)
270 void maketchar(String & ch, unsigned unich, bool & lossy)
272 static unsigned codepage = CP_ACP;
273 // NB: Windows always draws in CP_ACP, not CP_THREAD_ACP, so we must use CP_ACP as an internal codepage
275 maketchar(ch, unich, lossy, codepage);
279 * @brief convert character passed (Unicode codepoint) to a TCHAR (set lossy flag if imperfect conversion)
281 void maketchar(String & ch, unsigned unich, bool & lossy, unsigned codepage)
286 ch = static_cast<TCHAR>(unich);
289 else if (unich < 0x110000)
291 ch = static_cast<TCHAR>(((unich - 0x10000)/0x400 + 0xd800));
292 ch += static_cast<TCHAR>(((unich % 0x400) + 0xdc00));
304 wchar_t wch = (wchar_t)unich;
307 // So far it isn't lossy, so try for lossless conversion
309 BOOL defaulted = FALSE;
310 DWORD flags = WC_NO_BEST_FIT_CHARS;
311 if (WideCharToMultiByte(codepage, flags, &wch, 1, outch, sizeof(outch), nullptr, &defaulted)
319 // already lossy, so make our best shot
320 DWORD flags = WC_COMPOSITECHECK + WC_DISCARDNS + WC_SEPCHARS + WC_DEFAULTCHAR;
322 int n = WideCharToMultiByte(codepage, flags, &wch, 1, outbuff, sizeof(outbuff) - 1, nullptr, nullptr);
334 * @brief convert 8-bit character input to Unicode codepoint and return it
336 unsigned byteToUnicode(unsigned char ch)
338 static unsigned codepage = CP_ACP;
339 // NB: Windows always draws in CP_ACP, not CP_THREAD_ACP, so we must use CP_ACP as an internal codepage
341 return byteToUnicode(ch, codepage);
345 * @brief convert 8-bit character input to Unicode codepoint and return it
347 unsigned byteToUnicode(unsigned char ch, unsigned codepage)
355 int n = MultiByteToWideChar(codepage, flags, (const char*) & ch, 1, &wbuff, 1);
363 * @brief Return encoding used for TCHAR & String
365 void getInternalEncoding(UNICODESET * unicoding, int * codepage)
369 *codepage = CP_UCS2LE;
371 // NB: Windows always draws in CP_ACP, not CP_THREAD_ACP, so we must use CP_ACP as an internal codepage
378 * @brief Write appropriate BOM (Unicode byte order marker)
379 * returns #bytes written
381 int writeBom(void* dest, UNICODESET unicoding)
383 unsigned char * lpd = reinterpret_cast<unsigned char *>(dest);
384 // write Unicode byte order marker (BOM)
385 if (unicoding == UCS2LE)
391 else if (unicoding == UCS2BE)
397 else if (unicoding == UTF8)
407 int getBomSize(UNICODESET unicoding)
409 if (unicoding == UCS2LE)
411 else if (unicoding == UCS2BE)
413 else if (unicoding == UTF8)
419 * @brief Extract character from pointer, handling UCS-2 codesets
420 * This does not handle MBCS or UTF-8 codepages correctly!
421 * Client should not use this except for Unicode or SBCS codepages.
423 unsigned get_unicode_char(unsigned char * ptr, UNICODESET codeset, int codepage)
432 ch = (ptr[0] << 8) + ptr[1];
435 // TODO: How do we recognize valid codepage ?
436 // if not, use byteToUnicode(*ptr)
437 ch = byteToUnicode(*ptr, codepage);
443 * @brief Convert series of bytes (8-bit chars) to TCHARs.
445 * @param [out] str String returned.
446 * @param [in] lpd Original byte array to convert.
447 * @param [in] len Length of the original byte array.
448 * @param [in] codepage Codepage used.
449 * @param [out] lossy Was conversion lossy?
450 * @return true if conversion succeeds, false otherwise.
451 * @todo This doesn't inform the caller whether translation was lossy
452 * In fact, this doesn't even know. Probably going to have to make
453 * two passes, the first with MB_ERR_INVALID_CHARS. Ugh. :(
455 bool maketstring(String & str, const char* lpd, size_t len, int codepage, bool * lossy)
463 int defcodepage = getDefaultCodepage();
465 // 0 is a valid value (CP_ACP)!
467 codepage = defcodepage;
470 // Convert input to Unicode, using specified codepage
471 // TCHAR is wchar_t, so convert into String (str)
472 DWORD flags = MB_ERR_INVALID_CHARS;
473 size_t wlen = len * 2 + 6;
474 assert(wlen < INT_MAX);
480 catch (std::bad_alloc&)
482 // Not enough memory - exit
486 LPWSTR wbuff = &*str.begin();
487 if (codepage == CP_ACP || IsValidCodePage(codepage))
489 int n = MultiByteToWideChar(codepage, flags, lpd, static_cast<int>(len), wbuff, static_cast<int>(wlen - 1));
493 NB: MultiByteToWideChar is documented as only zero-terminating
494 if input was zero-terminated, but it appears that it can
495 zero-terminate even if input wasn't.
496 So we check if it zero-terminated and adjust count accordingly.
498 //>2007-01-11 jtuc: We must preserve an embedded zero even if it is
499 // the last input character. As we don't expect MultiByteToWideChar to
500 // add a zero that does not originate from the input string, it is a
501 // good idea to ASSERT that the assumption holds.
502 if (wbuff[n-1] == 0 && lpd[len-1] != 0)
512 catch (std::bad_alloc&)
514 // Not enough memory - exit
521 if (GetLastError() == ERROR_INVALID_FLAGS)
523 n = MultiByteToWideChar(codepage, 0, lpd, static_cast<int>(len), wbuff, static_cast<int>(wlen-1));
526 /* NB: MultiByteToWideChar is documented as only zero-terminating
527 if input was zero-terminated, but it appears that it can
528 zero-terminate even if input wasn't.
529 So we check if it zero-terminated and adjust count accordingly.
531 if (wbuff[n-1] == 0 && lpd[len-1] != 0)
541 catch (std::bad_alloc&)
543 // Not enough memory - exit
549 if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION)
553 // wlen & wbuff are still fine
554 n = MultiByteToWideChar(codepage, flags, lpd, static_cast<int>(len), wbuff, static_cast<int>(wlen-1));
561 catch (std::bad_alloc&)
563 // Not enough memory - exit
575 IExconverter *pexconv = Exconverter::getInstance();
576 if (pexconv != nullptr)
579 if (pexconv->convertToUnicode(codepage, lpd, &len, wbuff, &n))
585 catch (std::bad_alloc&)
587 // Not enough memory - exit
608 int dstcodepage = IsValidCodePage(defcodepage) ? defcodepage : GetACP();
610 if (EqualCodepages(codepage, dstcodepage))
612 // trivial case, they want the bytes in the file interpreted in our current codepage
613 // Only caveat is that input (lpd) is not zero-terminated
614 str = String(lpd, len);
618 if (codepage == CP_ACP || IsValidCodePage(codepage))
620 str = CrossConvertToStringA(lpd, len, codepage, dstcodepage, lossy);
627 IExconverter *pexconv = Exconverter::getInstance();
628 if (pexconv != nullptr)
630 size_t n = len * 6 + 6;
635 catch (std::bad_alloc&)
637 // Not enough memory - exit
640 char *buff = &*str.begin();
641 pexconv->convert(codepage, dstcodepage, (const unsigned char *)lpd, &len, (unsigned char *)buff, &n);
648 catch (std::bad_alloc&)
650 // Not enough memory - exit
665 * @brief (ANSI build only) Convert from one 8 bit codepage to another
668 String CrossConvertToStringA(const char* src, unsigned srclen, int cpin, int cpout, bool * lossy)
670 int wlen = srclen * 2 + 6;
671 int clen = wlen * 2 + 6;
674 char* cbuff = &*str.begin();
675 int nbytes = CrossConvert(src, srclen, cbuff, clen, cpin, cpout, lossy);
682 * @brief Convert from one 8-bit codepage to another
684 * destsize must be at least 2
686 int CrossConvert(const char* src, unsigned srclen, char* dest, unsigned destsize, int cpin, int cpout, bool * lossy)
688 assert(destsize > 1);
690 // Convert input to Unicode, using specified codepage
692 int wlen = srclen * 2 + 6;
693 auto wbuff = std::make_unique<wchar_t[]>(wlen);
695 if (cpin == CP_UCS2LE)
698 srclen = static_cast<unsigned>(wcslen((wchar_t *)src) * sizeof(wchar_t));
699 memcpy(wbuff.get(), src, srclen);
700 n = srclen / sizeof(wchar_t);
702 else if (cpin == CP_UCS2BE)
705 srclen = static_cast<unsigned>(wcslen((wchar_t *)src) * sizeof(wchar_t));
706 _swab((char *)src, (char *)wbuff.get(), srclen);
707 n = srclen / sizeof(wchar_t);
711 n = MultiByteToWideChar(cpin, flags, (const char*)src, srclen, wbuff.get(), wlen - 1);
714 int nsyserr = ::GetLastError();
720 NB: MultiByteToWideChar is documented as only zero-terminating
721 if input was zero-terminated, but it appears that it can
722 zero-terminate even if input wasn't.
723 So we check if it zero-terminated and adjust count accordingly.
727 wbuff[n] = 0; // zero-terminate string
729 // Now convert to TCHAR (which means defcodepage)
730 flags = WC_NO_BEST_FIT_CHARS; // TODO: Think about this
731 BOOL defaulted = FALSE;
732 BOOL * pdefaulted = &defaulted;
733 if (cpout == CP_UTF8)
736 pdefaulted = nullptr;
738 if (cpout == CP_UCS2LE)
740 memcpy(dest, wbuff.get(), n * sizeof(wchar_t));
741 n = n * sizeof(wchar_t);
745 else if (cpout == CP_UCS2BE)
747 _swab((char *)wbuff.get(), dest, n * sizeof(wchar_t));
748 n = n * sizeof(wchar_t);
754 n = WideCharToMultiByte(cpout, flags, wbuff.get(), n, dest, destsize - 1, nullptr, pdefaulted);
757 int nsyserr = ::GetLastError();
762 *lossy = !!defaulted;
767 * @brief Buffer constructor.
768 * The constructor creates buffer with given size.
769 * @param [in] initialSize Buffer's size.
771 buffer::buffer(size_t initialSize)
774 capacity = initialSize;
775 ptr = (unsigned char *)calloc(capacity, 1);
779 * @brief Buffer destructor.
780 * Frees the reserved buffer.
788 * @brief Resize the buffer.
789 * @param [in] newSize New size of the buffer.
791 void buffer::resize(size_t newSize)
793 if (capacity < newSize)
796 unsigned char *tmp = static_cast<unsigned char *>(realloc(ptr, capacity));
798 throw std::bad_alloc();
803 unsigned char *convertTtoUTF8(buffer * buf, const TCHAR *src, int srcbytes/* = -1*/)
807 bSucceeded = convert(CP_UCS2LE,
808 (unsigned char *)src, (int)((srcbytes < 0) ? wcslen((const wchar_t *)src) * sizeof(wchar_t) : srcbytes),
811 bSucceeded = convert(GetACP(),
812 (unsigned char *)src, (int)((srcbytes < 0) ? strlen((const char *)src) : srcbytes),
816 *((unsigned char *)buf->ptr) = 0;
820 unsigned char *convertTtoUTF8(const TCHAR *src, int srcbytes/* = -1*/)
823 convertTtoUTF8(&buf, src, srcbytes);
824 return (unsigned char *)_strdup((const char *)buf.ptr);
827 TCHAR *convertUTF8toT(buffer * buf, const char *src, int srcbytes/* = -1*/)
831 bSucceeded = convert(CP_UTF8,
832 (const unsigned char *)src, (int)((srcbytes < 0) ? strlen((const char *)src) : srcbytes),
835 bSucceeded = convert(CP_UTF8,
836 (const unsigned char *)src, (int)((srcbytes < 0) ? strlen((const char *)src) : srcbytes),
840 *((TCHAR *)buf->ptr) = 0;
841 return (TCHAR *)buf->ptr;
844 TCHAR *convertUTF8toT(const char *src, int srcbytes/* = -1*/)
847 convertUTF8toT(&buf, src, srcbytes);
848 return (TCHAR *)_tcsdup((LPCTSTR)buf.ptr);
851 void dealloc(void *ptr)
856 String toTString(const std::wstring& str)
861 return toThreadCP(str);
865 String toTString(const std::string& str)
869 UnicodeConverter::toUTF16(str, wstr);
872 const char *p = convertUTF8toT(str.c_str(), str.length());
873 std::string astr = p;
879 void toUTF16(const String& tstr, std::wstring& wstr)
884 UnicodeConverter::toUTF16(tstr, wstr);
888 std::string toUTF8(const String& tstr)
895 void toUTF8(const String& tstr, std::string& u8str)
899 size_t len = tstr.length();
902 u8str.resize(len * 3);
904 for (String::const_iterator it = tstr.begin(); it != tstr.end(); ++it)
907 if (uc >= 0xd800 && uc < 0xdc00)
910 if (it != tstr.end())
913 uc = ((uc & 0x3ff) << 10) + (uc2 & 0x3ff) + 0x10000;
916 p += Ucs4_to_Utf8(uc, reinterpret_cast<unsigned char *>(p));
918 u8str.resize(p - &u8str[0]);
920 const char *p = (const char *)convertTtoUTF8(tstr.c_str(), tstr.length());
926 bool convert(int codepage1, const unsigned char * src, int srcbytes, int codepage2, buffer * dest)
928 UNICODESET unicoding[2];
929 int codepage[2] = {codepage1, codepage2};
932 for (i = 0; i < 2; i++)
937 unicoding[i] = UCS2LE; break;
939 unicoding[i] = UCS2BE; break;
941 unicoding[i] = UTF8; break;
943 unicoding[i] = NONE; break;
947 return convert(unicoding[0], codepage1, src, srcbytes, unicoding[1], codepage2, dest);
951 * @brief Convert from one text encoding to another; return false if any lossing conversions
953 bool convert(UNICODESET unicoding1, int codepage1, const unsigned char * src, size_t srcbytes, UNICODESET unicoding2, int codepage2, buffer * dest)
955 if (unicoding1 == unicoding2 && (unicoding1 || EqualCodepages(codepage1, codepage2)))
958 dest->resize(srcbytes + 2);
959 CopyMemory(dest->ptr, src, srcbytes);
960 dest->ptr[srcbytes] = 0;
961 dest->ptr[srcbytes+1] = 0;
962 dest->size = srcbytes;
965 if ((unicoding1 == UCS2LE && unicoding2 == UCS2BE)
966 || (unicoding1 == UCS2BE && unicoding2 == UCS2LE))
969 dest->resize(srcbytes + 2);
970 for (size_t i = 0; i < srcbytes; i += 2)
972 // Byte-swap into destination
973 dest->ptr[i] = src[i+1];
974 dest->ptr[i+1] = src[i];
976 dest->ptr[srcbytes] = 0;
977 dest->ptr[srcbytes+1] = 0;
978 dest->size = srcbytes;
981 if (unicoding1 != UCS2LE && unicoding2 != UCS2LE)
983 // Break problem into two simpler pieces by converting through UCS-2LE
984 buffer intermed(dest->capacity + 2);
985 bool step1 = convert(unicoding1, codepage1, src, srcbytes, UCS2LE, 0, &intermed);
986 bool step2 = convert(UCS2LE, 0, intermed.ptr, intermed.size, unicoding2, codepage2, dest);
987 return step1 && step2;
989 if (unicoding1 == UCS2LE)
991 // From UCS-2LE to 8-bit (or UTF-8)
993 // WideCharToMultiByte: lpDefaultChar & lpUsedDefaultChar must be `nullptr` when using UTF-8
995 int destcp = (unicoding2 == UTF8 ? CP_UTF8 : codepage2);
996 if (destcp == CP_ACP || IsValidCodePage(destcp))
999 int bytes = WideCharToMultiByte(destcp, flags, (LPCWSTR)src, static_cast<int>(srcbytes/2), 0, 0, nullptr, nullptr);
1000 dest->resize(bytes + 2);
1002 bytes = WideCharToMultiByte(destcp, flags, (LPCWSTR)src, static_cast<int>(srcbytes/2), (char *)dest->ptr, static_cast<int>(dest->capacity), nullptr, nullptr);
1003 dest->ptr[bytes] = 0;
1004 dest->ptr[bytes+1] = 0;
1010 size_t srcsize = srcbytes / 2;
1011 size_t dstsize = srcbytes * 6;
1012 dest->resize(dstsize + 2);
1013 IExconverter *pexconv = Exconverter::getInstance();
1014 if (pexconv != nullptr)
1016 bool result = pexconv->convertFromUnicode(destcp, (LPWSTR)src, &srcsize, (char *)dest->ptr, &dstsize);
1017 dest->ptr[dstsize] = 0;
1018 dest->ptr[dstsize+1] = 0;
1019 dest->size = dstsize;
1028 // From 8-bit (or UTF-8) to UCS-2LE
1029 int srccp = (unicoding1 == UTF8 ? CP_UTF8 : codepage1);
1030 if (srccp == CP_ACP || IsValidCodePage(srccp))
1033 int wchars = MultiByteToWideChar(srccp, flags, (LPCSTR)src, static_cast<int>(srcbytes), 0, 0);
1034 dest->resize((wchars + 1) *2);
1035 wchars = MultiByteToWideChar(srccp, flags, (LPCSTR)src, static_cast<int>(srcbytes), (LPWSTR)dest->ptr, static_cast<int>(dest->capacity/2));
1036 dest->ptr[wchars * 2] = 0;
1037 dest->ptr[wchars * 2 + 1] = 0;
1038 dest->size = wchars * 2;
1043 size_t srcsize = srcbytes;
1044 size_t dstsize = srcbytes;
1045 dest->resize((srcbytes + 1) * sizeof(wchar_t));
1046 IExconverter *pexconv = Exconverter::getInstance();
1047 if (pexconv != nullptr)
1049 bool result = pexconv->convertToUnicode(srccp, (LPCSTR)src, &srcsize, (LPWSTR)dest->ptr, &dstsize);
1050 dest->ptr[dstsize * sizeof(wchar_t)] = 0;
1051 dest->ptr[dstsize * sizeof(wchar_t) + 1] = 0;
1052 dest->size = dstsize * sizeof(wchar_t);
1062 * @brief Convert from Unicode to Ansi using given codepage.
1063 * @param [in] from String to convert.
1064 * @param [in] codepage Codepage to use in conversion.
1065 * @param [out] to Ansi string.
1067 static void convert(const std::wstring& from, unsigned codepage, std::string& to)
1069 int len = WideCharToMultiByte(codepage, 0, from.c_str(), static_cast<int>(from.length()), 0, 0, 0, 0);
1073 WideCharToMultiByte(codepage, 0, from.c_str(), static_cast<int>(from.length()), &to[0], static_cast<int>(len), nullptr, nullptr);
1082 * @brief Convert from Unicode to Ansi using system codepage.
1083 * This function converts Unicode string to ansi string using system codepage.
1084 * This conversion function should be used when converting strings containing
1085 * paths. As paths are handled by the system and are not file content.
1086 * @param [in] str String to convert.
1087 * @return Ansi string.
1089 std::string toSystemCP(const std::string& str)
1094 std::string toSystemCP(const std::wstring& str)
1097 convert(str, CP_ACP, to);
1102 * @brief Convert from Unicode to Ansi using thread codepage.
1103 * This function converts Unicode string to ansi string using thread codepage.
1104 * Thread codepage is practically the codepage WinMerge is using internally.
1105 * @param [in] str String to convert.
1106 * @return Ansi string.
1108 std::string toThreadCP(const std::string& str)
1113 std::string toThreadCP(const std::wstring& str)
1116 convert(str, CP_THREAD_ACP, to);
1120 // Algorithm originally from:
1121 // TortoiseMerge - a Diff/Patch program
1122 // Copyright (C) 2007 - TortoiseSVN
1124 * @brief Check for invalid UTF-8 bytes in buffer.
1125 * This function checks if there are invalid UTF-8 bytes in the given buffer.
1126 * If such bytes are found, caller knows this buffer is not valid UTF-8 file.
1127 * @param [in] pBuffer Pointer to begin of the buffer.
1128 * @param [in] size Size of the buffer in bytes.
1129 * @return true if invalid bytes found, false otherwise.
1131 bool CheckForInvalidUtf8(const char* pBuffer, size_t size)
1134 for (unsigned char* pb = (unsigned char*)pBuffer, *end = pb + size; pb < end;)
1138 if (!(c & 0x80)) continue;
1140 if ((c >= 0xF5) || (c == 0xC0) || (c == 0xC1))
1143 uint32_t v = 0x80808000; //1st 0-byte covers scenario if no any next "if" fired at all
1145 if ((c & 0xE0) == 0xC0)
1149 *reinterpret_cast<unsigned char*>(&v) = *pb++;
1151 else if ((c & 0xF0) == 0xE0)
1155 *reinterpret_cast<uint16_t*>(&v) = *reinterpret_cast<uint16_t*>(pb);
1158 else if ((c & 0xF8) == 0xF0)
1162 static_assert(sizeof(char) == sizeof(uint8_t), "unexpected char-size");
1164 *reinterpret_cast<uint16_t*>(&v) = *reinterpret_cast<uint16_t*>(pb);
1165 reinterpret_cast<uint8_t*>(&v)[2] = pb[2];
1169 if ((v & (0xC0C0C0C0)) != 0x80808080)
1177 * @brief Determine encoding from byte buffer.
1178 * @param [in] pBuffer Pointer to the begin of the buffer.
1179 * @param [in] size Size of the buffer.
1180 * @param [out] pBom Returns true if buffer had BOM bytes, false otherwise.
1181 * @return One of UNICODESET values as encoding.
1183 * FF FE UTF-16, little endian
1184 * FE FF UTF-16, big endian
1185 * FF FE 00 00 UTF-32, little endian
1186 * 00 00 FE FF UTF-32, big-endian
1188 UNICODESET DetermineEncoding(const unsigned char *pBuffer, uint64_t size, bool * pBom)
1190 UNICODESET unicoding = NONE;
1195 if (pBuffer[0] == 0xFF && pBuffer[1] == 0xFE)
1197 unicoding = UCS2LE; //UNI little endian
1200 else if (pBuffer[0] == 0xFE && pBuffer[1] == 0xFF)
1202 unicoding = UCS2BE; //UNI big endian
1208 if (pBuffer[0] == 0xEF && pBuffer[1] == 0xBB && pBuffer[2] == 0xBF)
1216 if (pBuffer[0] == 0xFF && pBuffer[1] == 0xFE &&
1217 pBuffer[2] == 0x00 && pBuffer[3] == 0x00)
1219 unicoding = UCS4LE; //UTF-32, little endian
1222 else if (pBuffer[0] == 0x00 && pBuffer[1] == 0x00 &&
1223 pBuffer[2] == 0xFE && pBuffer[3] == 0xFF)
1225 unicoding = UCS4BE; //UTF-32, big endian
1234 * @brief Change any special codepage constants into real codepage numbers
1236 static int NormalizeCodepage(int cp)
1238 if (cp == CP_THREAD_ACP) // should only happen on Win2000+
1241 if (GetLocaleInfo(GetThreadLocale(), LOCALE_IDEFAULTANSICODEPAGE, buff, sizeof(buff) / sizeof(buff[0])))
1244 // a valid codepage is better than no codepage
1247 if (cp == CP_ACP) cp = GetACP();
1248 if (cp == CP_OEMCP) cp = GetOEMCP();
1253 * @brief Compare two codepages for equality
1255 bool EqualCodepages(int cp1, int cp2)
1258 || (NormalizeCodepage(cp1) == NormalizeCodepage(cp2));
1261 int getDefaultCodepage()
1263 return f_nDefaultCodepage;
1266 void setDefaultCodepage(int cp)
1268 f_nDefaultCodepage = cp;