* @file UniFile.cpp
* @author Perry Rapp, Creator, 2003
* @date Created: 2003-10
- * @date Edited: 2003-10-21 (Perry)
+ * @date Edited: 2003-11-14 (Perry)
*
* @brief Implementation of Memory-Mapped Unicode enabled file class
*/
if (sizelo == 0)
{
- // Empty file (but should be accepted anyway)
+ // Allow opening empty file, but memory mapping doesn't work on such
+ // m_base and m_current are 0 from the Close call above
+ // so ReadString will correctly return empty EOF immediately
m_lineno = 0;
return true;
}
break;
}
}
- line = ucr::maketstring(m_current, eolptr-m_current, m_codepage);
+ bool lossy=false;
+ line = ucr::maketstring((LPCSTR)m_current, eolptr-m_current, m_codepage, &lossy);
if (!eof)
{
eol += (TCHAR)*eolptr;
++m_lineno;
}
m_current = eolptr + eol.GetLength();
+ // TODO: What do we do if save was lossy ?
return !eof;
}
/**
* @file UniFile.h
* @author Perry Rapp, Creator, 2003
- * @date Created: 2003-10
- * @date Edited: 2003-10-21 (Perry)
+ * @date Created: 2003-10 (Perry)
+ * @date Edited: 2003-11-15 (Perry)
*
* @brief Declaration of Memory-Mapped Unicode enabled file class
*/
virtual CString GetFullyQualifiedPath() const = 0;
- virtual UniError GetLastUniError() const = 0;
+ virtual const UniError & GetLastUniError() const = 0;
virtual bool ReadBom() = 0;
- virtual int GetUnicoding() = 0;
+ virtual int GetUnicoding() const = 0;
+ virtual void SetUnicoding(int unicoding) = 0;
+
+ virtual int GetCodepage() const = 0;
virtual void SetCodepage(int codepage) = 0;
virtual BOOL ReadString(CString & line) = 0;
virtual const txtstats & GetTxtStats() const = 0;
};
-
/**
- * @brief Memory-Mapped disk file
+ * @brief Memory-Mapped disk file (read-only access)
*/
class UniMemFile : public UniFile
{
virtual CString GetFullyQualifiedPath() const { return m_filepath; }
const CFileStatus & GetFileStatus() const { return m_filestatus; }
- virtual UniError GetLastUniError() const { return m_lastError; }
+ virtual const UniError & GetLastUniError() const { return m_lastError; }
virtual bool ReadBom();
- virtual int GetUnicoding() { return m_unicoding; }
+ virtual int GetUnicoding() const { return m_unicoding; }
+ virtual void SetUnicoding(int unicoding) { m_unicoding = unicoding; }
+
+ virtual int GetCodepage() const { return m_codepage; }
virtual void SetCodepage(int codepage) { m_codepage = codepage; }
virtual BOOL ReadString(CString & line);
* @file unicoder.cpp
* @author Perry Rapp, Creator, 2003
* @date Created: 2003-10
- * @date Edited: 2003-10-21 (Perry)
+ * @date Edited: 2003-11-09 (Perry)
*
* @brief Implementation of utility unicode conversion routines
*/
f_osvi_fetched = true;
}
+static LPCTSTR f_unicodesetNames[] = { _T("<NONE>"), _T("UCS-2LE"), _T("UCS-2BE"), _T("UTF-8") };
+/**
+ * @brief return string for enum value
+ */
+CString GetUnicodesetName(UNICODESET codeset)
+{
+ if (codeset>=0 && codeset<sizeof(f_unicodesetNames)/sizeof(f_unicodesetNames[0]))
+ return f_unicodesetNames[codeset];
+ else
+ return _T("?");
+}
+
/**
* @brief Convert unicode codepoint to UTF-8 byte string
*
}
/**
- * @brief Write src string (TCHAR) to destination buffer in codeset specified.
- *
- * This is designed for use writing to memory-mapped file.
- * NB: for loop is deliberately repeated inside the if statements (for speed)
+ * @brief Get appropriate default codepage for this operating system
*/
-void
-convertToBuffer(const CString & src, LPVOID dest, UNICODESET codeset)
+int
+getDefaultCodepage()
+{
+ // default codepage is CP_THREAD_ACP if available, else CP_ACP
+ static bool vercheck=false;
+ static int defcodepage = CP_ACP;
+ if (!vercheck)
+ {
+ if (!f_osvi_fetched) fetch_verinfo();
+ // Need 2000 or better for CP_THREAD_ACP
+ if (f_osvi.dwMajorVersion>=5)
+ defcodepage = CP_THREAD_ACP;
+ vercheck = true;
+ }
+ return defcodepage;
+}
+
+/**
+ * @brief Write appropriate BOM (Unicode byte order marker)
+ * returns #bytes written
+ */
+int
+writeBom(LPVOID dest, UNICODESET codeset)
{
unsigned char * lpd = reinterpret_cast<unsigned char *>(dest);
+ // write Unicode byte order marker (BOM)
if (codeset == UCS2LE)
{
*lpd++ = 0xFF;
*lpd++ = 0xFE;
+ return 2;
}
else if (codeset == UCS2BE)
{
*lpd++ = 0xFE;
*lpd++ = 0xFF;
+ return 2;
}
else if (codeset == UTF8)
{
*lpd++ = 0xEF;
*lpd++ = 0xBB;
*lpd++ = 0xBF;
+ return 3;
}
+ return 0;
+}
+
+/**
+ * @brief Write src string (TCHAR) to destination buffer in codeset specified.
+ *
+ * In ANSI build, source string is assumed to be in defcodepage.
+ * This is designed for use writing to memory-mapped file.
+ * NB: Destination is not zero-terminated!
+ */
+int
+convertToBuffer(const CString & src, LPVOID dest, UNICODESET codeset, int codepage)
+{
+ static int defcodepage = getDefaultCodepage();
+
+ unsigned char * lpd = reinterpret_cast<unsigned char *>(dest);
+ unsigned char * start = lpd;
+
#ifdef _UNICODE
if (codeset == UCS2LE)
{
- CopyMemory(lpd, src, src.GetLength() * 2);
- return;
+ int nbytes = src.GetLength() * 2;
+ CopyMemory(lpd, src, nbytes);
+ return nbytes;
}
else if (codeset == UCS2BE)
{
*lpd++ = (unsigned char)(u >> 8);
*lpd++ = (unsigned char)(u & 0xFF);
}
+ return lpd - start;
}
else if (codeset == UTF8)
{
UINT u = src[i];
ucr::to_utf8_advance(u, lpd);
}
- }
- else if (codeset == NONE)
- {
- for (int i=0; i<src.GetLength(); ++i)
- {
- UINT u = src[i];
- if (u <= 0xFF)
- *lpd++ = (unsigned char)u;
- else
- *lpd++ = '?';
- }
+ return lpd - start;
}
else
{
- ASSERT(0);
+ ASSERT(codeset == NONE); // there aren't any other values in UNICODESET
+ // Write wchars to chars
+ DWORD flags = 0;
+ // Take a swag at the maximum length :(
+ int maxlen = src.GetLength() * 3;
+ BOOL replaced=FALSE;
+ int nbytes = WideCharToMultiByte(codepage, flags, src, src.GetLength(), (char *)lpd, maxlen, NULL, &replaced);
+ // TODO: Ought to output replaced flag to caller
+ return nbytes;
}
+
#else
- // We really need the codeset of the 8-bit source
+ // ANSI build, TCHAR=char
if (codeset == UCS2LE)
{
*lpd++ = (unsigned char)u;
*lpd++ = 0;
}
+ return lpd - start;
}
else if (codeset == UCS2BE)
{
*lpd++ = 0;
*lpd++ = (unsigned char)u;
}
+ return lpd - start;
}
else if (codeset == UTF8)
{
UINT u = src[i];
to_utf8_advance(u, lpd);
}
+ return lpd - start;
}
else
{
- ASSERT(0);
+ ASSERT(codeset == NONE); // there aren't any other values in UNICODESET
+
+ if (codepage == defcodepage)
+ {
+ // trivial case, string is already in the correct codepage
+ CopyMemory(lpd, (LPCTSTR)src, src.GetLength());
+ return src.GetLength();
+ }
+ bool lossy=false;
+ // Take a swag at the maximum length :(
+ int maxlen = src.GetLength() * 3;
+ int nbytes = CrossConvert(src, src.GetLength(), (LPSTR)lpd, maxlen, defcodepage, codepage, &lossy);
+ return nbytes;
}
#endif
}
ch = (ptr[0] << 8) + ptr[1];
break;
default:
- ch = (codepage ? byteToUnicode(*ptr, codepage) : byteToUnicode(*ptr));
+ // TODO: How do we recognize valid codepage ?
+ // if not, use byteToUnicode(*ptr)
+ ch = byteToUnicode(*ptr, codepage);
}
return ch;
}
* In fact, this doesn't even know. Probably going to have to make
* two passes, the first with MB_ERR_INVALID_CHARS. Ugh. :(
*/
-CString maketstring(unsigned char * lpd, UINT len, int codepage)
+CString maketstring(LPCSTR lpd, UINT len, int codepage, bool * lossy)
{
- static bool vercheck=false;
- static int defcodepage = CP_ACP;
- if (!vercheck)
- {
- if (!f_osvi_fetched) fetch_verinfo();
- // Need 2000 or better for CP_THREAD_ACP
- if (f_osvi.dwMajorVersion>=5)
- defcodepage = CP_THREAD_ACP;
- vercheck = true;
- }
+ static int defcodepage = getDefaultCodepage();
+
if (!len) return _T("");
if (!codepage)
codepage = defcodepage;
+#ifdef UNICODE
+ // Convert input to Unicode, using specified codepage
+ // TCHAR is wchar_t, so convert into CString (str)
+ CString str = lpd; // TODO: fix 2003-11-13
+ DWORD flags = 0;
+ int wlen = len*2+6;
+ LPWSTR wbuff = str.GetBuffer(wlen);
+ int n = MultiByteToWideChar(codepage, flags, (LPCSTR)lpd, len, wbuff, wlen-1);
+ if (n)
+ {
+ str.ReleaseBuffer(n);
+ }
+ else
+ {
+ str = _T("?");
+ }
+ return str;
+
+#else
if (codepage == defcodepage)
{
// trivial case, they want the bytes in the file interpreted in our current codepage
-#ifndef UNICODE
return lpd;
-#else
- CString s;
- for (UINT i=0; i<len; ++i)
- s += (wchar_t)(*lpd++);
- return s;
-#endif
}
+ CString str = CrossConvertToStringA(lpd, len, codepage, defcodepage, lossy);
+ return str;
+#endif
+}
+
+/**
+ * @brief (ANSI build only) Convert from one 8 bit codepage to another
+ */
+#ifndef UNICODE
+CString
+CrossConvertToStringA(LPCSTR src, UINT srclen, int cpin, int cpout, bool * lossy)
+{
+
+ CString str;
+ int wlen = srclen*2+6;
+ int clen = wlen * 2 + 6;
+ LPSTR cbuff = str.GetBuffer(clen);
+ int nbytes = CrossConvert(src, srclen, cbuff, clen, cpin, cpout, lossy);
+ str.ReleaseBuffer(nbytes);
+ return str;
+}
+#endif
+
+/**
+ * @brief Convert from one 8-bit codepage to another
+ *
+ * destsize must be at least 2
+ */
+int
+CrossConvert(LPCSTR src, UINT srclen, LPSTR dest, UINT destsize, int cpin, int cpout, bool * lossy)
+{
+ ASSERT(destsize > 1);
+
// Convert input to Unicode, using specified codepage
DWORD flags = 0;
- int wlen = len*2+6;
+ int wlen = srclen*2+6;
wchar_t * wbuff = new wchar_t[wlen];
- int n = MultiByteToWideChar(codepage, flags, (LPCSTR)lpd, len, wbuff, wlen-1);
+ int n = MultiByteToWideChar(cpin, flags, (LPCSTR)src, srclen, wbuff, wlen-1);
if (!n)
{
delete [] wbuff;
- return _T("?");
+ dest[0] = '?';
+ return 1;
}
wbuff[n] = 0; // zero-terminate string
-#ifdef UNICODE
- // wchar_t is TCHAR, so we're done
- CString str = wbuff;
- delete [] wbuff;
- return str;
-#else
// Now convert to TCHAR (which means defcodepage)
flags = WC_NO_BEST_FIT_CHARS; // TODO: Think about this
- CString str;
wlen = n;
int clen = wlen * 2 + 6;
- LPSTR cbuff = str.GetBuffer(clen);
BOOL defaulted=FALSE;
- n = WideCharToMultiByte(defcodepage, flags, wbuff, n, cbuff, clen-1, NULL, &defaulted);
- if (n)
- {
- cbuff[n] = 0; // zero-terminate string
- str.ReleaseBuffer();
- }
- else
- {
- str = _T("?");
- }
+ LPSTR cdest = reinterpret_cast<LPSTR>(dest);
+ n = WideCharToMultiByte(cpout, flags, wbuff, n, cdest, destsize-1, NULL, &defaulted);
+ dest[n] = 0;
delete [] wbuff;
- return str;
-#endif
+ if (lossy)
+ *lossy = !!defaulted;
+ return n;
}
} // namespace ucr
* @file unicoder.h
* @author Perry Rapp, Creator, 2003
* @date Created: 2003-10
- * @date Edited: 2003-10-21 (Perry)
+ * @date Edited: 2003-11-13 (Perry)
*
* @brief Declaration of utility unicode conversion routines
*/
namespace ucr {
typedef enum { NONE=0, UCS2LE, UCS2BE, UTF8 } UNICODESET;
+CString GetUnicodesetName(UNICODESET codeset);
int Ucs4_to_Utf8(UINT unich, unsigned char * utf8);
int Utf8len_fromLeadByte(unsigned char ch);
UINT GetUtf8Char(unsigned char * str);
int to_utf8_advance(UINT u, unsigned char * &lpd);
CString maketchar(UINT ch, bool & lossy);
-void convertToBuffer(const CString & src, LPVOID dest, UNICODESET codeset);
+int writeBom(LPVOID dest, UNICODESET codeset);
+int convertToBuffer(const CString & src, LPVOID dest, UNICODESET codeset, int codepage);
UINT get_unicode_char(unsigned char * ptr, UNICODESET codeset, int codepage=0);
-CString maketstring(unsigned char * lpd, UINT len, int codepage=0);
+CString maketstring(LPCSTR lpd, UINT len, int codepage, bool * lossy);
CString maketchar(UINT unich, bool & lossy);
CString maketchar(UINT unich, bool & lossy, UINT codepage);
UINT byteToUnicode(unsigned char ch);
UINT byteToUnicode(unsigned char ch, UINT codepage);
+int getDefaultCodepage();
+int CrossConvert(LPCSTR src, UINT srclen, LPSTR dest, UINT destsize, int cpin, int cpout, bool * lossy);
+#ifndef UNICODE
+CString CrossConvertToStringA(LPCSTR src, UINT srclen, int cpin, int cpout, bool * lossy);
+#endif
} // namespace ucr
{
// we subvert the buffer's memory of the original file encoding
int temp=buf.m_nSourceEncoding;
+ int unicoding = buf.getUnicoding();
- if (sizeof(TCHAR)>1
- || buf.m_nSourceEncoding==-20 // source file was UCS-2LE
- || buf.m_nSourceEncoding==-21 // source file was UCS-2BE
- || buf.m_nSourceEncoding==-22) // source file was UTF-8
+ // If Unicode build, or file was in Unicode
+ if (sizeof(TCHAR)>1 || unicoding!=ucr::NONE)
{
buf.m_nSourceEncoding = -20; // write as UCS-2LE (for preprocessing)
}
nRetVal = FRESULT_OK_IMPURE;
// stash original encoding away
- switch (pufile->GetUnicoding())
- {
- case ucr::UCS2LE:
- m_nSourceEncoding = -20;
- break;
- case ucr::UCS2BE:
- m_nSourceEncoding = -21;
- break;
- case ucr::UTF8:
- m_nSourceEncoding = -22;
- break;
- }
+ m_unicoding = pufile->GetUnicoding();
+ m_codepage = pufile->GetCodepage();
+
if (pufile->GetTxtStats().nlosses)
readOnly = TRUE;
}
UINT nchars = text.GetLength();
UINT nbytes = -1;
- if (m_nSourceEncoding == -20)
+ if (m_unicoding == ucr::UCS2LE)
{
// UCS-2LE (+ 2 byte BOM)
nbytes = 2 * nchars + 2;
}
- else if (m_nSourceEncoding == -21)
+ else if (m_unicoding == ucr::UCS2BE)
{
// UCS-2BE (+ 2 byte BOM)
nbytes = 2 * nchars + 2;
}
- else if (m_nSourceEncoding == -22)
+ else if (m_unicoding == ucr::UTF8)
{
// UTF-8 (+ 3 byte BOM)
nbytes = ucr::Utf8len_of_string(text) + 3;
// Should these Unicode codeset conversions be moved
// into the iconvert (editlib/cs2cs.*) module ?
- if (m_nSourceEncoding == -20)
- {
- ucr::convertToBuffer(text, fileData.pMapBase, ucr::UCS2LE);
- }
- else if (m_nSourceEncoding == -21)
- {
- ucr::convertToBuffer(text, fileData.pMapBase, ucr::UCS2BE);
- }
- else if (m_nSourceEncoding == -22)
+ if (m_unicoding != ucr::NONE)
{
- ucr::convertToBuffer(text, fileData.pMapBase, ucr::UTF8);
+ ucr::writeBom(fileData.pMapBase, (ucr::UNICODESET)m_unicoding);
+ ucr::convertToBuffer(text, fileData.pMapBase, (ucr::UNICODESET)m_unicoding, m_codepage);
}
else
{
#ifdef _UNICODE
- // (We're ignoring m_nSourceEncoding here, which ought to be relevant
- // altho I don't think it ever gets set right now -- Perry, 2003-09-24)
- ucr::convertToBuffer(text, fileData.pMapBase, ucr::NONE);
+ // WinMerge doesn't use crystal's m_nSourceEncoding
+ // which anyway never gets set
+ // WinMerge uses its own buffer's m_unicoding and m_codepage
+ ucr::convertToBuffer(text, fileData.pMapBase, (ucr::UNICODESET)m_unicoding, m_codepage);
#else
if (m_nSourceEncoding >= 0)
{
BOOL FlagIsSet(UINT line, DWORD flag);
CString m_strTempPath;
int unpackerSubcode;
+ int m_unicoding; /**< Unicode encoding from ucr::UNICODESET */
+ int m_codepage; /**< @brief 8-bit codepage, if relevant m_unicoding==ucr::NONE */
int NoteCRLFStyleFromBuffer(TCHAR *lpLineBegin, DWORD dwLineLen = 0);
void ReadLineFromBuffer(TCHAR *lpLineBegin, DWORD dwLineNum, DWORD dwLineLen = 0);
BOOL LoadFromFile(LPCTSTR pszFileName, PackingInfo * infoUnpacker, CString filteredFilenames, BOOL & readOnly, int nCrlfStyle, int codepage);
BOOL SaveToFile (LPCTSTR pszFileName, BOOL bTempFile, PackingInfo * infoUnpacker = NULL,
int nCrlfStyle = CRLF_STYLE_AUTOMATIC, BOOL bClearModifiedFlag = TRUE );
+ int getUnicoding() const { return m_unicoding; }
CDiffTextBuffer (CMergeDoc * pDoc, BOOL bLeft)
{
m_pOwnerDoc = pDoc;
m_bIsLeft=bLeft;
+ unpackerSubcode = 0;
+ m_unicoding = 0;
+ m_codepage = 0;
}
// If line has text (excluding eol), set strLine to text (excluding eol)
BOOL GetLine( int nLineIndex, CString &strLine )
2003-11-15 Perry
PATCH: [ 838290 ] Move inf creation down in RunFileDiff (cosmetic)
WinMerge: DiffWrapper.cpp
+ PATCH: [ 841878 ] Fix wchar to char conversions at save
+ WinMerge: MergeDoc.cpp MergeDoc.h
+ common: unicoder.cpp unicoder.h UniFile.cpp UniFile.h
2003-11-15 WinMerge experimental release 2.1.3.9 (cvs)
OSDN Git Service
