}
}
-
-
-
-
-
-//////////////////////////////////////////////////////////////////////////////////
-// other conversion functions
-// TODO : put in unicoder.cpp or merge in class storageForPlugins
-
-/**
- * @brief Copy UCS-2LE string to UTF-8 string
- *
- * @param nUcs is the size in wide characters of the source string
- * @param nUtf is the size in bytes of the resulting string
- *
- * @return if nUtf = 0, return the size required for the translation buffer
- */
-static size_t TransformUcs2ToUtf8(const wchar_t *psUcs, size_t nUcs, char *pcsUtf, size_t nUtf)
-{
- if (nUtf == 0)
- // just tell required length
- return ucr::Utf8len_of_string(psUcs, nUcs);
-
- // the buffer is allocated, output in it directly
- unsigned char * pc = (unsigned char *) pcsUtf;
- size_t nremains = nUtf;
-
- // quick way
- size_t i=0;
- for (i = 0 ; i < nUcs && nremains > 10; ++i)
- nremains -= ucr::to_utf8_advance(psUcs[i], pc);
-
- // be careful for the end of the buffer, risk of overflow because
- // of the variable length of the UTF-8 character
- unsigned char smallTempBuffer[20];
- size_t nremainsend = nremains;
- unsigned char * pcTemp = (unsigned char *) smallTempBuffer;
- for ( ; i < nUcs && nremainsend > 0; ++i)
- nremainsend -= ucr::to_utf8_advance(psUcs[i], pcTemp);
-
- size_t ncomplement = std::min((size_t)nremains, (size_t)(pcTemp-smallTempBuffer));
- std::memcpy(pc, smallTempBuffer, ncomplement);
- nremains -= ncomplement;
-
- // return number of written bytes
- return (nUtf - nremains);
-}
-
/**
* @brief Copy UTF-8 string to UCS-2LE string
*
}
}
-/**
- * @brief If file is Ansi, then convert it to UTF8
- *
- * If file has Ansi (is ascii), then convert it to UTF8.
- * (No other file conversions are done here;
- * (UCS-2LE is the Windows standard Unicode encoding)
- *
- * Returns false if file is Unicode but opening it fails.
- * Returns false if file has Unicode BOM but is not Ansi.
- * Returns true if file is not Unicode, or if converted file successfully.
- */
-bool AnsiToUTF8(const String& filepath, const String& filepathDst, int & nFileChanged, bool bWriteBOM)
-{
- UniMemFile ufile;
- if (!ufile.OpenReadOnly(filepath))
- return false; // error
- if (ufile.IsUnicode())
- {
- // Finished with examing file contents
- ufile.Close();
- return false; // no ansi file
- }
-
- ucr::UNICODESET unicoding = ufile.GetUnicoding(); //ucr::UCS2LE
- // Finished with examing file contents
- ufile.Close();
-
- // AnsiToUTF16 only converts Ascii files to UTF-16
- if (unicoding != ucr::NONE)
- return false;
-
- try
- {
- // Init filedataIn struct and open file as memory mapped (input)
- TFile fileIn(filepath);
- SharedMemory shmIn(fileIn, SharedMemory::AM_READ);
-
- char * pszBuf = shmIn.begin();
- size_t nBufSize = shmIn.end() - shmIn.begin();
-
- // first pass : get the size of the destination file
- size_t nSizeOldBOM = 0;
- size_t nchars = (nBufSize - nSizeOldBOM)/sizeof(char);
-
- size_t nSizeBOM = (bWriteBOM) ? 3 : 0;
- DWORD flags =0;
- int nDstSize;
- //First convert to unicode UCS16
- nDstSize = MultiByteToWideChar(ufile.GetCodepage(), flags, (const char*)(pszBuf + nSizeOldBOM),nchars,0,0);
- std::vector<wchar_t> szTemp(nDstSize);
- nDstSize = MultiByteToWideChar(ufile.GetCodepage(), flags, (const char*)(pszBuf + nSizeOldBOM),nchars,&szTemp[0],nDstSize);
-
- //now convert to unicode UTF8
- nDstSize = WideCharToMultiByte(CP_UTF8, flags, &szTemp[0], nDstSize, NULL, 0, NULL, NULL);
-
- // create the destination file
- TFile fileOut(filepathDst);
- fileOut.setSize(nDstSize + nSizeBOM);
- SharedMemory shmOut(fileOut, SharedMemory::AM_WRITE);
-
- // write BOM
- if (bWriteBOM)
- ucr::writeBom(shmOut.begin(), ucr::UTF8);
-
- // write data
- WideCharToMultiByte(CP_UTF8, flags, &szTemp[0], nDstSize,
- (shmOut.begin()+nSizeBOM), nDstSize, NULL, NULL);
-
- nFileChanged ++;
- return true;
- }
- catch (...)
- {
- return false;
- }
-}
-/**
- * @brief From UCS-2LE to 8-bit (or UTF-8)
- *
- * (No other file conversions are done here;
- * (UCS-2LE is the Windows standard Unicode encoding)
- *
- * Returns false if file is Unicode but opening it fails.
- * Returns false if file has Unicode BOM but is not Ansi.
- * Returns true if file is not Unicode, or if converted file successfully.
- */
-bool UCS2LEToUTF8(const String& filepath, const String& filepathDst, int & nFileChanged, bool bWriteBOM)
-{
- UniMemFile ufile;
- if (!ufile.OpenReadOnly(filepath))
- return false; // error
- if (!ufile.IsUnicode())
- {
- // Finished with examing file contents
- ufile.Close();
- return false; // no ansi file
- }
-
- ucr::UNICODESET unicoding = ufile.GetUnicoding(); //ucr::UCS2LE
- // Finished with examing file contents
- ufile.Close();
-
- // UCS-2LE to 8-bit (or UTF-8)
- if (unicoding != ucr::UCS2LE)
- return false;
-
- try
- {
- // Init filedataIn struct and open file as memory mapped (input)
- TFile fileIn(filepath);
- SharedMemory shmIn(fileIn, SharedMemory::AM_READ);
-
- wchar_t * pszBuf = (wchar_t *)shmIn.begin();
- size_t nBufSize = shmIn.end() - shmIn.begin();
-
- // first pass : get the size of the destination file
- size_t nSizeOldBOM = 1;
- size_t nchars = (nBufSize - nSizeOldBOM) / sizeof(wchar_t);
-
- size_t nSizeBOM = (bWriteBOM) ? 3 : 0;
- DWORD flags =0;
- size_t nDstSize;
-
- //now convert to unicode UTF8
- nDstSize = WideCharToMultiByte(CP_UTF8, flags, (pszBuf + nSizeOldBOM), nchars, NULL, 0, NULL, NULL);
-
- // create the destination file
- TFile fileOut(filepathDst);
- fileOut.setSize(nDstSize + nSizeBOM);
- SharedMemory shmOut(fileOut, SharedMemory::AM_WRITE);
-
- // write BOM
- if (bWriteBOM)
- ucr::writeBom(shmOut.begin(), ucr::UTF8);
-
- // write data
- WideCharToMultiByte(CP_UTF8, flags, (pszBuf + nSizeOldBOM), nchars,
- (shmOut.begin()+nSizeBOM), nDstSize, NULL, NULL);
-
- nFileChanged ++;
- return true;
- }
- catch (...)
- {
- return false;
- }
-}
-/**
- * @brief From UCS-2BE to 8-bit (or UTF-8)
- *
- * (No other file conversions are done here;
- * (UCS-2LE is the Windows standard Unicode encoding)
- *
- * Returns false if file is Unicode but opening it fails.
- * Returns false if file has Unicode BOM but is not Ansi.
- * Returns true if file is not Unicode, or if converted file successfully.
- */
-bool UCS2BEToUTF8(const String& filepath, const String& filepathDst, int & nFileChanged, bool bWriteBOM)
-{
- UniMemFile ufile;
- if (!ufile.OpenReadOnly(filepath))
- return false; // error
- if (!ufile.IsUnicode())
- {
- // Finished with examing file contents
- ufile.Close();
- return false; // no ansi file
- }
-
- ucr::UNICODESET unicoding = ufile.GetUnicoding(); //ucr::UCS2LE
- // Finished with examing file contents
- ufile.Close();
-
- // UCS-2BE to 8-bit (or UTF-8)
- if (unicoding != ucr::UCS2BE)
- return false;
-
- try
- {
- // Init filedataIn struct and open file as memory mapped (input)
- TFile fileIn(filepath);
- SharedMemory shmIn(fileIn, SharedMemory::AM_READ);
-
- char * pszBuf = shmIn.begin();
- size_t nBufSize = shmIn.end() - shmIn.begin();
-
- // first pass : get the size of the destination file
- size_t nSizeOldBOM = 2;
- size_t nchars = (nBufSize - nSizeOldBOM) / sizeof(wchar_t);
-
- size_t nSizeBOM = (bWriteBOM) ? 3 : 0;
- DWORD flags =0;
- std::vector<char> szTemp(nBufSize);
-
- // simple byte swap BE->LE
- for (size_t i = 0; i < nBufSize; i += 2)
- {
- // Byte-swap into destination
- szTemp[i] = pszBuf[i + 1];
- szTemp[i + 1] = pszBuf[i];
- }
-
- //now convert to unicode UTF8
- size_t nDstSize = WideCharToMultiByte(CP_UTF8, flags,(wchar_t *)(&szTemp[0] + nSizeOldBOM), nchars, NULL, 0, NULL, NULL);
-
- // create the destination file
- TFile fileOut(filepathDst);
- fileOut.setSize(nDstSize + nSizeBOM);
- SharedMemory shmOut(fileOut, SharedMemory::AM_WRITE);
-
- // write BOM
- if (bWriteBOM)
- ucr::writeBom(shmOut.begin(), ucr::UTF8);
-
- // write data
- WideCharToMultiByte(CP_UTF8, flags,(wchar_t *)(&szTemp[0] + nSizeOldBOM), nchars,
- (shmOut.begin()+nSizeBOM), nDstSize, NULL, NULL);
-
- nFileChanged++;
- return true;
- }
- catch (...)
- {
- return false;
- }
-}
bool AnyCodepageToUTF8(int codepage, const String& filepath, const String& filepathDst, int & nFileChanged, bool bWriteBOM)
{
////////////////////////////////////////////////////////////////////////////////
-// for OLECHAR files, transform to UTF8 for diffutils
-// TODO : convert Ansi to UTF8 if other file is unicode or uses a different codepage
-bool FileTransform_UCS2ToUTF8(String & filepath, bool bMayOverwrite)
-{
- String tempDir = env_GetTempPath();
- if (tempDir.empty())
- return false;
- String tempFilepath = env_GetTempFileName(tempDir, _T("_W2"));
- if (tempFilepath.empty())
- return false;
-
- // TODO : is it better with the BOM or without (just change the last argument)
- int nFileChanged = 0;
- bool bSuccess = UCS2LEToUTF8(filepath, tempFilepath, nFileChanged, false);
- if (bSuccess && nFileChanged)
- {
- // we do not overwrite so we delete the old file
- if (bMayOverwrite)
- {
- try
- {
- TFile(filepath).remove();
- }
- catch (Exception& e)
- {
- LogErrorStringUTF8(e.displayText());
- }
- }
- // and change the filepath if everything works
- filepath = tempFilepath;
- }
- else
- {
- try
- {
- TFile(tempFilepath).remove();
- }
- catch (Exception& e)
- {
- LogErrorStringUTF8(e.displayText());
- }
- }
-
- return bSuccess;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
bool FileTransform_AnyCodepageToUTF8(int codepage, String & filepath, bool bMayOverwrite)
{
String tempDir = env_GetTempPath();
return (nChanged != 0);
}
-////////////////////////////////////////////////////////////////////////////////
-
-// for OLECHAR files, transform to UTF8 for diffutils
-// convert all Ansi or unicode-files to UTF8
-// if other file is unicode or uses a different codepage
-bool FileTransform_ToUTF8(String & filepath, bool bMayOverwrite)
-{
- bool bSuccess = false;
- String tempDir = env_GetTempPath();
- if (tempDir.empty())
- return false;
- String tempFilepath = env_GetTempFileName(tempDir, _T("_WM"));
- if (tempFilepath.empty())
- return false;
-
- int nFileChanged = 0;
-
- UniMemFile ufile;
- if (!ufile.OpenReadOnly(filepath))
- return false; // error
- ufile.IsUnicode();
- ucr::UNICODESET unicoding = ufile.GetUnicoding();
- // Finished with examing file contents
- ufile.Close();
-
- // TODO : is it better with the BOM or without (just change the last argument)
-
- switch (unicoding)
- {
- // AnsiToUTF16 only converts Ascii files to UTF-16
- case ( ucr::NONE):
- bSuccess = AnsiToUTF8(filepath, tempFilepath, nFileChanged, false);
- break;
- case (ucr::UCS2LE):
- bSuccess = UCS2LEToUTF8(filepath, tempFilepath, nFileChanged, false);
- break;
- case (ucr::UCS2BE):
- bSuccess = UCS2BEToUTF8(filepath, tempFilepath, nFileChanged, false);
- break;
- case (ucr::UCS4LE):
- bSuccess = false;
- break;
- case (ucr::UCS4BE):
- bSuccess = false;
- break;
-
- }
-
- if (!bSuccess)
- return false;
-
- if (nFileChanged)
- {
- // we do not overwrite so we delete the old file
- if (bMayOverwrite)
- {
- try
- {
- TFile(filepath).remove();
- }
- catch (Exception& e)
- {
- LogErrorStringUTF8(e.displayText());
- }
- }
- // and change the filepath if everything works
- filepath = tempFilepath;
- }
- else
- {
- try
- {
- TFile(tempFilepath).remove();
- }
- catch (Exception& e)
- {
- LogErrorStringUTF8(e.displayText());
- }
- }
- return true;
-}
-
-/**
- * @brief check for both are same (Ansi or UTF8)
- * otherwise convert both to UTF8
- * @param [in,out] filepath Most plugins change this filename
- */
-bool Transform2FilesToUTF8(String &strFile1Temp, String &strFile2Temp,bool m_bPathsAreTemp)
-{
- //check first file for unicodeing
- UniMemFile ufile;
- if (!ufile.OpenReadOnly(strFile1Temp))
- return false; // error
- ufile.IsUnicode();
- ucr::UNICODESET unicoding1 = ufile.GetUnicoding();
- // Finished with examing file contents
- ufile.Close();
-
- //check second file for unicodeing
- if (!ufile.OpenReadOnly(strFile2Temp))
- return false; // error
- ufile.IsUnicode();
-
- ucr::UNICODESET unicoding2 = ufile.GetUnicoding();
- ufile.Close();
-
- if ((unicoding1==ucr::NONE)&&(unicoding2==ucr::NONE))
- return true; //OK do nothing both are ansi
- if ((unicoding1==ucr::UTF8)&&(unicoding2==ucr::UTF8))
- return true; //OK do nothing both are UTF16
- // now both are different, convert to UTF8
- if (unicoding1!=ucr::UTF8)
- FileTransform_ToUTF8(strFile1Temp, m_bPathsAreTemp);
- if (unicoding2!=ucr::UTF8)
- FileTransform_ToUTF8(strFile2Temp, m_bPathsAreTemp);
- return true;
- }
////////////////////////////////////////////////////////////////////////////////
-
-// for OLECHAR files, transform to UTF8 for diffutils
-// TODO : convert Ansi to UTF8 if other file is unicode or uses a different codepage
-bool copyToUTF8(String & filepath, String &tempFilepath ,bool bMayOverwrite)
-{
- bool bSuccess = false;
-
- UniMemFile ufile;
- if (!ufile.OpenReadOnly(filepath))
- return false; // error
- ufile.IsUnicode();
- ucr::UNICODESET unicoding = ufile.GetUnicoding();
- // Finished with examing file contents
- ufile.Close();
-
- int nFileChanged = 0;
-
- // TODO : is it better with the BOM or without (just change the last argument)
-
- switch (unicoding)
- {
- // AnsiToUTF16 only converts Ascii files to UTF-16
- case ( ucr::NONE):
- bSuccess = AnsiToUTF8(filepath, tempFilepath, nFileChanged, false);
- break;
- case (ucr::UCS2LE):
- bSuccess = UCS2LEToUTF8(filepath, tempFilepath, nFileChanged, false);
- break;
- case (ucr::UCS2BE):
- bSuccess = UCS2BEToUTF8(filepath, tempFilepath, nFileChanged, false);
- break;
- case (ucr::UCS4LE):
- bSuccess = false;
- break;
- case (ucr::UCS4BE):
- bSuccess = false;
- break;
-
- }
-
- if (!bSuccess)
- return false;
-
- return true;
-}
OSDN Git Service
