2 * Copyright (C) 2004, 2006, 2008 Apple Inc. All rights reserved.
3 * Copyright (C) 2006 Alexey Proskuryakov <ap@nypop.com>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
15 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
22 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include "TextCodecMac.h"
30 #include "CharsetData.h"
31 #include "PlatformString.h"
32 #include "ThreadGlobalData.h"
33 #include <wtf/Assertions.h>
34 #include <wtf/PassOwnPtr.h>
35 #include <wtf/RetainPtr.h>
36 #include <wtf/Threading.h>
37 #include <wtf/text/CString.h>
38 #include <wtf/unicode/CharacterNames.h>
44 // We need to keep this because ICU doesn't support some of the encodings that we need:
45 // <http://bugs.webkit.org/show_bug.cgi?id=4195>.
47 const size_t ConversionBufferSize = 16384;
49 static TECConverterWrapper& cachedConverterTEC()
51 return threadGlobalData().cachedConverterTEC();
54 void TextCodecMac::registerEncodingNames(EncodingNameRegistrar registrar)
56 TECTextEncodingID lastEncoding = invalidEncoding;
57 const char* lastName = 0;
59 for (size_t i = 0; CharsetTable[i].name; ++i) {
60 if (CharsetTable[i].encoding != lastEncoding) {
61 lastEncoding = CharsetTable[i].encoding;
62 lastName = CharsetTable[i].name;
64 registrar(CharsetTable[i].name, lastName);
68 static PassOwnPtr<TextCodec> newTextCodecMac(const TextEncoding&, const void* additionalData)
70 return new TextCodecMac(*static_cast<const TECTextEncodingID*>(additionalData));
73 void TextCodecMac::registerCodecs(TextCodecRegistrar registrar)
75 TECTextEncodingID lastEncoding = invalidEncoding;
77 for (size_t i = 0; CharsetTable[i].name; ++i)
78 if (CharsetTable[i].encoding != lastEncoding) {
79 registrar(CharsetTable[i].name, newTextCodecMac, &CharsetTable[i].encoding);
80 lastEncoding = CharsetTable[i].encoding;
84 TextCodecMac::TextCodecMac(TECTextEncodingID encoding)
85 : m_encoding(encoding)
86 , m_numBufferedBytes(0)
91 TextCodecMac::~TextCodecMac()
93 releaseTECConverter();
96 void TextCodecMac::releaseTECConverter() const
99 TECConverterWrapper& cachedConverter = cachedConverterTEC();
100 if (cachedConverter.converter)
101 TECDisposeConverter(cachedConverter.converter);
102 cachedConverter.converter = m_converterTEC;
103 cachedConverter.encoding = m_encoding;
108 OSStatus TextCodecMac::createTECConverter() const
110 TECConverterWrapper& cachedConverter = cachedConverterTEC();
112 bool cachedEncodingEqual = cachedConverter.encoding == m_encoding;
113 cachedConverter.encoding = invalidEncoding;
115 if (cachedEncodingEqual && cachedConverter.converter) {
116 m_converterTEC = cachedConverter.converter;
117 cachedConverter.converter = 0;
119 TECClearConverterContextInfo(m_converterTEC);
121 OSStatus status = TECCreateConverter(&m_converterTEC, m_encoding,
122 CreateTextEncoding(kTextEncodingUnicodeDefault, kTextEncodingDefaultVariant, kUnicode16BitFormat));
126 TECSetBasicOptions(m_converterTEC, kUnicodeForceASCIIRangeMask);
132 OSStatus TextCodecMac::decode(const unsigned char* inputBuffer, int inputBufferLength, int& inputLength,
133 void *outputBuffer, int outputBufferLength, int& outputLength)
136 unsigned long bytesRead = 0;
137 unsigned long bytesWritten = 0;
139 if (m_numBufferedBytes != 0) {
140 // Finish converting a partial character that's in our buffer.
142 // First, fill the partial character buffer with as many bytes as are available.
143 ASSERT(m_numBufferedBytes < sizeof(m_bufferedBytes));
144 const int spaceInBuffer = sizeof(m_bufferedBytes) - m_numBufferedBytes;
145 const int bytesToPutInBuffer = min(spaceInBuffer, inputBufferLength);
146 ASSERT(bytesToPutInBuffer != 0);
147 memcpy(m_bufferedBytes + m_numBufferedBytes, inputBuffer, bytesToPutInBuffer);
149 // Now, do a conversion on the buffer.
150 status = TECConvertText(m_converterTEC, m_bufferedBytes, m_numBufferedBytes + bytesToPutInBuffer, &bytesRead,
151 reinterpret_cast<unsigned char*>(outputBuffer), outputBufferLength, &bytesWritten);
152 ASSERT(bytesRead <= m_numBufferedBytes + bytesToPutInBuffer);
154 if (status == kTECPartialCharErr && bytesRead == 0) {
155 // Handle the case where the partial character was not converted.
156 if (bytesToPutInBuffer >= spaceInBuffer) {
157 LOG_ERROR("TECConvertText gave a kTECPartialCharErr but read none of the %zu bytes in the buffer", sizeof(m_bufferedBytes));
158 m_numBufferedBytes = 0;
159 status = kTECUnmappableElementErr; // should never happen, but use this error code
161 // Tell the caller we read all the source bytes and keep them in the buffer.
162 m_numBufferedBytes += bytesToPutInBuffer;
163 bytesRead = bytesToPutInBuffer;
167 // We are done with the partial character buffer.
168 // Also, we have read some of the bytes from the main buffer.
169 if (bytesRead > m_numBufferedBytes) {
170 bytesRead -= m_numBufferedBytes;
172 LOG_ERROR("TECConvertText accepted some bytes it previously rejected with kTECPartialCharErr");
175 m_numBufferedBytes = 0;
176 if (status == kTECPartialCharErr) {
177 // While there may be a partial character problem in the small buffer,
178 // we have to try again and not get confused and think there is a partial
179 // character problem in the large buffer.
184 status = TECConvertText(m_converterTEC, inputBuffer, inputBufferLength, &bytesRead,
185 static_cast<unsigned char*>(outputBuffer), outputBufferLength, &bytesWritten);
186 ASSERT(static_cast<int>(bytesRead) <= inputBufferLength);
189 // Work around bug 3351093, where sometimes we get kTECBufferBelowMinimumSizeErr instead of kTECOutputBufferFullStatus.
190 if (status == kTECBufferBelowMinimumSizeErr && bytesWritten != 0)
191 status = kTECOutputBufferFullStatus;
193 inputLength = bytesRead;
194 outputLength = bytesWritten;
198 String TextCodecMac::decode(const char* bytes, size_t length, bool flush, bool stopOnError, bool& sawError)
200 // Get a converter for the passed-in encoding.
201 if (!m_converterTEC && createTECConverter() != noErr)
204 Vector<UChar> result;
206 const unsigned char* sourcePointer = reinterpret_cast<const unsigned char*>(bytes);
207 int sourceLength = length;
208 bool bufferWasFull = false;
209 UniChar buffer[ConversionBufferSize];
211 while ((sourceLength || bufferWasFull) && !sawError) {
213 int bytesWritten = 0;
214 OSStatus status = decode(sourcePointer, sourceLength, bytesRead, buffer, sizeof(buffer), bytesWritten);
215 ASSERT(bytesRead <= sourceLength);
216 sourcePointer += bytesRead;
217 sourceLength -= bytesRead;
221 case kTECOutputBufferFullStatus:
223 case kTextMalformedInputErr:
224 case kTextUndefinedElementErr:
225 // FIXME: Put FFFD character into the output string in this case?
226 TECClearConverterContextInfo(m_converterTEC);
236 case kTECPartialCharErr: {
237 // Put the partial character into the buffer.
238 ASSERT(m_numBufferedBytes == 0);
239 const int bufferSize = sizeof(m_numBufferedBytes);
240 if (sourceLength < bufferSize) {
241 memcpy(m_bufferedBytes, sourcePointer, sourceLength);
242 m_numBufferedBytes = sourceLength;
244 LOG_ERROR("TECConvertText gave a kTECPartialCharErr, but left %u bytes in the buffer", sourceLength);
254 ASSERT(!(bytesWritten % sizeof(UChar)));
255 result.append(buffer, bytesWritten / sizeof(UChar));
257 bufferWasFull = status == kTECOutputBufferFullStatus;
261 unsigned long bytesWritten = 0;
262 TECFlushText(m_converterTEC, reinterpret_cast<unsigned char*>(buffer), sizeof(buffer), &bytesWritten);
263 ASSERT(!(bytesWritten % sizeof(UChar)));
264 result.append(buffer, bytesWritten / sizeof(UChar));
267 String resultString = String::adopt(result);
269 // <rdar://problem/3225472>
270 // Simplified Chinese pages use the code A3A0 to mean "full-width space".
271 // But GB18030 decodes it to U+E5E5, which is correct in theory but not in practice.
272 // To work around, just change all occurences of U+E5E5 to U+3000 (ideographic space).
273 if (m_encoding == kCFStringEncodingGB_18030_2000)
274 resultString.replace(0xE5E5, ideographicSpace);
279 CString TextCodecMac::encode(const UChar* characters, size_t length, UnencodableHandling handling)
281 // FIXME: We should really use TEC here instead of CFString for consistency with the other direction.
283 // FIXME: Since there's no "force ASCII range" mode in CFString, we change the backslash into a yen sign.
284 // Encoding will change the yen sign back into a backslash.
285 String copy(characters, length);
286 copy.replace('\\', m_backslashAsCurrencySymbol);
287 RetainPtr<CFStringRef> cfs(AdoptCF, copy.createCFString());
289 CFIndex startPos = 0;
290 CFIndex charactersLeft = CFStringGetLength(cfs.get());
293 UInt8 lossByte = handling == QuestionMarksForUnencodables ? '?' : 0;
294 while (charactersLeft > 0) {
295 CFRange range = CFRangeMake(startPos, charactersLeft);
296 CFIndex bufferLength;
297 CFStringGetBytes(cfs.get(), range, m_encoding, lossByte, false, NULL, 0x7FFFFFFF, &bufferLength);
299 result.grow(size + bufferLength);
300 unsigned char* buffer = reinterpret_cast<unsigned char*>(result.data() + size);
301 CFIndex charactersConverted = CFStringGetBytes(cfs.get(), range, m_encoding, lossByte, false, buffer, bufferLength, &bufferLength);
302 size += bufferLength;
304 if (charactersConverted != charactersLeft) {
305 unsigned badChar = CFStringGetCharacterAtIndex(cfs.get(), startPos + charactersConverted);
306 ++charactersConverted;
307 if ((badChar & 0xFC00) == 0xD800 && charactersConverted != charactersLeft) { // is high surrogate
308 UniChar low = CFStringGetCharacterAtIndex(cfs.get(), startPos + charactersConverted);
309 if ((low & 0xFC00) == 0xDC00) { // is low surrogate
312 badChar += 0x10000 - (0xD800 << 10) - 0xDC00;
313 ++charactersConverted;
316 UnencodableReplacementArray entity;
317 int entityLength = getUnencodableReplacement(badChar, handling, entity);
318 result.grow(size + entityLength);
319 memcpy(result.data() + size, entity, entityLength);
320 size += entityLength;
323 startPos += charactersConverted;
324 charactersLeft -= charactersConverted;
326 return CString(result.data(), size);
329 } // namespace WebCore