write BOM only from QTextStream, on demand

ICU finally does not generate BOMs (since idk when), I guess someone took
note. chopping BOMs from the result may have to be done only if the input
data had BOM (for compat) but let's see if that is even needed

Kate encoding tests pass now btw

Signed-off-by: Ivailo Monev <xakepa10@gmail.com>

diff --git a/src/core/codecs/qtextcodec.cpp b/src/core/codecs/qtextcodec.cpp
index 509bae4..8e325d6 100644 (file)
--- a/src/core/codecs/qtextcodec.cpp
+++ b/src/core/codecs/qtextcodec.cpp
@@ -1832,7 +1832,6 @@ QTextCodec *QTextCodec::codecForText(const QByteArray &ba)
     \value DefaultConversion  No flag is set.
     \value ConvertInvalidToNull  If this flag is set, each invalid input
                                  character is output as a null character.
-    \value IgnoreHeader  Ignore any Unicode byte-order mark and don't generate any.
 */
 
 /*!
@@ -1958,7 +1957,6 @@ QByteArray QTextConverter::fromUnicode(const QChar *data, int length) const
     UErrorCode error = U_ZERO_ERROR;
     const int convresult = ucnv_fromUChars(conv, result, maxbytes,
         reinterpret_cast<const UChar *>(data), length, &error);
-
     return QByteArray(result, convresult);
 }
 
@@ -1976,21 +1974,9 @@ QString QTextConverter::toUnicode(const char *data, int length) const
 
     const int maxbytes = UCNV_GET_MAX_BYTES_FOR_STRING(length, ucnv_getMaxCharSize(conv));
     QSTACKARRAY(UChar, result, maxbytes);
-    int resultoffset = 0;
     UErrorCode error = U_ZERO_ERROR;
     const int convresult = ucnv_toUChars(conv, result, maxbytes, data, length, &error);
-
-    if (d_ptr->flags & QTextConverter::IgnoreHeader) {
-        // ICU always generates BOMs when converter is UTF-32/UTF-16 so no check is done to
-        // verify that, unless someone makes it an option
-        if (qstrnicmp("UTF-32", d_ptr->name.constData(), 6) == 0) {
-            resultoffset = 4;
-        } else if (qstrnicmp("UTF-16", d_ptr->name.constData(), 6) == 0) {
-            resultoffset = 2;
-        }
-    }
-
-    return QString(reinterpret_cast<QChar*>(result) + resultoffset, convresult - resultoffset);
+    return QString(reinterpret_cast<QChar*>(result), convresult);
 }
 
 #endif // QT_NO_TEXTCODEC

diff --git a/src/core/codecs/qtextcodec.h b/src/core/codecs/qtextcodec.h
index 9c27c7a..a5ad731 100644 (file)
--- a/src/core/codecs/qtextcodec.h
+++ b/src/core/codecs/qtextcodec.h
@@ -36,8 +36,7 @@ class Q_CORE_EXPORT QTextConverter
 public:
     enum ConversionFlag {
         DefaultConversion = 0x0,
-        IgnoreHeader = 0x1,
-        ConvertInvalidToNull = 0x2
+        ConvertInvalidToNull = 0x1
     };
     Q_DECLARE_FLAGS(ConversionFlags, ConversionFlag)
 

diff --git a/src/core/io/qtextstream.cpp b/src/core/io/qtextstream.cpp
index 9d834b5..2057ccd 100644 (file)
--- a/src/core/io/qtextstream.cpp
+++ b/src/core/io/qtextstream.cpp
@@ -337,6 +337,7 @@ public:
     inline void write(const QString &data);
     inline void putString(const QString &ch, bool number = false);
     void putNumber(qulonglong number, bool negative);
+    void writeBOM();
 
     // buffers
     bool fillReadBuffer(qint64 maxBytes = -1);
@@ -359,6 +360,8 @@ public:
 
     // status
     QTextStream::TextStatus status;
+    bool generatebom;
+    bool bomwritten;
 
     QLocale locale;
 };
@@ -409,13 +412,13 @@ void QTextStreamPrivate::reset()
 #ifndef QT_NO_TEXTCODEC
     codec = QTextCodec::codecForLocale();
     readConverter = codec->converter();
-    readConverter.setFlags(QTextConverter::DefaultConversion);
     writeConverter = codec->converter();
-    writeConverter.setFlags(QTextConverter::IgnoreHeader);
     readConverterSaved.reset();
-    readConverterSaved.setFlags(QTextConverter::DefaultConversion);
     autoDetectUnicode = true;
 #endif
+
+    generatebom = false;
+    bomwritten = false;
 }
 
 /*! \internal
@@ -447,7 +450,6 @@ bool QTextStreamPrivate::fillReadBuffer(qint64 maxBytes)
         codec = QTextCodec::codecForUtfText(buffer, codec);
         if (!codec) {
             codec = QTextCodec::codecForLocale();
-            writeConverter.setFlags(writeConverter.flags() | QTextConverter::IgnoreHeader);
         }
     }
 #if defined (QTEXTSTREAM_DEBUG)
@@ -545,7 +547,7 @@ void QTextStreamPrivate::flushWriteBuffer()
         codec = QTextCodec::codecForLocale();
 #if defined (QTEXTSTREAM_DEBUG)
     qDebug("QTextStreamPrivate::flushWriteBuffer(), using %s codec (%s generating BOM)",
-           codec->name().constData(), writeConverter.flags() & QTextConverter::IgnoreHeader ? "not" : "");
+           codec->name().constData(), generatebom ? "not" : "");
 #endif
 
     // convert from unicode to raw data
@@ -771,6 +773,10 @@ inline void QTextStreamPrivate::restoreToSavedConverterState()
 */
 inline void QTextStreamPrivate::write(const QString &data)
 {
+    if (generatebom && !bomwritten) {
+        writeBOM();
+    }
+
     if (string) {
         // ### What about seek()??
         string->append(data);
@@ -1052,11 +1058,8 @@ bool QTextStream::seek(qint64 pos)
 #ifndef QT_NO_TEXTCODEC
         // Reset the codec converter states.
         d->readConverter.reset();
-        d->readConverter.setFlags(QTextConverter::DefaultConversion);
         d->writeConverter.reset();
-        d->writeConverter.setFlags(QTextConverter::IgnoreHeader);
         d->readConverterSaved.reset();
-        d->readConverterSaved.setFlags(QTextConverter::DefaultConversion);
 #endif
         return true;
     }
@@ -2177,6 +2180,59 @@ void QTextStreamPrivate::putNumber(qulonglong number, bool negative)
 }
 
 /*!
+    \internal
+ */
+void QTextStreamPrivate::writeBOM()
+{
+    Q_ASSERT(generatebom);
+    Q_ASSERT(!bomwritten);
+
+#ifndef QT_NO_TEXTCODEC
+    bomwritten = true;
+    const QByteArray codecname = codec->name();
+    const uchar* bomptr = nullptr;
+    int bomptrsize = 0;
+    if (qstricmp("UTF-32BE", codecname.constData()) == 0) {
+        bomptr = q_utf32be_bom;
+        bomptrsize = sizeof(q_utf32be_bom);
+    } else if (qstricmp("UTF-32LE", codecname.constData()) == 0) {
+        bomptr = q_utf32le_bom;
+        bomptrsize = sizeof(q_utf32le_bom);
+    } else if (qstricmp("UTF-16BE", codecname.constData()) == 0) {
+        bomptr = q_utf16be_bom;
+        bomptrsize = sizeof(q_utf16be_bom);
+    } else if (qstricmp("UTF-16LE", codecname.constData()) == 0) {
+        bomptr = q_utf16le_bom;
+        bomptrsize = sizeof(q_utf16le_bom);
+    } else if (qstricmp("UTF-32", codecname.constData()) == 0) {
+#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN
+        bomptr = q_utf32le_bom;
+        bomptrsize = sizeof(q_utf32le_bom);
+#else
+        bomptr = q_utf32be_bom;
+        bomptrsize = sizeof(q_utf32be_bom);
+#endif
+    } else if (qstricmp("UTF-16", codecname.constData()) == 0) {
+#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN
+        bomptr = q_utf16le_bom;
+        bomptrsize = sizeof(q_utf16le_bom);
+#else
+        bomptr = q_utf16be_bom;
+        bomptrsize = sizeof(q_utf16be_bom);
+#endif
+    }
+
+    if (bomptr && bomptrsize) {
+        if (string) {
+            string->append(QString::fromRawData(reinterpret_cast<const QChar*>(bomptr), bomptrsize / sizeof(QChar)));
+        } else {
+            device->write(QByteArray::fromRawData(reinterpret_cast<const char*>(bomptr), bomptrsize));
+        }
+    }
+#endif // QT_NO_TEXTCODEC
+}
+
+/*!
     Writes the character \a c to the stream, then returns a reference
     to the QTextStream.
 
@@ -2527,10 +2583,7 @@ void QTextStream::setGenerateByteOrderMark(bool generate)
 {
     Q_D(QTextStream);
     if (d->writeBuffer.isEmpty()) {
-        if (generate)
-            d->writeConverter.setFlags(d->writeConverter.flags() & ~QTextConverter::IgnoreHeader);
-        else
-            d->writeConverter.setFlags(d->writeConverter.flags() | QTextConverter::IgnoreHeader);
+        d->generatebom = generate;
     }
 }
 
@@ -2544,9 +2597,8 @@ void QTextStream::setGenerateByteOrderMark(bool generate)
 bool QTextStream::generateByteOrderMark() const
 {
     Q_D(const QTextStream);
-    return (d->writeConverter.flags() & QTextConverter::IgnoreHeader) == 0;
+    return d->generatebom;
 }
-
 #endif
 
 /*!

diff --git a/src/core/qcorecommon_p.h b/src/core/qcorecommon_p.h
index 313598e..69b9e2c 100644 (file)
--- a/src/core/qcorecommon_p.h
+++ b/src/core/qcorecommon_p.h
@@ -29,6 +29,13 @@ QT_BEGIN_NAMESPACE
 static const qreal q_deg2rad = qreal(0.01745329251994329576923690768489); /* pi/180 */
 static const qreal q_rad2deg = qreal(57.295779513082320876798154814105); /* 180/pi */
 
+#ifndef QT_NO_TEXTCODEC
+static const uchar q_utf32le_bom[] = { 0xff, 0xfe, 0x00, 0x00 }; // MIB 1019
+static const uchar q_utf32be_bom[] = { 0x00, 0x00, 0xfe, 0xff }; // MIB 1018
+static const uchar q_utf16le_bom[] = { 0xff, 0xfe }; // MIB 1014
+static const uchar q_utf16be_bom[] = { 0xfe, 0xff }; // MIB 1013
+#endif
+
 static inline uint foldCase(const ushort *ch, const ushort *start)
 {
     uint c = *ch;

diff --git a/src/xml/kernel/qxmlstream.cpp b/src/xml/kernel/qxmlstream.cpp
index 71cd629..8945ef0 100644 (file)
--- a/src/xml/kernel/qxmlstream.cpp
+++ b/src/xml/kernel/qxmlstream.cpp
@@ -2836,7 +2836,6 @@ QXmlStreamWriterPrivate::QXmlStreamWriterPrivate()
 #ifndef QT_NO_TEXTCODEC
     codec = QTextCodec::codecForMib(106); // utf8
     encoder = new QTextConverter(codec->name());
-    encoder->setFlags(QTextConverter::IgnoreHeader); // no byte order mark for utf8
 #endif
     checkIfASCIICompatibleCodec();
     inStartElement = inEmptyElement = false;
@@ -3124,7 +3123,6 @@ void QXmlStreamWriter::setCodec(QTextCodec *codec)
         d->codec = codec;
         delete d->encoder;
         d->encoder = new QTextConverter(codec->name());
-        d->encoder->setFlags(QTextConverter::IgnoreHeader); // no byte order mark for utf8
         d->checkIfASCIICompatibleCodec();
     }
 }