vendor/golang.org/x/text/encoding/simplifiedchinese/all_test.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package simplifiedchinese
   6
   7 import (
   8         "strings"
   9         "testing"
  10
  11         "golang.org/x/text/encoding"
  12         "golang.org/x/text/encoding/internal"
  13         "golang.org/x/text/encoding/internal/enctest"
  14         "golang.org/x/text/transform"
  15 )
  16
  17 func dec(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
  18         return "Decode", e.NewDecoder(), nil
  19 }
  20 func enc(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
  21         return "Encode", e.NewEncoder(), internal.ErrASCIIReplacement
  22 }
  23
  24 func TestNonRepertoire(t *testing.T) {
  25         // Pick n large enough to overflow the destination buffer of transform.String.
  26         const n = 10000
  27         testCases := []struct {
  28                 init      func(e encoding.Encoding) (string, transform.Transformer, error)
  29                 e         encoding.Encoding
  30                 src, want string
  31         }{
  32                 {dec, GBK, "a\xfe\xfeb", "a\ufffdb"},
  33                 {dec, HZGB2312, "~{z~", "\ufffd"},
  34
  35                 {enc, GBK, "갂", ""},
  36                 {enc, GBK, "a갂", "a"},
  37                 {enc, GBK, "\u4e02갂", "\x81@"},
  38
  39                 {enc, HZGB2312, "갂", ""},
  40                 {enc, HZGB2312, "a갂", "a"},
  41                 {enc, HZGB2312, "\u6cf5갂", "~{1C~}"},
  42
  43                 {dec, GB18030, "\x80", "€"},
  44                 {dec, GB18030, "\x81", "\ufffd"},
  45                 {dec, GB18030, "\x81\x20", "\ufffd "},
  46                 {dec, GB18030, "\xfe\xfe", "\ufffd"},
  47                 {dec, GB18030, "\xfe\xff", "\ufffd\ufffd"},
  48                 {dec, GB18030, "\xfe\x30", "\ufffd0"},
  49                 {dec, GB18030, "\xfe\x30\x30 ", "\ufffd00 "},
  50                 {dec, GB18030, "\xfe\x30\xff ", "\ufffd0\ufffd "},
  51                 {dec, GB18030, "\xfe\x30\x81\x21", "\ufffd0\ufffd!"},
  52
  53                 {dec, GB18030, strings.Repeat("\xfe\x30", n), strings.Repeat("\ufffd0", n)},
  54
  55                 {dec, HZGB2312, "~/", "\ufffd"},
  56                 {dec, HZGB2312, "~{a\x80", "\ufffd"},
  57                 {dec, HZGB2312, "~{a\x80", "\ufffd"},
  58                 {dec, HZGB2312, "~{" + strings.Repeat("z~", n), strings.Repeat("\ufffd", n)},
  59                 {dec, HZGB2312, "~{" + strings.Repeat("\xfe\x30", n), strings.Repeat("\ufffd", n*2)},
  60         }
  61         for _, tc := range testCases {
  62                 dir, tr, wantErr := tc.init(tc.e)
  63
  64                 dst, _, err := transform.String(tr, tc.src)
  65                 if err != wantErr {
  66                         t.Errorf("%s %v(%q): got %v; want %v", dir, tc.e, tc.src, err, wantErr)
  67                 }
  68                 if got := string(dst); got != tc.want {
  69                         t.Errorf("%s %v(%q):\ngot  %q\nwant %q", dir, tc.e, tc.src, got, tc.want)
  70                 }
  71         }
  72 }
  73
  74 func TestBasics(t *testing.T) {
  75         // The encoded forms can be verified by the iconv program:
  76         // $ echo 月日は百代 | iconv -f UTF-8 -t SHIFT-JIS | xxd
  77         testCases := []struct {
  78                 e         encoding.Encoding
  79                 encPrefix string
  80                 encoded   string
  81                 utf8      string
  82         }{{
  83                 // "\u0081\u00de\u00df\u00e0\u00e1\u00e2\u00e3\uffff\U00010000" is a
  84                 // nonsense string that contains GB18030 encodable codepoints of which
  85                 // only U+00E0 and U+00E1 are GBK encodable.
  86                 //
  87                 // "A\u3000\u554a\u4e02\u4e90\u72dc\u7349\u02ca\u2588Z€" is a nonsense
  88                 // string that contains ASCII and GBK encodable codepoints from Levels
  89                 // 1-5 as well as the Euro sign.
  90                 //
  91                 // "A\u43f0\u4c32\U00027267\u3000\U0002910d\u79d4Z€" is a nonsense string
  92                 // that contains ASCII and Big5 encodable codepoints from the Basic
  93                 // Multilingual Plane and the Supplementary Ideographic Plane as well as
  94                 // the Euro sign.
  95                 //
  96                 // "花间一壶酒，独酌无相亲。" (simplified) and
  97                 // "花間一壺酒，獨酌無相親。" (traditional)
  98                 // are from the 8th century poem "Yuè Xià Dú Zhuó".
  99                 e: GB18030,
 100                 encoded: "\x81\x30\x81\x31\x81\x30\x89\x37\x81\x30\x89\x38\xa8\xa4\xa8\xa2" +
 101                         "\x81\x30\x89\x39\x81\x30\x8a\x30\x84\x31\xa4\x39\x90\x30\x81\x30",
 102                 utf8: "\u0081\u00de\u00df\u00e0\u00e1\u00e2\u00e3\uffff\U00010000",
 103         }, {
 104                 e: GB18030,
 105                 encoded: "\xbb\xa8\xbc\xe4\xd2\xbb\xba\xf8\xbe\xc6\xa3\xac\xb6\xc0\xd7\xc3" +
 106                         "\xce\xde\xcf\xe0\xc7\xd7\xa1\xa3",
 107                 utf8: "花间一壶酒，独酌无相亲。",
 108         }, {
 109                 e:       GBK,
 110                 encoded: "A\xa1\xa1\xb0\xa1\x81\x40\x81\x80\xaa\x40\xaa\x80\xa8\x40\xa8\x80Z\x80",
 111                 utf8:    "A\u3000\u554a\u4e02\u4e90\u72dc\u7349\u02ca\u2588Z€",
 112         }, {
 113                 e: GBK,
 114                 encoded: "\xbb\xa8\xbc\xe4\xd2\xbb\xba\xf8\xbe\xc6\xa3\xac\xb6\xc0\xd7\xc3" +
 115                         "\xce\xde\xcf\xe0\xc7\xd7\xa1\xa3",
 116                 utf8: "花间一壶酒，独酌无相亲。",
 117         }, {
 118                 e:       HZGB2312,
 119                 encoded: "A~{\x21\x21~~\x30\x21~}Z~~",
 120                 utf8:    "A\u3000~\u554aZ~",
 121         }, {
 122                 e:         HZGB2312,
 123                 encPrefix: "~{",
 124                 encoded:   ";(<dR;:x>F#,6@WCN^O`GW!#",
 125                 utf8:      "花间一壶酒，独酌无相亲。",
 126         }}
 127
 128         for _, tc := range testCases {
 129                 enctest.TestEncoding(t, tc.e, tc.encoded, tc.utf8, tc.encPrefix, "")
 130         }
 131 }
 132
 133 func TestFiles(t *testing.T) {
 134         enctest.TestFile(t, GB18030)
 135         enctest.TestFile(t, GBK)
 136         enctest.TestFile(t, HZGB2312)
 137 }
 138
 139 func BenchmarkEncoding(b *testing.B) {
 140         enctest.Benchmark(b, GB18030)
 141         enctest.Benchmark(b, GBK)
 142         enctest.Benchmark(b, HZGB2312)
 143 }