--- /dev/null
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+ "bytes"
+ "testing"
+
+ "golang.org/x/text/internal/colltab"
+ "golang.org/x/text/language"
+)
+
+type weightsTest struct {
+ opt opts
+ in, out ColElems
+}
+
+type opts struct {
+ lev int
+ alt alternateHandling
+ top int
+
+ backwards bool
+ caseLevel bool
+}
+
+// ignore returns an initialized boolean array based on the given Level.
+// A negative value means using the default setting of quaternary.
+func ignore(level colltab.Level) (ignore [colltab.NumLevels]bool) {
+ if level < 0 {
+ level = colltab.Quaternary
+ }
+ for i := range ignore {
+ ignore[i] = level < colltab.Level(i)
+ }
+ return ignore
+}
+
+func makeCE(w []int) colltab.Elem {
+ ce, err := colltab.MakeElem(w[0], w[1], w[2], uint8(w[3]))
+ if err != nil {
+ panic(err)
+ }
+ return ce
+}
+
+func (o opts) collator() *Collator {
+ c := &Collator{
+ options: options{
+ ignore: ignore(colltab.Level(o.lev - 1)),
+ alternate: o.alt,
+ backwards: o.backwards,
+ caseLevel: o.caseLevel,
+ variableTop: uint32(o.top),
+ },
+ }
+ return c
+}
+
+const (
+ maxQ = 0x1FFFFF
+)
+
+func wpq(p, q int) Weights {
+ return W(p, defaults.Secondary, defaults.Tertiary, q)
+}
+
+func wsq(s, q int) Weights {
+ return W(0, s, defaults.Tertiary, q)
+}
+
+func wq(q int) Weights {
+ return W(0, 0, 0, q)
+}
+
+var zero = W(0, 0, 0, 0)
+
+var processTests = []weightsTest{
+ // Shifted
+ { // simple sequence of non-variables
+ opt: opts{alt: altShifted, top: 100},
+ in: ColElems{W(200), W(300), W(400)},
+ out: ColElems{wpq(200, maxQ), wpq(300, maxQ), wpq(400, maxQ)},
+ },
+ { // first is a variable
+ opt: opts{alt: altShifted, top: 250},
+ in: ColElems{W(200), W(300), W(400)},
+ out: ColElems{wq(200), wpq(300, maxQ), wpq(400, maxQ)},
+ },
+ { // all but first are variable
+ opt: opts{alt: altShifted, top: 999},
+ in: ColElems{W(1000), W(200), W(300), W(400)},
+ out: ColElems{wpq(1000, maxQ), wq(200), wq(300), wq(400)},
+ },
+ { // first is a modifier
+ opt: opts{alt: altShifted, top: 999},
+ in: ColElems{W(0, 10), W(1000)},
+ out: ColElems{wsq(10, maxQ), wpq(1000, maxQ)},
+ },
+ { // primary ignorables
+ opt: opts{alt: altShifted, top: 250},
+ in: ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+ out: ColElems{wq(200), zero, wpq(300, maxQ), wsq(15, maxQ), wpq(400, maxQ)},
+ },
+ { // secondary ignorables
+ opt: opts{alt: altShifted, top: 250},
+ in: ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+ out: ColElems{wq(200), zero, wpq(300, maxQ), W(0, 0, 15, maxQ), wpq(400, maxQ)},
+ },
+ { // tertiary ignorables, no change
+ opt: opts{alt: altShifted, top: 250},
+ in: ColElems{W(200), zero, W(300), zero, W(400)},
+ out: ColElems{wq(200), zero, wpq(300, maxQ), zero, wpq(400, maxQ)},
+ },
+
+ // ShiftTrimmed (same as Shifted)
+ { // simple sequence of non-variables
+ opt: opts{alt: altShiftTrimmed, top: 100},
+ in: ColElems{W(200), W(300), W(400)},
+ out: ColElems{wpq(200, maxQ), wpq(300, maxQ), wpq(400, maxQ)},
+ },
+ { // first is a variable
+ opt: opts{alt: altShiftTrimmed, top: 250},
+ in: ColElems{W(200), W(300), W(400)},
+ out: ColElems{wq(200), wpq(300, maxQ), wpq(400, maxQ)},
+ },
+ { // all but first are variable
+ opt: opts{alt: altShiftTrimmed, top: 999},
+ in: ColElems{W(1000), W(200), W(300), W(400)},
+ out: ColElems{wpq(1000, maxQ), wq(200), wq(300), wq(400)},
+ },
+ { // first is a modifier
+ opt: opts{alt: altShiftTrimmed, top: 999},
+ in: ColElems{W(0, 10), W(1000)},
+ out: ColElems{wsq(10, maxQ), wpq(1000, maxQ)},
+ },
+ { // primary ignorables
+ opt: opts{alt: altShiftTrimmed, top: 250},
+ in: ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+ out: ColElems{wq(200), zero, wpq(300, maxQ), wsq(15, maxQ), wpq(400, maxQ)},
+ },
+ { // secondary ignorables
+ opt: opts{alt: altShiftTrimmed, top: 250},
+ in: ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+ out: ColElems{wq(200), zero, wpq(300, maxQ), W(0, 0, 15, maxQ), wpq(400, maxQ)},
+ },
+ { // tertiary ignorables, no change
+ opt: opts{alt: altShiftTrimmed, top: 250},
+ in: ColElems{W(200), zero, W(300), zero, W(400)},
+ out: ColElems{wq(200), zero, wpq(300, maxQ), zero, wpq(400, maxQ)},
+ },
+
+ // Blanked
+ { // simple sequence of non-variables
+ opt: opts{alt: altBlanked, top: 100},
+ in: ColElems{W(200), W(300), W(400)},
+ out: ColElems{W(200), W(300), W(400)},
+ },
+ { // first is a variable
+ opt: opts{alt: altBlanked, top: 250},
+ in: ColElems{W(200), W(300), W(400)},
+ out: ColElems{zero, W(300), W(400)},
+ },
+ { // all but first are variable
+ opt: opts{alt: altBlanked, top: 999},
+ in: ColElems{W(1000), W(200), W(300), W(400)},
+ out: ColElems{W(1000), zero, zero, zero},
+ },
+ { // first is a modifier
+ opt: opts{alt: altBlanked, top: 999},
+ in: ColElems{W(0, 10), W(1000)},
+ out: ColElems{W(0, 10), W(1000)},
+ },
+ { // primary ignorables
+ opt: opts{alt: altBlanked, top: 250},
+ in: ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+ out: ColElems{zero, zero, W(300), W(0, 15), W(400)},
+ },
+ { // secondary ignorables
+ opt: opts{alt: altBlanked, top: 250},
+ in: ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+ out: ColElems{zero, zero, W(300), W(0, 0, 15), W(400)},
+ },
+ { // tertiary ignorables, no change
+ opt: opts{alt: altBlanked, top: 250},
+ in: ColElems{W(200), zero, W(300), zero, W(400)},
+ out: ColElems{zero, zero, W(300), zero, W(400)},
+ },
+
+ // Non-ignorable: input is always equal to output.
+ { // all but first are variable
+ opt: opts{alt: altNonIgnorable, top: 999},
+ in: ColElems{W(1000), W(200), W(300), W(400)},
+ out: ColElems{W(1000), W(200), W(300), W(400)},
+ },
+ { // primary ignorables
+ opt: opts{alt: altNonIgnorable, top: 250},
+ in: ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+ out: ColElems{W(200), W(0, 10), W(300), W(0, 15), W(400)},
+ },
+ { // secondary ignorables
+ opt: opts{alt: altNonIgnorable, top: 250},
+ in: ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+ out: ColElems{W(200), W(0, 0, 10), W(300), W(0, 0, 15), W(400)},
+ },
+ { // tertiary ignorables, no change
+ opt: opts{alt: altNonIgnorable, top: 250},
+ in: ColElems{W(200), zero, W(300), zero, W(400)},
+ out: ColElems{W(200), zero, W(300), zero, W(400)},
+ },
+}
+
+func TestProcessWeights(t *testing.T) {
+ for i, tt := range processTests {
+ in := convertFromWeights(tt.in)
+ out := convertFromWeights(tt.out)
+ processWeights(tt.opt.alt, uint32(tt.opt.top), in)
+ for j, w := range in {
+ if w != out[j] {
+ t.Errorf("%d: Weights %d was %v; want %v", i, j, w, out[j])
+ }
+ }
+ }
+}
+
+type keyFromElemTest struct {
+ opt opts
+ in ColElems
+ out []byte
+}
+
+var defS = byte(defaults.Secondary)
+var defT = byte(defaults.Tertiary)
+
+const sep = 0 // separator byte
+
+var keyFromElemTests = []keyFromElemTest{
+ { // simple primary and secondary weights.
+ opts{alt: altShifted},
+ ColElems{W(0x200), W(0x7FFF), W(0, 0x30), W(0x100)},
+ []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+ sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+ sep, sep, defT, defT, defT, defT, // tertiary
+ sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+ },
+ },
+ { // same as first, but with zero element that need to be removed
+ opts{alt: altShifted},
+ ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+ []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+ sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+ sep, sep, defT, defT, defT, defT, // tertiary
+ sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+ },
+ },
+ { // same as first, with large primary values
+ opts{alt: altShifted},
+ ColElems{W(0x200), W(0x8000), W(0, 0x30), W(0x12345)},
+ []byte{0x2, 0, 0x80, 0x80, 0x00, 0x81, 0x23, 0x45, // primary
+ sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+ sep, sep, defT, defT, defT, defT, // tertiary
+ sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+ },
+ },
+ { // same as first, but with the secondary level backwards
+ opts{alt: altShifted, backwards: true},
+ ColElems{W(0x200), W(0x7FFF), W(0, 0x30), W(0x100)},
+ []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+ sep, sep, 0, defS, 0, 0x30, 0, defS, 0, defS, // secondary
+ sep, sep, defT, defT, defT, defT, // tertiary
+ sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary
+ },
+ },
+ { // same as first, ignoring quaternary level
+ opts{alt: altShifted, lev: 3},
+ ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+ []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+ sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+ sep, sep, defT, defT, defT, defT, // tertiary
+ },
+ },
+ { // same as first, ignoring tertiary level
+ opts{alt: altShifted, lev: 2},
+ ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+ []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+ sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+ },
+ },
+ { // same as first, ignoring secondary level
+ opts{alt: altShifted, lev: 1},
+ ColElems{W(0x200), zero, W(0x7FFF), W(0, 0x30), zero, W(0x100)},
+ []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00},
+ },
+ { // simple primary and secondary weights.
+ opts{alt: altShiftTrimmed, top: 0x250},
+ ColElems{W(0x300), W(0x200), W(0x7FFF), W(0, 0x30), W(0x800)},
+ []byte{0x3, 0, 0x7F, 0xFF, 0x8, 0x00, // primary
+ sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary
+ sep, sep, defT, defT, defT, defT, // tertiary
+ sep, 0xFF, 0x2, 0, // quaternary
+ },
+ },
+ { // as first, primary with case level enabled
+ opts{alt: altShifted, lev: 1, caseLevel: true},
+ ColElems{W(0x200), W(0x7FFF), W(0, 0x30), W(0x100)},
+ []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary
+ sep, sep, // secondary
+ sep, sep, defT, defT, defT, defT, // tertiary
+ },
+ },
+}
+
+func TestKeyFromElems(t *testing.T) {
+ buf := Buffer{}
+ for i, tt := range keyFromElemTests {
+ buf.Reset()
+ in := convertFromWeights(tt.in)
+ processWeights(tt.opt.alt, uint32(tt.opt.top), in)
+ tt.opt.collator().keyFromElems(&buf, in)
+ res := buf.key
+ if len(res) != len(tt.out) {
+ t.Errorf("%d: len(ws) was %d; want %d (%X should be %X)", i, len(res), len(tt.out), res, tt.out)
+ }
+ n := len(res)
+ if len(tt.out) < n {
+ n = len(tt.out)
+ }
+ for j, c := range res[:n] {
+ if c != tt.out[j] {
+ t.Errorf("%d: byte %d was %X; want %X", i, j, c, tt.out[j])
+ }
+ }
+ }
+}
+
+func TestGetColElems(t *testing.T) {
+ for i, tt := range appendNextTests {
+ c, err := makeTable(tt.in)
+ if err != nil {
+ // error is reported in TestAppendNext
+ continue
+ }
+ // Create one large test per table
+ str := make([]byte, 0, 4000)
+ out := ColElems{}
+ for len(str) < 3000 {
+ for _, chk := range tt.chk {
+ str = append(str, chk.in[:chk.n]...)
+ out = append(out, chk.out...)
+ }
+ }
+ for j, chk := range append(tt.chk, check{string(str), len(str), out}) {
+ out := convertFromWeights(chk.out)
+ ce := c.getColElems([]byte(chk.in)[:chk.n])
+ if len(ce) != len(out) {
+ t.Errorf("%d:%d: len(ws) was %d; want %d", i, j, len(ce), len(out))
+ continue
+ }
+ cnt := 0
+ for k, w := range ce {
+ w, _ = colltab.MakeElem(w.Primary(), w.Secondary(), int(w.Tertiary()), 0)
+ if w != out[k] {
+ t.Errorf("%d:%d: Weights %d was %X; want %X", i, j, k, w, out[k])
+ cnt++
+ }
+ if cnt > 10 {
+ break
+ }
+ }
+ }
+ }
+}
+
+type keyTest struct {
+ in string
+ out []byte
+}
+
+var keyTests = []keyTest{
+ {"abc",
+ []byte{0, 100, 0, 200, 1, 44, 0, 0, 0, 32, 0, 32, 0, 32, 0, 0, 2, 2, 2, 0, 255, 255, 255},
+ },
+ {"a\u0301",
+ []byte{0, 102, 0, 0, 0, 32, 0, 0, 2, 0, 255},
+ },
+ {"aaaaa",
+ []byte{0, 100, 0, 100, 0, 100, 0, 100, 0, 100, 0, 0,
+ 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 0,
+ 2, 2, 2, 2, 2, 0,
+ 255, 255, 255, 255, 255,
+ },
+ },
+ // Issue 16391: incomplete rune at end of UTF-8 sequence.
+ {"\xc2", []byte{133, 255, 253, 0, 0, 0, 32, 0, 0, 2, 0, 255}},
+ {"\xc2a", []byte{133, 255, 253, 0, 100, 0, 0, 0, 32, 0, 32, 0, 0, 2, 2, 0, 255, 255}},
+}
+
+func TestKey(t *testing.T) {
+ c, _ := makeTable(appendNextTests[4].in)
+ c.alternate = altShifted
+ c.ignore = ignore(colltab.Quaternary)
+ buf := Buffer{}
+ keys1 := [][]byte{}
+ keys2 := [][]byte{}
+ for _, tt := range keyTests {
+ keys1 = append(keys1, c.Key(&buf, []byte(tt.in)))
+ keys2 = append(keys2, c.KeyFromString(&buf, tt.in))
+ }
+ // Separate generation from testing to ensure buffers are not overwritten.
+ for i, tt := range keyTests {
+ if !bytes.Equal(keys1[i], tt.out) {
+ t.Errorf("%d: Key(%q) = %d; want %d", i, tt.in, keys1[i], tt.out)
+ }
+ if !bytes.Equal(keys2[i], tt.out) {
+ t.Errorf("%d: KeyFromString(%q) = %d; want %d", i, tt.in, keys2[i], tt.out)
+ }
+ }
+}
+
+type compareTest struct {
+ a, b string
+ res int // comparison result
+}
+
+var compareTests = []compareTest{
+ {"a\u0301", "a", 1},
+ {"a\u0301b", "ab", 1},
+ {"a", "a\u0301", -1},
+ {"ab", "a\u0301b", -1},
+ {"bc", "a\u0301c", 1},
+ {"ab", "aB", -1},
+ {"a\u0301", "a\u0301", 0},
+ {"a", "a", 0},
+ // Only clip prefixes of whole runes.
+ {"\u302E", "\u302F", 1},
+ // Don't clip prefixes when last rune of prefix may be part of contraction.
+ {"a\u035E", "a\u0301\u035F", -1},
+ {"a\u0301\u035Fb", "a\u0301\u035F", -1},
+}
+
+func TestCompare(t *testing.T) {
+ c, _ := makeTable(appendNextTests[4].in)
+ for i, tt := range compareTests {
+ if res := c.Compare([]byte(tt.a), []byte(tt.b)); res != tt.res {
+ t.Errorf("%d: Compare(%q, %q) == %d; want %d", i, tt.a, tt.b, res, tt.res)
+ }
+ if res := c.CompareString(tt.a, tt.b); res != tt.res {
+ t.Errorf("%d: CompareString(%q, %q) == %d; want %d", i, tt.a, tt.b, res, tt.res)
+ }
+ }
+}
+
+func TestNumeric(t *testing.T) {
+ c := New(language.English, Loose, Numeric)
+
+ for i, tt := range []struct {
+ a, b string
+ want int
+ }{
+ {"1", "2", -1},
+ {"2", "12", -1},
+ {"２", "１２", -1}, // Fullwidth is sorted as usual.
+ {"₂", "₁₂", 1}, // Subscript is not sorted as numbers.
+ {"②", "①②", 1}, // Circled is not sorted as numbers.
+ { // Imperial Aramaic, is not sorted as number.
+ "\U00010859",
+ "\U00010858\U00010859",
+ 1,
+ },
+ {"12", "2", 1},
+ {"A-1", "A-2", -1},
+ {"A-2", "A-12", -1},
+ {"A-12", "A-2", 1},
+ {"A-0001", "A-1", 0},
+ } {
+ if got := c.CompareString(tt.a, tt.b); got != tt.want {
+ t.Errorf("%d: CompareString(%s, %s) = %d; want %d", i, tt.a, tt.b, got, tt.want)
+ }
+ }
+}
OSDN Git Service
