// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build icu package main /* #cgo LDFLAGS: -licui18n -licuuc #include #include #include #include */ import "C" import ( "fmt" "log" "unicode/utf16" "unicode/utf8" "unsafe" ) func init() { AddFactory(CollatorFactory{"icu", newUTF16, "Main ICU collator, using native strings."}) AddFactory(CollatorFactory{"icu8", newUTF8iter, "ICU collator using ICU iterators to process UTF8."}) AddFactory(CollatorFactory{"icu16", newUTF8conv, "ICU collation by first converting UTF8 to UTF16."}) } func icuCharP(s []byte) *C.char { return (*C.char)(unsafe.Pointer(&s[0])) } func icuUInt8P(s []byte) *C.uint8_t { return (*C.uint8_t)(unsafe.Pointer(&s[0])) } func icuUCharP(s []uint16) *C.UChar { return (*C.UChar)(unsafe.Pointer(&s[0])) } func icuULen(s []uint16) C.int32_t { return C.int32_t(len(s)) } func icuSLen(s []byte) C.int32_t { return C.int32_t(len(s)) } // icuCollator implements a Collator based on ICU. type icuCollator struct { loc *C.char col *C.UCollator keyBuf []byte } const growBufSize = 10 * 1024 * 1024 func (c *icuCollator) init(locale string) error { err := C.UErrorCode(0) c.loc = C.CString(locale) c.col = C.ucol_open(c.loc, &err) if err > 0 { return fmt.Errorf("failed opening collator for %q", locale) } else if err < 0 { loc := C.ucol_getLocaleByType(c.col, 0, &err) fmt, ok := map[int]string{ -127: "warning: using default collator: %s", -128: "warning: using fallback collator: %s", }[int(err)] if ok { log.Printf(fmt, C.GoString(loc)) } } c.keyBuf = make([]byte, 0, growBufSize) return nil } func (c *icuCollator) buf() (*C.uint8_t, C.int32_t) { if len(c.keyBuf) == cap(c.keyBuf) { c.keyBuf = make([]byte, 0, growBufSize) } b := c.keyBuf[len(c.keyBuf):cap(c.keyBuf)] return icuUInt8P(b), icuSLen(b) } func (c *icuCollator) extendBuf(n C.int32_t) []byte { end := len(c.keyBuf) + int(n) if end > cap(c.keyBuf) { if len(c.keyBuf) == 0 { log.Fatalf("icuCollator: max string size exceeded: %v > %v", n, growBufSize) } c.keyBuf = make([]byte, 0, growBufSize) return nil } b := c.keyBuf[len(c.keyBuf):end] c.keyBuf = c.keyBuf[:end] return b } func (c *icuCollator) Close() error { C.ucol_close(c.col) C.free(unsafe.Pointer(c.loc)) return nil } // icuUTF16 implements the Collator interface. type icuUTF16 struct { icuCollator } func newUTF16(locale string) (Collator, error) { c := &icuUTF16{} return c, c.init(locale) } func (c *icuUTF16) Compare(a, b Input) int { return int(C.ucol_strcoll(c.col, icuUCharP(a.UTF16), icuULen(a.UTF16), icuUCharP(b.UTF16), icuULen(b.UTF16))) } func (c *icuUTF16) Key(s Input) []byte { bp, bn := c.buf() n := C.ucol_getSortKey(c.col, icuUCharP(s.UTF16), icuULen(s.UTF16), bp, bn) if b := c.extendBuf(n); b != nil { return b } return c.Key(s) } // icuUTF8iter implements the Collator interface // This implementation wraps the UTF8 string in an iterator // which is passed to the collator. type icuUTF8iter struct { icuCollator a, b C.UCharIterator } func newUTF8iter(locale string) (Collator, error) { c := &icuUTF8iter{} return c, c.init(locale) } func (c *icuUTF8iter) Compare(a, b Input) int { err := C.UErrorCode(0) C.uiter_setUTF8(&c.a, icuCharP(a.UTF8), icuSLen(a.UTF8)) C.uiter_setUTF8(&c.b, icuCharP(b.UTF8), icuSLen(b.UTF8)) return int(C.ucol_strcollIter(c.col, &c.a, &c.b, &err)) } func (c *icuUTF8iter) Key(s Input) []byte { err := C.UErrorCode(0) state := [2]C.uint32_t{} C.uiter_setUTF8(&c.a, icuCharP(s.UTF8), icuSLen(s.UTF8)) bp, bn := c.buf() n := C.ucol_nextSortKeyPart(c.col, &c.a, &(state[0]), bp, bn, &err) if n >= bn { // Force failure. if c.extendBuf(n+1) != nil { log.Fatal("expected extension to fail") } return c.Key(s) } return c.extendBuf(n) } // icuUTF8conv implements the Collator interface. // This implementation first converts the give UTF8 string // to UTF16 and then calls the main ICU collation function. type icuUTF8conv struct { icuCollator } func newUTF8conv(locale string) (Collator, error) { c := &icuUTF8conv{} return c, c.init(locale) } func (c *icuUTF8conv) Compare(sa, sb Input) int { a := encodeUTF16(sa.UTF8) b := encodeUTF16(sb.UTF8) return int(C.ucol_strcoll(c.col, icuUCharP(a), icuULen(a), icuUCharP(b), icuULen(b))) } func (c *icuUTF8conv) Key(s Input) []byte { a := encodeUTF16(s.UTF8) bp, bn := c.buf() n := C.ucol_getSortKey(c.col, icuUCharP(a), icuULen(a), bp, bn) if b := c.extendBuf(n); b != nil { return b } return c.Key(s) } func encodeUTF16(b []byte) []uint16 { a := []uint16{} for len(b) > 0 { r, sz := utf8.DecodeRune(b) b = b[sz:] r1, r2 := utf16.EncodeRune(r) if r1 != 0xFFFD { a = append(a, uint16(r1), uint16(r2)) } else { a = append(a, uint16(r)) } } return a }