1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
7 // This file contains Form-specific logic and wrappers for data in tables.go.
9 // Rune info is stored in a separate trie per composing form. A composing form
10 // and its corresponding decomposing form share the same trie. Each trie maps
11 // a rune to a uint16. The values take two forms. For v >= 0x8000:
13 // 15: 1 (inverse of NFD_QC bit of qcInfo)
14 // 13..7: qcInfo (see below). isYesD is always true (no decompostion).
15 // 6..0: ccc (compressed CCC value).
16 // For v < 0x8000, the respective rune has a decomposition and v is an index
17 // into a byte array of UTF-8 decomposition sequences and additional info and
19 // <header> <decomp_byte>* [<tccc> [<lccc>]]
20 // The header contains the number of bytes in the decomposition (excluding this
21 // length byte). The two most significant bits of this length byte correspond
22 // to bit 5 and 4 of qcInfo (see below). The byte sequence itself starts at v+1.
23 // The byte sequence is followed by a trailing and leading CCC if the values
24 // for these are not zero. The value of v determines which ccc are appended
25 // to the sequences. For v < firstCCC, there are none, for v >= firstCCC,
26 // the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
27 // there is an additional leading ccc. The value of tccc itself is the
28 // trailing CCC shifted left 2 bits. The two least-significant bits of tccc
29 // are the number of trailing non-starters.
32 qcInfoMask = 0x3F // to clear all but the relevant bits in a qcInfo
33 headerLenMask = 0x3F // extract the length value from the header byte
34 headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
37 // Properties provides access to normalization properties of a rune.
38 type Properties struct {
39 pos uint8 // start position in reorderBuffer; used in composition.go
40 size uint8 // length of UTF-8 encoding of this rune
41 ccc uint8 // leading canonical combining class (ccc if not decomposition)
42 tccc uint8 // trailing canonical combining class (ccc if not decomposition)
43 nLead uint8 // number of leading non-starters.
44 flags qcInfo // quick check flags
48 // functions dispatchable per form
49 type lookupFunc func(b input, i int) Properties
51 // formInfo holds Form-specific functions and tables.
52 type formInfo struct {
54 composing, compatibility bool // form type
59 var formTable = []*formInfo{{
64 nextMain: nextComposed,
70 nextMain: nextDecomposed,
76 nextMain: nextComposed,
82 nextMain: nextDecomposed,
85 // We do not distinguish between boundaries for NFC, NFD, etc. to avoid
86 // unexpected behavior for the user. For example, in NFD, there is a boundary
87 // after 'a'. However, 'a' might combine with modifiers, so from the application's
88 // perspective it is not a good boundary. We will therefore always use the
89 // boundaries for the combining variants.
91 // BoundaryBefore returns true if this rune starts a new segment and
92 // cannot combine with any rune on the left.
93 func (p Properties) BoundaryBefore() bool {
94 if p.ccc == 0 && !p.combinesBackward() {
97 // We assume that the CCC of the first character in a decomposition
98 // is always non-zero if different from info.ccc and that we can return
99 // false at this point. This is verified by maketables.
103 // BoundaryAfter returns true if runes cannot combine with or otherwise
104 // interact with this or previous runes.
105 func (p Properties) BoundaryAfter() bool {
106 // TODO: loosen these conditions.
110 // We pack quick check data in 4 bits:
111 // 5: Combines forward (0 == false, 1 == true)
112 // 4..3: NFC_QC Yes(00), No (10), or Maybe (11)
113 // 2: NFD_QC Yes (0) or No (1). No also means there is a decomposition.
114 // 1..0: Number of trailing non-starters.
116 // When all 4 bits are zero, the character is inert, meaning it is never
117 // influenced by normalization.
120 func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
121 func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
123 func (p Properties) combinesForward() bool { return p.flags&0x20 != 0 }
124 func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
125 func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
127 func (p Properties) isInert() bool {
128 return p.flags&qcInfoMask == 0 && p.ccc == 0
131 func (p Properties) multiSegment() bool {
132 return p.index >= firstMulti && p.index < endMulti
135 func (p Properties) nLeadingNonStarters() uint8 {
139 func (p Properties) nTrailingNonStarters() uint8 {
140 return uint8(p.flags & 0x03)
143 // Decomposition returns the decomposition for the underlying rune
144 // or nil if there is none.
145 func (p Properties) Decomposition() []byte {
146 // TODO: create the decomposition for Hangul?
151 n := decomps[i] & headerLenMask
153 return decomps[i : i+uint16(n)]
156 // Size returns the length of UTF-8 encoding of the rune.
157 func (p Properties) Size() int {
161 // CCC returns the canonical combining class of the underlying rune.
162 func (p Properties) CCC() uint8 {
163 if p.index >= firstCCCZeroExcept {
169 // LeadCCC returns the CCC of the first rune in the decomposition.
170 // If there is no decomposition, LeadCCC equals CCC.
171 func (p Properties) LeadCCC() uint8 {
175 // TrailCCC returns the CCC of the last rune in the decomposition.
176 // If there is no decomposition, TrailCCC equals CCC.
177 func (p Properties) TrailCCC() uint8 {
182 // We use 32-bit keys instead of 64-bit for the two codepoint keys.
183 // This clips off the bits of three entries, but we know this will not
184 // result in a collision. In the unlikely event that changes to
185 // UnicodeData.txt introduce collisions, the compiler will catch it.
186 // Note that the recomposition map for NFC and NFKC are identical.
188 // combine returns the combined rune or 0 if it doesn't exist.
189 func combine(a, b rune) rune {
190 key := uint32(uint16(a))<<16 + uint32(uint16(b))
191 return recompMap[key]
194 func lookupInfoNFC(b input, i int) Properties {
195 v, sz := b.charinfoNFC(i)
196 return compInfo(v, sz)
199 func lookupInfoNFKC(b input, i int) Properties {
200 v, sz := b.charinfoNFKC(i)
201 return compInfo(v, sz)
204 // Properties returns properties for the first rune in s.
205 func (f Form) Properties(s []byte) Properties {
206 if f == NFC || f == NFD {
207 return compInfo(nfcData.lookup(s))
209 return compInfo(nfkcData.lookup(s))
212 // PropertiesString returns properties for the first rune in s.
213 func (f Form) PropertiesString(s string) Properties {
214 if f == NFC || f == NFD {
215 return compInfo(nfcData.lookupString(s))
217 return compInfo(nfkcData.lookupString(s))
220 // compInfo converts the information contained in v and sz
221 // to a Properties. See the comment at the top of the file
222 // for more information on the format.
223 func compInfo(v uint16, sz int) Properties {
225 return Properties{size: uint8(sz)}
226 } else if v >= 0x8000 {
231 flags: qcInfo(v >> 8),
233 if p.ccc > 0 || p.combinesBackward() {
234 p.nLead = uint8(p.flags & 0x3)
240 f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
241 p := Properties{size: uint8(sz), flags: f, index: v}
243 v += uint16(h&headerLenMask) + 1
246 p.flags |= qcInfo(c & 0x3)
247 if v >= firstLeadingCCC {
249 if v >= firstStarterWithNLead {
250 // We were tricked. Remove the decomposition.