1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
12 // MaxSegmentSize is the maximum size of a byte buffer needed to consider any
13 // sequence of starter and non-starter runes for the purpose of normalization.
14 const MaxSegmentSize = maxByteBufferSize
16 // An Iter iterates over a string or byte slice, while normalizing it
20 buf [maxByteBufferSize]byte
21 info Properties // first character saved from previous iteration
22 next iterFunc // implementation of next depends on form
25 p int // current position in input source
26 multiSeg []byte // remainder of multi-segment decomposition
29 type iterFunc func(*Iter) []byte
31 // Init initializes i to iterate over src after normalizing it to Form f.
32 func (i *Iter) Init(f Form, src []byte) {
41 i.next = i.rb.f.nextMain
42 i.asciiF = nextASCIIBytes
43 i.info = i.rb.f.info(i.rb.src, i.p)
47 // InitString initializes i to iterate over src after normalizing it to Form f.
48 func (i *Iter) InitString(f Form, src string) {
56 i.rb.initString(f, src)
57 i.next = i.rb.f.nextMain
58 i.asciiF = nextASCIIString
59 i.info = i.rb.f.info(i.rb.src, i.p)
63 // Seek sets the segment to be returned by the next call to Next to start
64 // at position p. It is the responsibility of the caller to set p to the
65 // start of a segment.
66 func (i *Iter) Seek(offset int64, whence int) (int64, error) {
72 abs = int64(i.p) + offset
74 abs = int64(i.rb.nsrc) + offset
76 return 0, fmt.Errorf("norm: invalid whence")
79 return 0, fmt.Errorf("norm: negative position")
81 if int(abs) >= i.rb.nsrc {
83 return int64(i.p), nil
87 i.next = i.rb.f.nextMain
88 i.info = i.rb.f.info(i.rb.src, i.p)
93 // returnSlice returns a slice of the underlying input type as a byte slice.
94 // If the underlying is of type []byte, it will simply return a slice.
95 // If the underlying is of type string, it will copy the slice to the buffer
97 func (i *Iter) returnSlice(a, b int) []byte {
98 if i.rb.src.bytes == nil {
99 return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
101 return i.rb.src.bytes[a:b]
104 // Pos returns the byte position at which the next call to Next will commence processing.
105 func (i *Iter) Pos() int {
109 func (i *Iter) setDone() {
114 // Done returns true if there is no more input to process.
115 func (i *Iter) Done() bool {
116 return i.p >= i.rb.nsrc
119 // Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
120 // For any input a and b for which f(a) == f(b), subsequent calls
121 // to Next will return the same segments.
122 // Modifying runes are grouped together with the preceding starter, if such a starter exists.
123 // Although not guaranteed, n will typically be the smallest possible n.
124 func (i *Iter) Next() []byte {
128 func nextASCIIBytes(i *Iter) []byte {
132 return i.rb.src.bytes[i.p:p]
134 if i.rb.src.bytes[p] < utf8.RuneSelf {
137 return i.rb.src.bytes[p0:p]
139 i.info = i.rb.f.info(i.rb.src, i.p)
140 i.next = i.rb.f.nextMain
144 func nextASCIIString(i *Iter) []byte {
147 i.buf[0] = i.rb.src.str[i.p]
151 if i.rb.src.str[p] < utf8.RuneSelf {
152 i.buf[0] = i.rb.src.str[i.p]
156 i.info = i.rb.f.info(i.rb.src, i.p)
157 i.next = i.rb.f.nextMain
161 func nextHangul(i *Iter) []byte {
163 next := p + hangulUTF8Size
164 if next >= i.rb.nsrc {
166 } else if i.rb.src.hangul(next) == 0 {
168 i.info = i.rb.f.info(i.rb.src, i.p)
169 i.next = i.rb.f.nextMain
173 return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
176 func nextDone(i *Iter) []byte {
180 // nextMulti is used for iterating over multi-segment decompositions
181 // for decomposing normal forms.
182 func nextMulti(i *Iter) []byte {
186 for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
189 info := i.rb.f.info(input{bytes: d}, j)
190 if info.BoundaryBefore() {
196 // treat last segment as normal decomposition
197 i.next = i.rb.f.nextMain
201 // nextMultiNorm is used for iterating over multi-segment decompositions
202 // for composing normal forms.
203 func nextMultiNorm(i *Iter) []byte {
207 info := i.rb.f.info(input{bytes: d}, j)
208 if info.BoundaryBefore() {
210 seg := i.buf[:i.rb.flushCopy(i.buf[:])]
211 i.rb.insertUnsafe(input{bytes: d}, j, info)
212 i.multiSeg = d[j+int(info.size):]
215 i.rb.insertUnsafe(input{bytes: d}, j, info)
219 i.next = nextComposed
220 return doNormComposed(i)
223 // nextDecomposed is the implementation of Next for forms NFD and NFKD.
224 func nextDecomposed(i *Iter) (next []byte) {
226 inCopyStart, outCopyStart := i.p, 0
228 if sz := int(i.info.size); sz <= 1 {
231 i.p++ // ASCII or illegal byte. Either way, advance by 1.
232 if i.p >= i.rb.nsrc {
234 return i.returnSlice(p, i.p)
235 } else if i.rb.src._byte(i.p) < utf8.RuneSelf {
237 return i.returnSlice(p, i.p)
240 } else if d := i.info.Decomposition(); d != nil {
241 // Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
242 // Case 1: there is a leftover to copy. In this case the decomposition
243 // must begin with a modifier and should always be appended.
244 // Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
247 i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
248 // TODO: this condition should not be possible, but we leave it
249 // in for defensive purposes.
253 } else if i.info.multiSegment() {
254 // outp must be 0 as multi-segment decompositions always
255 // start a new segment.
256 if i.multiSeg == nil {
261 // We are in the last segment. Treat as normal decomposition.
266 prevCC := i.info.tccc
267 if i.p += sz; i.p >= i.rb.nsrc {
269 i.info = Properties{} // Force BoundaryBefore to succeed.
271 i.info = i.rb.f.info(i.rb.src, i.p)
273 switch i.rb.ss.next(i.info) {
275 i.next = nextCGJDecompose
279 copy(i.buf[outp:], d)
284 copy(i.buf[outp:], d)
286 inCopyStart, outCopyStart = i.p, outp
287 if i.info.ccc < prevCC {
291 } else if r := i.rb.src.hangul(i.p); r != 0 {
292 outp = decomposeHangul(i.buf[:], r)
293 i.p += hangulUTF8Size
294 inCopyStart, outCopyStart = i.p, outp
295 if i.p >= i.rb.nsrc {
298 } else if i.rb.src.hangul(i.p) != 0 {
310 if i.p >= i.rb.nsrc {
314 prevCC := i.info.tccc
315 i.info = i.rb.f.info(i.rb.src, i.p)
316 if v := i.rb.ss.next(i.info); v == ssStarter {
318 } else if v == ssOverflow {
319 i.next = nextCGJDecompose
322 if i.info.ccc < prevCC {
326 if outCopyStart == 0 {
327 return i.returnSlice(inCopyStart, i.p)
328 } else if inCopyStart < i.p {
329 i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
333 // Insert what we have decomposed so far in the reorderBuffer.
334 // As we will only reorder, there will always be enough room.
335 i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
336 i.rb.insertDecomposed(i.buf[0:outp])
337 return doNormDecomposed(i)
340 func doNormDecomposed(i *Iter) []byte {
342 i.rb.insertUnsafe(i.rb.src, i.p, i.info)
343 if i.p += int(i.info.size); i.p >= i.rb.nsrc {
347 i.info = i.rb.f.info(i.rb.src, i.p)
351 if s := i.rb.ss.next(i.info); s == ssOverflow {
352 i.next = nextCGJDecompose
356 // new segment or too many combining characters: exit normalization
357 return i.buf[:i.rb.flushCopy(i.buf[:])]
360 func nextCGJDecompose(i *Iter) []byte {
363 i.next = nextDecomposed
364 i.rb.ss.first(i.info)
365 buf := doNormDecomposed(i)
369 // nextComposed is the implementation of Next for forms NFC and NFKC.
370 func nextComposed(i *Iter) []byte {
371 outp, startp := 0, i.p
374 if !i.info.isYesC() {
378 sz := int(i.info.size)
380 sz = 1 // illegal rune: copy byte-by-byte
388 if i.p >= i.rb.nsrc {
391 } else if i.rb.src._byte(i.p) < utf8.RuneSelf {
396 i.info = i.rb.f.info(i.rb.src, i.p)
397 if v := i.rb.ss.next(i.info); v == ssStarter {
399 } else if v == ssOverflow {
400 i.next = nextCGJCompose
403 if i.info.ccc < prevCC {
407 return i.returnSlice(startp, i.p)
409 // reset to start position
411 i.info = i.rb.f.info(i.rb.src, i.p)
412 i.rb.ss.first(i.info)
413 if i.info.multiSegment() {
414 d := i.info.Decomposition()
415 info := i.rb.f.info(input{bytes: d}, 0)
416 i.rb.insertUnsafe(input{bytes: d}, 0, info)
417 i.multiSeg = d[int(info.size):]
418 i.next = nextMultiNorm
419 return nextMultiNorm(i)
421 i.rb.ss.first(i.info)
422 i.rb.insertUnsafe(i.rb.src, i.p, i.info)
423 return doNormComposed(i)
426 func doNormComposed(i *Iter) []byte {
427 // First rune should already be inserted.
429 if i.p += int(i.info.size); i.p >= i.rb.nsrc {
433 i.info = i.rb.f.info(i.rb.src, i.p)
434 if s := i.rb.ss.next(i.info); s == ssStarter {
436 } else if s == ssOverflow {
437 i.next = nextCGJCompose
440 i.rb.insertUnsafe(i.rb.src, i.p, i.info)
443 seg := i.buf[:i.rb.flushCopy(i.buf[:])]
447 func nextCGJCompose(i *Iter) []byte {
448 i.rb.ss = 0 // instead of first
450 i.next = nextComposed
451 // Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
452 // even if they are not. This is particularly dubious for U+FF9E and UFF9A.
453 // If we ever change that, insert a check here.
454 i.rb.ss.first(i.info)
455 i.rb.insertUnsafe(i.rb.src, i.p, i.info)
456 return doNormComposed(i)