1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 //go:generate go run gen.go gen_common.go -output tables.go
6 //go:generate go run gen_index.go
10 // TODO: Remove above NOTE after:
11 // - verifying that tables are dropped correctly (most notably matcher tables).
20 // maxCoreSize is the maximum size of a BCP 47 tag without variants and
21 // extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes.
24 // max99thPercentileSize is a somewhat arbitrary buffer size that presumably
25 // is large enough to hold at least 99% of the BCP 47 tags.
26 max99thPercentileSize = 32
28 // maxSimpleUExtensionSize is the maximum size of a -u extension with one
29 // key-type pair. Equals len("-u-") + key (2) + dash + max value (8).
30 maxSimpleUExtensionSize = 14
33 // Tag represents a BCP 47 language tag. It is used to specify an instance of a
34 // specific language or locale. All language tag values are guaranteed to be
39 // TODO: we will soon run out of positions for script. Idea: instead of
40 // storing lang, region, and script codes, store only the compact index and
41 // have a lookup table from this code to its expansion. This greatly speeds
42 // up table lookup, speed up common variant cases.
43 // This will also immediately free up 3 extra bytes. Also, the pVariant
44 // field can now be moved to the lookup table, as the compact index uniquely
45 // determines the offset of a possible variant.
47 pVariant byte // offset in str, includes preceding '-'
48 pExt uint16 // offset of first extension, includes preceding '-'
50 // str is the string representation of the Tag. It will only be used if the
51 // tag has variants or extensions.
55 // Make is a convenience wrapper for Parse that omits the error.
56 // In case of an error, a sensible default is returned.
57 func Make(s string) Tag {
58 return Default.Make(s)
61 // Make is a convenience wrapper for c.Parse that omits the error.
62 // In case of an error, a sensible default is returned.
63 func (c CanonType) Make(s string) Tag {
68 // Raw returns the raw base language, script and region, without making an
69 // attempt to infer their values.
70 func (t Tag) Raw() (b Base, s Script, r Region) {
71 return Base{t.lang}, Script{t.script}, Region{t.region}
74 // equalTags compares language, script and region subtags only.
75 func (t Tag) equalTags(a Tag) bool {
76 return t.lang == a.lang && t.script == a.script && t.region == a.region
79 // IsRoot returns true if t is equal to language "und".
80 func (t Tag) IsRoot() bool {
81 if int(t.pVariant) < len(t.str) {
84 return t.equalTags(und)
87 // private reports whether the Tag consists solely of a private use tag.
88 func (t Tag) private() bool {
89 return t.str != "" && t.pVariant == 0
92 // CanonType can be used to enable or disable various types of canonicalization.
96 // Replace deprecated base languages with their preferred replacements.
97 DeprecatedBase CanonType = 1 << iota
98 // Replace deprecated scripts with their preferred replacements.
100 // Replace deprecated regions with their preferred replacements.
102 // Remove redundant scripts.
104 // Normalize legacy encodings. This includes legacy languages defined in
105 // CLDR as well as bibliographic codes defined in ISO-639.
107 // Map the dominant language of a macro language group to the macro language
108 // subtag. For example cmn -> zh.
110 // The CLDR flag should be used if full compatibility with CLDR is required.
111 // There are a few cases where language.Tag may differ from CLDR. To follow all
112 // of CLDR's suggestions, use All|CLDR.
115 // Raw can be used to Compose or Parse without Canonicalization.
118 // Replace all deprecated tags with their preferred replacements.
119 Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion
121 // All canonicalizations recommended by BCP 47.
122 BCP47 = Deprecated | SuppressScript
124 // All canonicalizations.
125 All = BCP47 | Legacy | Macro
127 // Default is the canonicalization used by Parse, Make and Compose. To
128 // preserve as much information as possible, canonicalizations that remove
129 // potentially valuable information are not included. The Matcher is
130 // designed to recognize similar tags that would be the same if
131 // they were canonicalized using All.
132 Default = Deprecated | Legacy
134 canonLang = DeprecatedBase | Legacy | Macro
136 // TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
139 // canonicalize returns the canonicalized equivalent of the tag and
140 // whether there was any change.
141 func (t Tag) canonicalize(c CanonType) (Tag, bool) {
146 if c&SuppressScript != 0 {
147 if t.lang < langNoIndexOffset && uint8(t.script) == suppressScript[t.lang] {
152 if c&canonLang != 0 {
154 if l, aliasType := normLang(t.lang); l != t.lang {
158 if t.lang == _sh && t.script == 0 {
166 // We deviate here from CLDR. The mapping "nb" -> "no"
167 // qualifies as a typical Macro language mapping. However,
168 // for legacy reasons, CLDR maps "no", the macro language
169 // code for Norwegian, to the dominant variant "nb". This
170 // change is currently under consideration for CLDR as well.
171 // See http://unicode.org/cldr/trac/ticket/2698 and also
172 // http://unicode.org/cldr/trac/ticket/1790 for some of the
173 // practical implications. TODO: this check could be removed
174 // if CLDR adopts this change.
175 if c&CLDR == 0 || t.lang != _nb {
181 if c&DeprecatedBase != 0 {
182 if t.lang == _mo && t.region == 0 {
187 // Other canonicalization types may still apply.
191 } else if c&Legacy != 0 && t.lang == _no && c&CLDR != 0 {
198 if c&DeprecatedScript != 0 {
199 if t.script == _Qaai {
204 if c&DeprecatedRegion != 0 {
205 if r := normRegion(t.region); r != 0 {
213 // Canonicalize returns the canonicalized equivalent of the tag.
214 func (c CanonType) Canonicalize(t Tag) (Tag, error) {
215 t, changed := t.canonicalize(c)
222 // Confidence indicates the level of certainty for a given return value.
223 // For example, Serbian may be written in Cyrillic or Latin script.
224 // The confidence level indicates whether a value was explicitly specified,
225 // whether it is typically the only possible value, or whether there is
230 No Confidence = iota // full confidence that there was no match
231 Low // most likely value picked out of a set of alternatives
232 High // value is generally assumed to be the correct match
233 Exact // exact match or explicitly specified value
236 var confName = []string{"No", "Low", "High", "Exact"}
238 func (c Confidence) String() string {
242 // remakeString is used to update t.str in case lang, script or region changed.
243 // It is assumed that pExt and pVariant still point to the start of the
245 func (t *Tag) remakeString() {
249 extra := t.str[t.pVariant:]
253 if t.equalTags(und) && strings.HasPrefix(extra, "x-") {
259 var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases.
260 b := buf[:t.genCoreBytes(buf[:])]
262 diff := len(b) - int(t.pVariant)
264 b = append(b, extra...)
265 t.pVariant = uint8(int(t.pVariant) + diff)
266 t.pExt = uint16(int(t.pExt) + diff)
268 t.pVariant = uint8(len(b))
269 t.pExt = uint16(len(b))
274 // genCoreBytes writes a string for the base languages, script and region tags
275 // to the given buffer and returns the number of bytes written. It will never
276 // write more than maxCoreSize bytes.
277 func (t *Tag) genCoreBytes(buf []byte) int {
278 n := t.lang.stringToBuf(buf[:])
280 n += copy(buf[n:], "-")
281 n += copy(buf[n:], t.script.String())
284 n += copy(buf[n:], "-")
285 n += copy(buf[n:], t.region.String())
290 // String returns the canonical string representation of the language tag.
291 func (t Tag) String() string {
295 if t.script == 0 && t.region == 0 {
296 return t.lang.String()
298 buf := [maxCoreSize]byte{}
299 return string(buf[:t.genCoreBytes(buf[:])])
302 // Base returns the base language of the language tag. If the base language is
303 // unspecified, an attempt will be made to infer it from the context.
304 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
305 func (t Tag) Base() (Base, Confidence) {
307 return Base{t.lang}, Exact
310 if t.script == 0 && !(Region{t.region}).IsCountry() {
313 if tag, err := addTags(t); err == nil && tag.lang != 0 {
314 return Base{tag.lang}, c
319 // Script infers the script for the language tag. If it was not explicitly given, it will infer
320 // a most likely candidate.
321 // If more than one script is commonly used for a language, the most likely one
322 // is returned with a low confidence indication. For example, it returns (Cyrl, Low)
324 // If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
325 // as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
326 // common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
327 // See http://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
328 // unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified.
329 // Note that an inferred script is never guaranteed to be the correct one. Latin is
330 // almost exclusively used for Afrikaans, but Arabic has been used for some texts
331 // in the past. Also, the script that is commonly used may change over time.
332 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
333 func (t Tag) Script() (Script, Confidence) {
335 return Script{t.script}, Exact
337 sc, c := scriptID(_Zzzz), No
338 if t.lang < langNoIndexOffset {
339 if scr := scriptID(suppressScript[t.lang]); scr != 0 {
340 // Note: it is not always the case that a language with a suppress
341 // script value is only written in one script (e.g. kk, ms, pa).
343 return Script{scriptID(scr)}, High
348 if tag, err := addTags(t); err == nil {
349 if tag.script != sc {
350 sc, c = tag.script, Low
353 t, _ = (Deprecated | Macro).Canonicalize(t)
354 if tag, err := addTags(t); err == nil && tag.script != sc {
355 sc, c = tag.script, Low
361 // Region returns the region for the language tag. If it was not explicitly given, it will
362 // infer a most likely candidate from the context.
363 // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
364 func (t Tag) Region() (Region, Confidence) {
366 return Region{t.region}, Exact
368 if t, err := addTags(t); err == nil {
369 return Region{t.region}, Low // TODO: differentiate between high and low.
371 t, _ = (Deprecated | Macro).Canonicalize(t)
372 if tag, err := addTags(t); err == nil {
373 return Region{tag.region}, Low
375 return Region{_ZZ}, No // TODO: return world instead of undetermined?
378 // Variant returns the variants specified explicitly for this language tag.
379 // or nil if no variant was specified.
380 func (t Tag) Variants() []Variant {
382 if int(t.pVariant) < int(t.pExt) {
383 for x, str := "", t.str[t.pVariant:t.pExt]; str != ""; {
384 x, str = nextToken(str)
385 v = append(v, Variant{x})
391 // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
392 // specific language are substituted with fields from the parent language.
393 // The parent for a language may change for newer versions of CLDR.
394 func (t Tag) Parent() Tag {
396 // Strip the variants and extensions.
397 t, _ = Raw.Compose(t.Raw())
398 if t.region == 0 && t.script != 0 && t.lang != 0 {
399 base, _ := addTags(Tag{lang: t.lang})
400 if base.script == t.script {
401 return Tag{lang: t.lang}
408 maxScript := t.script
411 maxScript = max.script
414 for i := range parents {
415 if langID(parents[i].lang) == t.lang && scriptID(parents[i].maxScript) == maxScript {
416 for _, r := range parents[i].fromRegion {
417 if regionID(r) == t.region {
420 script: scriptID(parents[i].script),
421 region: regionID(parents[i].toRegion),
428 // Strip the script if it is the default one.
429 base, _ := addTags(Tag{lang: t.lang})
430 if base.script != maxScript {
431 return Tag{lang: t.lang, script: maxScript}
433 return Tag{lang: t.lang}
434 } else if t.script != 0 {
435 // The parent for an base-script pair with a non-default script is
436 // "und" instead of the base language.
437 base, _ := addTags(Tag{lang: t.lang})
438 if base.script != t.script {
441 return Tag{lang: t.lang}
447 // returns token t and the rest of the string.
448 func nextToken(s string) (t, tail string) {
449 p := strings.Index(s[1:], "-")
457 // Extension is a single BCP 47 extension.
458 type Extension struct {
462 // String returns the string representation of the extension, including the
464 func (e Extension) String() string {
468 // ParseExtension parses s as an extension and returns it on success.
469 func ParseExtension(s string) (e Extension, err error) {
470 scan := makeScannerString(s)
472 if n := len(scan.token); n != 1 {
473 return Extension{}, errSyntax
475 scan.toLower(0, len(scan.b))
476 end = parseExtension(&scan)
478 return Extension{}, errSyntax
480 return Extension{string(scan.b)}, nil
483 // Type returns the one-byte extension type of e. It returns 0 for the zero
485 func (e Extension) Type() byte {
492 // Tokens returns the list of tokens of e.
493 func (e Extension) Tokens() []string {
494 return strings.Split(e.s, "-")
497 // Extension returns the extension of type x for tag t. It will return
498 // false for ok if t does not have the requested extension. The returned
499 // extension will be invalid in this case.
500 func (t Tag) Extension(x byte) (ext Extension, ok bool) {
501 for i := int(t.pExt); i < len(t.str)-1; {
503 i, ext = getExtension(t.str, i)
505 return Extension{ext}, true
508 return Extension{}, false
511 // Extensions returns all extensions of t.
512 func (t Tag) Extensions() []Extension {
514 for i := int(t.pExt); i < len(t.str)-1; {
516 i, ext = getExtension(t.str, i)
517 e = append(e, Extension{ext})
522 // TypeForKey returns the type associated with the given key, where key and type
523 // are of the allowed values defined for the Unicode locale extension ('u') in
524 // http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
525 // TypeForKey will traverse the inheritance chain to get the correct value.
526 func (t Tag) TypeForKey(key string) string {
527 if start, end, _ := t.findTypeForKey(key); end != start {
528 return t.str[start:end]
534 errPrivateUse = errors.New("cannot set a key on a private use tag")
535 errInvalidArguments = errors.New("invalid key or type")
538 // SetTypeForKey returns a new Tag with the key set to type, where key and type
539 // are of the allowed values defined for the Unicode locale extension ('u') in
540 // http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
541 // An empty value removes an existing pair with the same key.
542 func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
544 return t, errPrivateUse
547 return t, errInvalidArguments
550 // Remove the setting if value is "".
552 start, end, _ := t.findTypeForKey(key)
554 // Remove key tag and leading '-'.
557 // Remove a possible empty extension.
558 if (end == len(t.str) || t.str[end+2] == '-') && t.str[start-2] == '-' {
561 if start == int(t.pVariant) && end == len(t.str) {
563 t.pVariant, t.pExt = 0, 0
565 t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:])
571 if len(value) < 3 || len(value) > 8 {
572 return t, errInvalidArguments
576 buf [maxCoreSize + maxSimpleUExtensionSize]byte
577 uStart int // start of the -u extension.
580 // Generate the tag string if needed.
582 uStart = t.genCoreBytes(buf[:])
587 // Create new key-type pair and parse it to verify.
592 b = b[:5+copy(b[5:], value)]
593 scan := makeScanner(b)
594 if parseExtensions(&scan); scan.err != nil {
598 // Assemble the replacement string.
600 t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1)
601 t.str = string(buf[:uStart+len(b)])
604 start, end, hasExt := t.findTypeForKey(key)
609 t.str = fmt.Sprintf("%s-%s%s", s[:start], b, s[end:])
611 t.str = fmt.Sprintf("%s%s%s", s[:start], value, s[end:])
617 // findKeyAndType returns the start and end position for the type corresponding
618 // to key or the point at which to insert the key-value pair if the type
619 // wasn't found. The hasExt return value reports whether an -u extension was present.
620 // Note: the extensions are typically very small and are likely to contain
621 // only one key-type pair.
622 func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) {
624 if len(key) != 2 || p == len(t.str) || p == 0 {
629 // Find the correct extension.
630 for p++; s[p] != 'u'; p++ {
635 if p = nextExtension(s, p); p == len(s) {
636 return len(s), len(s), false
639 // Proceed to the hyphen following the extension name.
642 // curKey is the key currently being processed.
645 // Iterate over keys until we get the end of a section.
647 // p points to the hyphen preceding the current token.
648 if p3 := p + 3; s[p3] == '-' {
650 // Check whether we just processed the key that was requested.
652 return start, p, true
654 // Set to the next key and continue scanning type tokens.
659 // Start of the type token sequence.
661 // A type is at least 3 characters long.
664 // Attribute or type, which is at least 3 characters long.
667 // p points past the third character of a type or attribute.
668 max := p + 5 // maximum length of token plus hyphen.
672 for ; p < max && s[p] != '-'; p++ {
674 // Bail if we have exhausted all tokens or if the next token starts
676 if p == len(s) || s[p+2] == '-' {
678 return start, p, true
685 // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
686 // for which data exists in the text repository. The index will change over time
687 // and should not be stored in persistent storage. Extensions, except for the
688 // 'va' type of the 'u' extension, are ignored. It will return 0, false if no
689 // compact tag exists, where 0 is the index for the root language (Und).
690 func CompactIndex(t Tag) (index int, ok bool) {
691 // TODO: perhaps give more frequent tags a lower index.
692 // TODO: we could make the indexes stable. This will excluded some
693 // possibilities for optimization, so don't do this quite yet.
696 if strings.HasPrefix(t.str, "x-") {
697 // We have no entries for user-defined tags.
700 if uint16(t.pVariant) != t.pExt {
701 // There are no tags with variants and an u-va type.
702 if t.TypeForKey("va") != "" {
705 t, _ = Raw.Compose(b, s, r, t.Variants())
706 } else if _, ok := t.Extension('u'); ok {
707 // Strip all but the 'va' entry.
708 variant := t.TypeForKey("va")
709 t, _ = Raw.Compose(b, s, r)
710 t, _ = t.SetTypeForKey("va", variant)
713 // We have some variants.
714 for i, s := range specialTags {
722 // No variants specified: just compare core components.
723 // The key has the form lllssrrr, where l, s, and r are nibbles for
724 // respectively the langID, scriptID, and regionID.
725 key := uint32(b.langID) << (8 + 12)
726 key |= uint32(s.scriptID) << 12
727 key |= uint32(r.regionID)
728 x, ok := coreTags[key]
732 // Base is an ISO 639 language code, used for encoding the base language
733 // of a language tag.
738 // ParseBase parses a 2- or 3-letter ISO 639 code.
739 // It returns a ValueError if s is a well-formed but unknown language identifier
740 // or another error if another error occurred.
741 func ParseBase(s string) (Base, error) {
742 if n := len(s); n < 2 || 3 < n {
743 return Base{}, errSyntax
746 l, err := getLangID(buf[:copy(buf[:], s)])
750 // Script is a 4-letter ISO 15924 code for representing scripts.
751 // It is idiomatically represented in title case.
756 // ParseScript parses a 4-letter ISO 15924 code.
757 // It returns a ValueError if s is a well-formed but unknown script identifier
758 // or another error if another error occurred.
759 func ParseScript(s string) (Script, error) {
761 return Script{}, errSyntax
764 sc, err := getScriptID(script, buf[:copy(buf[:], s)])
765 return Script{sc}, err
768 // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
773 // EncodeM49 returns the Region for the given UN M.49 code.
774 // It returns an error if r is not a valid code.
775 func EncodeM49(r int) (Region, error) {
776 rid, err := getRegionM49(r)
777 return Region{rid}, err
780 // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
781 // It returns a ValueError if s is a well-formed but unknown region identifier
782 // or another error if another error occurred.
783 func ParseRegion(s string) (Region, error) {
784 if n := len(s); n < 2 || 3 < n {
785 return Region{}, errSyntax
788 r, err := getRegionID(buf[:copy(buf[:], s)])
789 return Region{r}, err
792 // IsCountry returns whether this region is a country or autonomous area. This
793 // includes non-standard definitions from CLDR.
794 func (r Region) IsCountry() bool {
795 if r.regionID == 0 || r.IsGroup() || r.IsPrivateUse() && r.regionID != _XK {
801 // IsGroup returns whether this region defines a collection of regions. This
802 // includes non-standard definitions from CLDR.
803 func (r Region) IsGroup() bool {
807 return int(regionInclusion[r.regionID]) < len(regionContainment)
810 // Contains returns whether Region c is contained by Region r. It returns true
812 func (r Region) Contains(c Region) bool {
813 return r.regionID.contains(c.regionID)
816 func (r regionID) contains(c regionID) bool {
820 g := regionInclusion[r]
821 if g >= nRegionGroups {
824 m := regionContainment[g]
826 d := regionInclusion[c]
827 b := regionInclusionBits[d]
829 // A contained country may belong to multiple disjoint groups. Matching any
830 // of these indicates containment. If the contained region is a group, it
831 // must strictly be a subset.
832 if d >= nRegionGroups {
838 var errNoTLD = errors.New("language: region is not a valid ccTLD")
840 // TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
841 // In all other cases it returns either the region itself or an error.
843 // This method may return an error for a region for which there exists a
844 // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
845 // region will already be canonicalized it was obtained from a Tag that was
846 // obtained using any of the default methods.
847 func (r Region) TLD() (Region, error) {
848 // See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the
849 // difference between ISO 3166-1 and IANA ccTLD.
850 if r.regionID == _GB {
853 if (r.typ() & ccTLD) == 0 {
854 return Region{}, errNoTLD
859 // Canonicalize returns the region or a possible replacement if the region is
860 // deprecated. It will not return a replacement for deprecated regions that
861 // are split into multiple regions.
862 func (r Region) Canonicalize() Region {
863 if cr := normRegion(r.regionID); cr != 0 {
869 // Variant represents a registered variant of a language as defined by BCP 47.
870 type Variant struct {
874 // ParseVariant parses and returns a Variant. An error is returned if s is not
876 func ParseVariant(s string) (Variant, error) {
877 s = strings.ToLower(s)
878 if _, ok := variantIndex[s]; ok {
879 return Variant{s}, nil
881 return Variant{}, mkErrInvalid([]byte(s))
884 // String returns the string representation of the variant.
885 func (v Variant) String() string {