summaryrefslogtreecommitdiffstats
path: root/vendor/golang.org/x/text/language/match.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/golang.org/x/text/language/match.go')
-rw-r--r--vendor/golang.org/x/text/language/match.go306
1 files changed, 153 insertions, 153 deletions
diff --git a/vendor/golang.org/x/text/language/match.go b/vendor/golang.org/x/text/language/match.go
index 63bc744a3..15b74d125 100644
--- a/vendor/golang.org/x/text/language/match.go
+++ b/vendor/golang.org/x/text/language/match.go
@@ -16,6 +16,29 @@ func PreferSameScript(preferSame bool) MatchOption {
return func(m *matcher) { m.preferSameScript = preferSame }
}
+// TODO(v1.0.0): consider making Matcher a concrete type, instead of interface.
+// There doesn't seem to be too much need for multiple types.
+// Making it a concrete type allows MatchStrings to be a method, which will
+// improve its discoverability.
+
+// MatchStrings parses and matches the given strings until one of them matches
+// the language in the Matcher. A string may be an Accept-Language header as
+// handled by ParseAcceptLanguage. The default language is returned if no
+// other language matched.
+func MatchStrings(m Matcher, lang ...string) (tag Tag, index int) {
+ for _, accept := range lang {
+ desired, _, err := ParseAcceptLanguage(accept)
+ if err != nil {
+ continue
+ }
+ if tag, index, conf := m.Match(desired...); conf != No {
+ return tag, index
+ }
+ }
+ tag, index, _ = m.Match()
+ return
+}
+
// Matcher is the interface that wraps the Match method.
//
// Match returns the best match for any of the given tags, along with
@@ -83,9 +106,8 @@ func (m *matcher) Match(want ...Tag) (t Tag, index int, c Confidence) {
// to do after the fact, so we do it here.
// TODO: add in alternative variants to -u-va-.
// TODO: add preferred region to -u-rg-.
- // TODO: add other extensions. Merge with existing extensions.
- if u, ok := w.Extension('u'); ok {
- t, _ = Raw.Compose(t, u)
+ if e := w.Extensions(); len(e) > 0 {
+ t, _ = Raw.Compose(t, e)
}
return t, index, c
}
@@ -122,7 +144,7 @@ var ErrMissingLikelyTagsData = errors.New("missing likely tags data")
// addLikelySubtags sets subtags to their most likely value, given the locale.
// In most cases this means setting fields for unknown values, but in some
-// cases it may alter a value. It returns a ErrMissingLikelyTagsData error
+// cases it may alter a value. It returns an ErrMissingLikelyTagsData error
// if the given locale cannot be expanded.
func (t Tag) addLikelySubtags() (Tag, error) {
id, err := addTags(t)
@@ -331,8 +353,9 @@ func minimizeTags(t Tag) (Tag, error) {
// 1) compute the match between the two tags.
// 2) if the match is better than the previous best match, replace it
// with the new match. (see next section)
-// b) if the current best match is above a certain threshold, return this
-// match without proceeding to the next tag in "desired". [See Note 1]
+// b) if the current best match is Exact and pin is true the result will be
+// frozen to the language found thusfar, although better matches may
+// still be found for the same language.
// 3) If the best match so far is below a certain threshold, return "default".
//
// Ranking:
@@ -381,9 +404,6 @@ func minimizeTags(t Tag) (Tag, error) {
// found wins.
//
// Notes:
-// [1] Note that even if we may not have a perfect match, if a match is above a
-// certain threshold, it is considered a better match than any other match
-// to a tag later in the list of preferred language tags.
// [2] In practice, as matching of Exact is done in a separate phase from
// matching the other levels, we reuse the Exact level to mean MaxExact in
// the second phase. As a consequence, we only need the levels defined by
@@ -429,8 +449,8 @@ type matcher struct {
// matchHeader has the lists of tags for exact matches and matches based on
// maximized and canonicalized tags for a given language.
type matchHeader struct {
- exact []*haveTag
- max []*haveTag
+ haveTags []*haveTag
+ original bool
}
// haveTag holds a supported Tag and its maximized script and region. The maximized
@@ -460,7 +480,7 @@ type haveTag struct {
func makeHaveTag(tag Tag, index int) (haveTag, langID) {
max := tag
- if tag.lang != 0 {
+ if tag.lang != 0 || tag.region != 0 || tag.script != 0 {
max, _ = max.canonicalize(All)
max, _ = addTags(max)
max.remakeString()
@@ -485,29 +505,27 @@ func altScript(l langID, s scriptID) scriptID {
// addIfNew adds a haveTag to the list of tags only if it is a unique tag.
// Tags that have the same maximized values are linked by index.
func (h *matchHeader) addIfNew(n haveTag, exact bool) {
+ h.original = h.original || exact
// Don't add new exact matches.
- for _, v := range h.exact {
+ for _, v := range h.haveTags {
if v.tag.equalsRest(n.tag) {
return
}
}
- if exact {
- h.exact = append(h.exact, &n)
- }
// Allow duplicate maximized tags, but create a linked list to allow quickly
// comparing the equivalents and bail out.
- for i, v := range h.max {
+ for i, v := range h.haveTags {
if v.maxScript == n.maxScript &&
v.maxRegion == n.maxRegion &&
v.tag.variantOrPrivateTagStr() == n.tag.variantOrPrivateTagStr() {
- for h.max[i].nextMax != 0 {
- i = int(h.max[i].nextMax)
+ for h.haveTags[i].nextMax != 0 {
+ i = int(h.haveTags[i].nextMax)
}
- h.max[i].nextMax = uint16(len(h.max))
+ h.haveTags[i].nextMax = uint16(len(h.haveTags))
break
}
}
- h.max = append(h.max, &n)
+ h.haveTags = append(h.haveTags, &n)
}
// header returns the matchHeader for the given language. It creates one if
@@ -553,27 +571,26 @@ func newMatcher(supported []Tag, options []MatchOption) *matcher {
m.header(tag.lang).addIfNew(pair, true)
m.supported = append(m.supported, &pair)
}
- m.default_ = m.header(supported[0].lang).exact[0]
+ m.default_ = m.header(supported[0].lang).haveTags[0]
+ // Keep these in two different loops to support the case that two equivalent
+ // languages are distinguished, such as iw and he.
for i, tag := range supported {
pair, max := makeHaveTag(tag, i)
if max != tag.lang {
- m.header(max).addIfNew(pair, false)
+ m.header(max).addIfNew(pair, true)
}
}
- // TODO: include alt script.
- // - don't replace regions, but allow regions to be made more specific.
-
// update is used to add indexes in the map for equivalent languages.
- // If force is true, the update will also apply to derived entries. To
- // avoid applying a "transitive closure", use false.
- update := func(want, have uint16, conf Confidence, force bool) {
+ // update will only add entries to original indexes, thus not computing any
+ // transitive relations.
+ update := func(want, have uint16, conf Confidence) {
if hh := m.index[langID(have)]; hh != nil {
- if !force && len(hh.exact) == 0 {
+ if !hh.original {
return
}
hw := m.header(langID(want))
- for _, ht := range hh.max {
+ for _, ht := range hh.haveTags {
v := *ht
if conf < v.conf {
v.conf = conf
@@ -582,7 +599,7 @@ func newMatcher(supported []Tag, options []MatchOption) *matcher {
if v.altScript != 0 {
v.altScript = altScript(langID(want), v.maxScript)
}
- hw.addIfNew(v, conf == Exact && len(hh.exact) > 0)
+ hw.addIfNew(v, conf == Exact && hh.original)
}
}
}
@@ -590,9 +607,9 @@ func newMatcher(supported []Tag, options []MatchOption) *matcher {
// Add entries for languages with mutual intelligibility as defined by CLDR's
// languageMatch data.
for _, ml := range matchLang {
- update(ml.want, ml.have, toConf(ml.distance), false)
+ update(ml.want, ml.have, toConf(ml.distance))
if !ml.oneway {
- update(ml.have, ml.want, toConf(ml.distance), false)
+ update(ml.have, ml.want, toConf(ml.distance))
}
}
@@ -602,10 +619,6 @@ func newMatcher(supported []Tag, options []MatchOption) *matcher {
// (their canonicalization simply substitutes a different language code, but
// nothing else), the match confidence is Exact, otherwise it is High.
for i, lm := range langAliasMap {
- if lm.from == _sh {
- continue
- }
-
// If deprecated codes match and there is no fiddling with the script or
// or region, we consider it an exact match.
conf := Exact
@@ -613,9 +626,9 @@ func newMatcher(supported []Tag, options []MatchOption) *matcher {
if !isExactEquivalent(langID(lm.from)) {
conf = High
}
- update(lm.to, lm.from, conf, true)
+ update(lm.to, lm.from, conf)
}
- update(lm.from, lm.to, conf, true)
+ update(lm.from, lm.to, conf)
}
return m
}
@@ -624,28 +637,29 @@ func newMatcher(supported []Tag, options []MatchOption) *matcher {
// account the order of preference of the given tags.
func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
best := bestMatch{}
- for _, w := range want {
+ for i, w := range want {
var max Tag
// Check for exact match first.
h := m.index[w.lang]
if w.lang != 0 {
- // Base language is defined.
if h == nil {
continue
}
- for i := range h.exact {
- have := h.exact[i]
- if have.tag.equalsRest(w) {
- return have, w, Exact
- }
+ // Base language is defined.
+ max, _ = w.canonicalize(Legacy | Deprecated | Macro)
+ // A region that is added through canonicalization is stronger than
+ // a maximized region: set it in the original (e.g. mo -> ro-MD).
+ if w.region != max.region {
+ w.region = max.region
}
- max, _ = w.canonicalize(Legacy | Deprecated)
+ // TODO: should we do the same for scripts?
+ // See test case: en, sr, nl ; sh ; sr
max, _ = addTags(max)
} else {
// Base language is not defined.
if h != nil {
- for i := range h.exact {
- have := h.exact[i]
+ for i := range h.haveTags {
+ have := h.haveTags[i]
if have.tag.equalsRest(w) {
return have, w, Exact
}
@@ -661,16 +675,23 @@ func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
continue
}
}
+ pin := true
+ for _, t := range want[i+1:] {
+ if w.lang == t.lang {
+ pin = false
+ break
+ }
+ }
// Check for match based on maximized tag.
- for i := range h.max {
- have := h.max[i]
- best.update(have, w, max.script, max.region)
+ for i := range h.haveTags {
+ have := h.haveTags[i]
+ best.update(have, w, max.script, max.region, pin)
if best.conf == Exact {
for have.nextMax != 0 {
- have = h.max[have.nextMax]
- best.update(have, w, max.script, max.region)
+ have = h.haveTags[have.nextMax]
+ best.update(have, w, max.script, max.region, pin)
}
- return best.have, best.want, High
+ return best.have, best.want, best.conf
}
}
}
@@ -685,43 +706,68 @@ func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
// bestMatch accumulates the best match so far.
type bestMatch struct {
- have *haveTag
- want Tag
- conf Confidence
+ have *haveTag
+ want Tag
+ conf Confidence
+ pinnedRegion regionID
+ pinLanguage bool
+ sameRegionGroup bool
// Cached results from applying tie-breaking rules.
origLang bool
origReg bool
+ paradigmReg bool
regGroupDist uint8
- regDist uint8
origScript bool
- parentDist uint8 // 255 if have is not an ancestor of want tag.
}
// update updates the existing best match if the new pair is considered to be a
-// better match.
-// To determine if the given pair is a better match, it first computes the rough
-// confidence level. If this surpasses the current match, it will replace it and
-// update the tie-breaker rule cache. If there is a tie, it proceeds with applying
-// a series of tie-breaker rules. If there is no conclusive winner after applying
-// the tie-breaker rules, it leaves the current match as the preferred match.
-func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion regionID) {
+// better match. To determine if the given pair is a better match, it first
+// computes the rough confidence level. If this surpasses the current match, it
+// will replace it and update the tie-breaker rule cache. If there is a tie, it
+// proceeds with applying a series of tie-breaker rules. If there is no
+// conclusive winner after applying the tie-breaker rules, it leaves the current
+// match as the preferred match.
+//
+// If pin is true and have and tag are a strong match, it will henceforth only
+// consider matches for this language. This corresponds to the nothing that most
+// users have a strong preference for the first defined language. A user can
+// still prefer a second language over a dialect of the preferred language by
+// explicitly specifying dialects, e.g. "en, nl, en-GB". In this case pin should
+// be false.
+func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion regionID, pin bool) {
// Bail if the maximum attainable confidence is below that of the current best match.
c := have.conf
if c < m.conf {
return
}
- if have.maxScript != maxScript {
+ // Don't change the language once we already have found an exact match.
+ if m.pinLanguage && tag.lang != m.want.lang {
+ return
+ }
+ // Pin the region group if we are comparing tags for the same language.
+ if tag.lang == m.want.lang && m.sameRegionGroup {
+ _, sameGroup := regionGroupDist(m.pinnedRegion, have.maxRegion, have.maxScript, m.want.lang)
+ if !sameGroup {
+ return
+ }
+ }
+ if c == Exact && have.maxScript == maxScript {
+ // If there is another language and then another entry of this language,
+ // don't pin anything, otherwise pin the language.
+ m.pinLanguage = pin
+ }
+ if have.tag.equalsRest(tag) {
+ } else if have.maxScript != maxScript {
// There is usually very little comprehension between different scripts.
- // In a few cases there may still be Low comprehension. This possibility is
- // pre-computed and stored in have.altScript.
+ // In a few cases there may still be Low comprehension. This possibility
+ // is pre-computed and stored in have.altScript.
if Low < m.conf || have.altScript != maxScript {
return
}
c = Low
} else if have.maxRegion != maxRegion {
- // There is usually a small difference between languages across regions.
- // We use the region distance (below) to disambiguate between equal matches.
if High < c {
+ // There is usually a small difference between languages across regions.
c = High
}
}
@@ -748,14 +794,6 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
beaten = true
}
- regGroupDist := regionGroupDist(have.maxRegion, maxRegion, maxScript, tag.lang)
- if !beaten && m.regGroupDist != regGroupDist {
- if regGroupDist > m.regGroupDist {
- return
- }
- beaten = true
- }
-
// We prefer if the pre-maximized region was specified and identical.
origReg := have.tag.region == tag.region && tag.region != 0
if !beaten && m.origReg != origReg {
@@ -765,45 +803,26 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
beaten = true
}
- // TODO: remove the region distance rule. Region distance has been replaced
- // by the region grouping rule. For now we leave it as it still seems to
- // have a net positive effect when applied after the grouping rule.
- // Possible solutions:
- // - apply the primary locale rule first to effectively disable region
- // region distance if groups are defined.
- // - express the following errors in terms of grouping (if possible)
- // - find another method of handling the following cases.
- // maximization of legacy: find mo in
- // "sr-Cyrl, sr-Latn, ro, ro-MD": have ro; want ro-MD (High)
- // region distance French: find fr-US in
- // "en, fr, fr-CA, fr-CH": have fr; want fr-CA (High)
-
- // Next we prefer smaller distances between regions, as defined by
- // regionDist.
- regDist := uint8(regionDistance(have.maxRegion, maxRegion))
- if !beaten && m.regDist != regDist {
- if regDist > m.regDist {
+ regGroupDist, sameGroup := regionGroupDist(have.maxRegion, maxRegion, maxScript, tag.lang)
+ if !beaten && m.regGroupDist != regGroupDist {
+ if regGroupDist > m.regGroupDist {
return
}
beaten = true
}
- // Next we prefer if the pre-maximized script was specified and identical.
- origScript := have.tag.script == tag.script && tag.script != 0
- if !beaten && m.origScript != origScript {
- if m.origScript {
+ paradigmReg := isParadigmLocale(tag.lang, have.maxRegion)
+ if !beaten && m.paradigmReg != paradigmReg {
+ if !paradigmReg {
return
}
beaten = true
}
- // Finally we prefer tags which have a closer parent relationship.
- // TODO: the parent relationship no longer seems necessary. It doesn't hurt
- // to leave it in as the final tie-breaker, though, especially until the
- // grouping data has further matured.
- parentDist := parentDistance(have.tag.region, tag)
- if !beaten && m.parentDist != parentDist {
- if parentDist > m.parentDist {
+ // Next we prefer if the pre-maximized script was specified and identical.
+ origScript := have.tag.script == tag.script && tag.script != 0
+ if !beaten && m.origScript != origScript {
+ if m.origScript {
return
}
beaten = true
@@ -814,35 +833,30 @@ func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion
m.have = have
m.want = tag
m.conf = c
+ m.pinnedRegion = maxRegion
+ m.sameRegionGroup = sameGroup
m.origLang = origLang
m.origReg = origReg
+ m.paradigmReg = paradigmReg
m.origScript = origScript
m.regGroupDist = regGroupDist
- m.regDist = regDist
- m.parentDist = parentDist
}
}
-// parentDistance returns the number of times Parent must be called before the
-// regions match. It is assumed that it has already been checked that lang and
-// script are identical. If haveRegion does not occur in the ancestor chain of
-// tag, it returns 255.
-func parentDistance(haveRegion regionID, tag Tag) uint8 {
- p := tag.Parent()
- d := uint8(1)
- for haveRegion != p.region {
- if p.region == 0 {
- return 255
+func isParadigmLocale(lang langID, r regionID) bool {
+ for _, e := range paradigmLocales {
+ if langID(e[0]) == lang && (r == regionID(e[1]) || r == regionID(e[2])) {
+ return true
}
- p = p.Parent()
- d++
}
- return d
+ return false
}
// regionGroupDist computes the distance between two regions based on their
// CLDR grouping.
-func regionGroupDist(a, b regionID, script scriptID, lang langID) uint8 {
+func regionGroupDist(a, b regionID, script scriptID, lang langID) (dist uint8, same bool) {
+ const defaultDistance = 4
+
aGroup := uint(regionToGroups[a]) << 1
bGroup := uint(regionToGroups[b]) << 1
for _, ri := range matchRegion {
@@ -850,40 +864,16 @@ func regionGroupDist(a, b regionID, script scriptID, lang langID) uint8 {
group := uint(1 << (ri.group &^ 0x80))
if 0x80&ri.group == 0 {
if aGroup&bGroup&group != 0 { // Both regions are in the group.
- return ri.distance
+ return ri.distance, ri.distance == defaultDistance
}
} else {
if (aGroup|bGroup)&group == 0 { // Both regions are not in the group.
- return ri.distance
+ return ri.distance, ri.distance == defaultDistance
}
}
}
}
- const defaultDistance = 4
- return defaultDistance
-}
-
-// regionDistance computes the distance between two regions based on the
-// distance in the graph of region containments as defined in CLDR. It iterates
-// over increasingly inclusive sets of groups, represented as bit vectors, until
-// the source bit vector has bits in common with the destination vector.
-func regionDistance(a, b regionID) int {
- if a == b {
- return 0
- }
- p, q := regionInclusion[a], regionInclusion[b]
- if p < nRegionGroups {
- p, q = q, p
- }
- set := regionInclusionBits
- if q < nRegionGroups && set[p]&(1<<q) != 0 {
- return 1
- }
- d := 2
- for goal := set[q]; set[p]&goal == 0; p = regionInclusionNext[p] {
- d++
- }
- return d
+ return defaultDistance, true
}
func (t Tag) variants() string {
@@ -930,4 +920,14 @@ func init() {
notEquivalent = append(notEquivalent, langID(lm.from))
}
}
+ // Maximize undefined regions of paradigm locales.
+ for i, v := range paradigmLocales {
+ max, _ := addTags(Tag{lang: langID(v[0])})
+ if v[1] == 0 {
+ paradigmLocales[i][1] = uint16(max.region)
+ }
+ if v[2] == 0 {
+ paradigmLocales[i][2] = uint16(max.region)
+ }
+ }
}
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-06-02 17:22:21 +0000