From 54d3d47daf9190275bbdaf8703b84969a4593451 Mon Sep 17 00:00:00 2001 From: Corey Hulen Date: Fri, 24 Mar 2017 23:31:34 -0700 Subject: PLT-6076 Adding viper libs for config file changes (#5871) * Adding viper libs for config file changes * Removing the old fsnotify lib * updating some missing libs --- vendor/golang.org/x/text/runes/cond.go | 187 +++++++ vendor/golang.org/x/text/runes/cond_test.go | 282 +++++++++++ vendor/golang.org/x/text/runes/example_test.go | 60 +++ vendor/golang.org/x/text/runes/runes.go | 355 +++++++++++++ vendor/golang.org/x/text/runes/runes_test.go | 664 +++++++++++++++++++++++++ 5 files changed, 1548 insertions(+) create mode 100644 vendor/golang.org/x/text/runes/cond.go create mode 100644 vendor/golang.org/x/text/runes/cond_test.go create mode 100644 vendor/golang.org/x/text/runes/example_test.go create mode 100644 vendor/golang.org/x/text/runes/runes.go create mode 100644 vendor/golang.org/x/text/runes/runes_test.go (limited to 'vendor/golang.org/x/text/runes') diff --git a/vendor/golang.org/x/text/runes/cond.go b/vendor/golang.org/x/text/runes/cond.go new file mode 100644 index 000000000..df7aa02db --- /dev/null +++ b/vendor/golang.org/x/text/runes/cond.go @@ -0,0 +1,187 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runes + +import ( + "unicode/utf8" + + "golang.org/x/text/transform" +) + +// Note: below we pass invalid UTF-8 to the tIn and tNotIn transformers as is. +// This is done for various reasons: +// - To retain the semantics of the Nop transformer: if input is passed to a Nop +// one would expect it to be unchanged. +// - It would be very expensive to pass a converted RuneError to a transformer: +// a transformer might need more source bytes after RuneError, meaning that +// the only way to pass it safely is to create a new buffer and manage the +// intermingling of RuneErrors and normal input. +// - Many transformers leave ill-formed UTF-8 as is, so this is not +// inconsistent. Generally ill-formed UTF-8 is only replaced if it is a +// logical consequence of the operation (as for Map) or if it otherwise would +// pose security concerns (as for Remove). +// - An alternative would be to return an error on ill-formed UTF-8, but this +// would be inconsistent with other operations. + +// If returns a transformer that applies tIn to consecutive runes for which +// s.Contains(r) and tNotIn to consecutive runes for which !s.Contains(r). Reset +// is called on tIn and tNotIn at the start of each run. A Nop transformer will +// substitute a nil value passed to tIn or tNotIn. Invalid UTF-8 is translated +// to RuneError to determine which transformer to apply, but is passed as is to +// the respective transformer. +func If(s Set, tIn, tNotIn transform.Transformer) Transformer { + if tIn == nil && tNotIn == nil { + return Transformer{transform.Nop} + } + if tIn == nil { + tIn = transform.Nop + } + if tNotIn == nil { + tNotIn = transform.Nop + } + sIn, ok := tIn.(transform.SpanningTransformer) + if !ok { + sIn = dummySpan{tIn} + } + sNotIn, ok := tNotIn.(transform.SpanningTransformer) + if !ok { + sNotIn = dummySpan{tNotIn} + } + + a := &cond{ + tIn: sIn, + tNotIn: sNotIn, + f: s.Contains, + } + a.Reset() + return Transformer{a} +} + +type dummySpan struct{ transform.Transformer } + +func (d dummySpan) Span(src []byte, atEOF bool) (n int, err error) { + return 0, transform.ErrEndOfSpan +} + +type cond struct { + tIn, tNotIn transform.SpanningTransformer + f func(rune) bool + check func(rune) bool // current check to perform + t transform.SpanningTransformer // current transformer to use +} + +// Reset implements transform.Transformer. +func (t *cond) Reset() { + t.check = t.is + t.t = t.tIn + t.t.Reset() // notIn will be reset on first usage. +} + +func (t *cond) is(r rune) bool { + if t.f(r) { + return true + } + t.check = t.isNot + t.t = t.tNotIn + t.tNotIn.Reset() + return false +} + +func (t *cond) isNot(r rune) bool { + if !t.f(r) { + return true + } + t.check = t.is + t.t = t.tIn + t.tIn.Reset() + return false +} + +// This implementation of Span doesn't help all too much, but it needs to be +// there to satisfy this package's Transformer interface. +// TODO: there are certainly room for improvements, though. For example, if +// t.t == transform.Nop (which will a common occurrence) it will save a bundle +// to special-case that loop. +func (t *cond) Span(src []byte, atEOF bool) (n int, err error) { + p := 0 + for n < len(src) && err == nil { + // Don't process too much at a time as the Spanner that will be + // called on this block may terminate early. + const maxChunk = 4096 + max := len(src) + if v := n + maxChunk; v < max { + max = v + } + atEnd := false + size := 0 + current := t.t + for ; p < max; p += size { + r := rune(src[p]) + if r < utf8.RuneSelf { + size = 1 + } else if r, size = utf8.DecodeRune(src[p:]); size == 1 { + if !atEOF && !utf8.FullRune(src[p:]) { + err = transform.ErrShortSrc + break + } + } + if !t.check(r) { + // The next rune will be the start of a new run. + atEnd = true + break + } + } + n2, err2 := current.Span(src[n:p], atEnd || (atEOF && p == len(src))) + n += n2 + if err2 != nil { + return n, err2 + } + // At this point either err != nil or t.check will pass for the rune at p. + p = n + size + } + return n, err +} + +func (t *cond) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + p := 0 + for nSrc < len(src) && err == nil { + // Don't process too much at a time, as the work might be wasted if the + // destination buffer isn't large enough to hold the result or a + // transform returns an error early. + const maxChunk = 4096 + max := len(src) + if n := nSrc + maxChunk; n < len(src) { + max = n + } + atEnd := false + size := 0 + current := t.t + for ; p < max; p += size { + r := rune(src[p]) + if r < utf8.RuneSelf { + size = 1 + } else if r, size = utf8.DecodeRune(src[p:]); size == 1 { + if !atEOF && !utf8.FullRune(src[p:]) { + err = transform.ErrShortSrc + break + } + } + if !t.check(r) { + // The next rune will be the start of a new run. + atEnd = true + break + } + } + nDst2, nSrc2, err2 := current.Transform(dst[nDst:], src[nSrc:p], atEnd || (atEOF && p == len(src))) + nDst += nDst2 + nSrc += nSrc2 + if err2 != nil { + return nDst, nSrc, err2 + } + // At this point either err != nil or t.check will pass for the rune at p. + p = nSrc + size + } + return nDst, nSrc, err +} diff --git a/vendor/golang.org/x/text/runes/cond_test.go b/vendor/golang.org/x/text/runes/cond_test.go new file mode 100644 index 000000000..44cd36d0b --- /dev/null +++ b/vendor/golang.org/x/text/runes/cond_test.go @@ -0,0 +1,282 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runes + +import ( + "strings" + "testing" + "unicode" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + "golang.org/x/text/transform" +) + +var ( + toUpper = cases.Upper(language.Und) + toLower = cases.Lower(language.Und) +) + +type spanformer interface { + transform.SpanningTransformer +} + +func TestPredicate(t *testing.T) { + testConditional(t, func(rt *unicode.RangeTable, t, f spanformer) spanformer { + return If(Predicate(func(r rune) bool { + return unicode.Is(rt, r) + }), t, f) + }) +} + +func TestIn(t *testing.T) { + testConditional(t, func(rt *unicode.RangeTable, t, f spanformer) spanformer { + return If(In(rt), t, f) + }) +} + +func TestNotIn(t *testing.T) { + testConditional(t, func(rt *unicode.RangeTable, t, f spanformer) spanformer { + return If(NotIn(rt), f, t) + }) +} + +func testConditional(t *testing.T, f func(rt *unicode.RangeTable, t, f spanformer) spanformer) { + lower := f(unicode.Latin, toLower, toLower) + + for i, tt := range []transformTest{{ + desc: "empty", + szDst: large, + atEOF: true, + in: "", + out: "", + outFull: "", + t: lower, + }, { + desc: "small", + szDst: 1, + atEOF: true, + in: "B", + out: "b", + outFull: "b", + errSpan: transform.ErrEndOfSpan, + t: lower, + }, { + desc: "short dst", + szDst: 2, + atEOF: true, + in: "AAA", + out: "aa", + outFull: "aaa", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: lower, + }, { + desc: "short dst writing error", + szDst: 1, + atEOF: false, + in: "A\x80", + out: "a", + outFull: "a\x80", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: lower, + }, { + desc: "short dst writing incomplete rune", + szDst: 2, + atEOF: true, + in: "Σ\xc2", + out: "Σ", + outFull: "Σ\xc2", + err: transform.ErrShortDst, + t: f(unicode.Latin, toLower, nil), + }, { + desc: "short dst, longer", + szDst: 5, + atEOF: true, + in: "Hellø", + out: "Hell", + outFull: "Hellø", + err: transform.ErrShortDst, + // idem is used to test short buffers by forcing processing of full-rune increments. + t: f(unicode.Latin, Map(idem), nil), + }, { + desc: "short dst, longer, writing error", + szDst: 6, + atEOF: false, + in: "\x80Hello\x80", + out: "\x80Hello", + outFull: "\x80Hello\x80", + err: transform.ErrShortDst, + t: f(unicode.Latin, Map(idem), nil), + }, { + desc: "short src", + szDst: 2, + atEOF: false, + in: "A\xc2", + out: "a", + outFull: "a\xc2", + err: transform.ErrShortSrc, + errSpan: transform.ErrEndOfSpan, + t: lower, + }, { + desc: "short src no change", + szDst: 2, + atEOF: false, + in: "a\xc2", + out: "a", + outFull: "a\xc2", + err: transform.ErrShortSrc, + errSpan: transform.ErrShortSrc, + nSpan: 1, + t: lower, + }, { + desc: "invalid input, atEOF", + szDst: large, + atEOF: true, + in: "\x80", + out: "\x80", + outFull: "\x80", + t: lower, + }, { + desc: "invalid input, !atEOF", + szDst: large, + atEOF: false, + in: "\x80", + out: "\x80", + outFull: "\x80", + t: lower, + }, { + desc: "invalid input, incomplete rune atEOF", + szDst: large, + atEOF: true, + in: "\xc2", + out: "\xc2", + outFull: "\xc2", + t: lower, + }, { + desc: "nop", + szDst: large, + atEOF: true, + in: "Hello World!", + out: "Hello World!", + outFull: "Hello World!", + t: f(unicode.Latin, nil, nil), + }, { + desc: "nop in", + szDst: large, + atEOF: true, + in: "THIS IS α ΤΕΣΤ", + out: "this is α ΤΕΣΤ", + outFull: "this is α ΤΕΣΤ", + errSpan: transform.ErrEndOfSpan, + t: f(unicode.Greek, nil, toLower), + }, { + desc: "nop in latin", + szDst: large, + atEOF: true, + in: "THIS IS α ΤΕΣΤ", + out: "THIS IS α τεστ", + outFull: "THIS IS α τεστ", + errSpan: transform.ErrEndOfSpan, + t: f(unicode.Latin, nil, toLower), + }, { + desc: "nop not in", + szDst: large, + atEOF: true, + in: "THIS IS α ΤΕΣΤ", + out: "this is α ΤΕΣΤ", + outFull: "this is α ΤΕΣΤ", + errSpan: transform.ErrEndOfSpan, + t: f(unicode.Latin, toLower, nil), + }, { + desc: "pass atEOF is true when at end", + szDst: large, + atEOF: true, + in: "hello", + out: "HELLO", + outFull: "HELLO", + errSpan: transform.ErrEndOfSpan, + t: f(unicode.Latin, upperAtEOF{}, nil), + }, { + desc: "pass atEOF is true when at end of segment", + szDst: large, + atEOF: true, + in: "hello ", + out: "HELLO ", + outFull: "HELLO ", + errSpan: transform.ErrEndOfSpan, + t: f(unicode.Latin, upperAtEOF{}, nil), + }, { + desc: "don't pass atEOF is true when atEOF is false", + szDst: large, + atEOF: false, + in: "hello", + out: "", + outFull: "HELLO", + err: transform.ErrShortSrc, + errSpan: transform.ErrShortSrc, + t: f(unicode.Latin, upperAtEOF{}, nil), + }, { + desc: "pass atEOF is true when at end, no change", + szDst: large, + atEOF: true, + in: "HELLO", + out: "HELLO", + outFull: "HELLO", + t: f(unicode.Latin, upperAtEOF{}, nil), + }, { + desc: "pass atEOF is true when at end of segment, no change", + szDst: large, + atEOF: true, + in: "HELLO ", + out: "HELLO ", + outFull: "HELLO ", + t: f(unicode.Latin, upperAtEOF{}, nil), + }, { + desc: "large input ASCII", + szDst: 12000, + atEOF: false, + in: strings.Repeat("HELLO", 2000), + out: strings.Repeat("hello", 2000), + outFull: strings.Repeat("hello", 2000), + errSpan: transform.ErrEndOfSpan, + err: nil, + t: lower, + }, { + desc: "large input non-ASCII", + szDst: 12000, + atEOF: false, + in: strings.Repeat("\u3333", 2000), + out: strings.Repeat("\u3333", 2000), + outFull: strings.Repeat("\u3333", 2000), + err: nil, + t: lower, + }} { + tt.check(t, i) + } +} + +// upperAtEOF is a strange Transformer that converts text to uppercase, but only +// if atEOF is true. +type upperAtEOF struct{ transform.NopResetter } + +func (upperAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + if !atEOF { + return 0, 0, transform.ErrShortSrc + } + return toUpper.Transform(dst, src, atEOF) +} + +func (upperAtEOF) Span(src []byte, atEOF bool) (n int, err error) { + if !atEOF { + return 0, transform.ErrShortSrc + } + return toUpper.Span(src, atEOF) +} + +func BenchmarkConditional(b *testing.B) { + doBench(b, If(In(unicode.Hangul), transform.Nop, transform.Nop)) +} diff --git a/vendor/golang.org/x/text/runes/example_test.go b/vendor/golang.org/x/text/runes/example_test.go new file mode 100644 index 000000000..a60bfd9d2 --- /dev/null +++ b/vendor/golang.org/x/text/runes/example_test.go @@ -0,0 +1,60 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runes_test + +import ( + "fmt" + "unicode" + + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" + "golang.org/x/text/width" +) + +func ExampleRemove() { + t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + s, _, _ := transform.String(t, "résumé") + fmt.Println(s) + + // Output: + // resume +} + +func ExampleMap() { + replaceHyphens := runes.Map(func(r rune) rune { + if unicode.Is(unicode.Hyphen, r) { + return '|' + } + return r + }) + s, _, _ := transform.String(replaceHyphens, "a-b‐c⸗d﹣e") + fmt.Println(s) + + // Output: + // a|b|c|d|e +} + +func ExampleIn() { + // Convert Latin characters to their canonical form, while keeping other + // width distinctions. + t := runes.If(runes.In(unicode.Latin), width.Fold, nil) + s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / アルアノリウ ｔｅｃｈ") + fmt.Println(s) + + // Output: + // ｱﾙｱﾉﾘｳ tech / アルアノリウ tech +} + +func ExampleIf() { + // Widen everything but ASCII. + isASCII := func(r rune) bool { return r <= unicode.MaxASCII } + t := runes.If(runes.Predicate(isASCII), nil, width.Widen) + s, _, _ := transform.String(t, "ｱﾙｱﾉﾘｳ tech / 中國 / 5₩") + fmt.Println(s) + + // Output: + // アルアノリウ tech / 中國 / 5￦ +} diff --git a/vendor/golang.org/x/text/runes/runes.go b/vendor/golang.org/x/text/runes/runes.go new file mode 100644 index 000000000..71933696f --- /dev/null +++ b/vendor/golang.org/x/text/runes/runes.go @@ -0,0 +1,355 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package runes provide transforms for UTF-8 encoded text. +package runes // import "golang.org/x/text/runes" + +import ( + "unicode" + "unicode/utf8" + + "golang.org/x/text/transform" +) + +// A Set is a collection of runes. +type Set interface { + // Contains returns true if r is contained in the set. + Contains(r rune) bool +} + +type setFunc func(rune) bool + +func (s setFunc) Contains(r rune) bool { + return s(r) +} + +// Note: using funcs here instead of wrapping types result in cleaner +// documentation and a smaller API. + +// In creates a Set with a Contains method that returns true for all runes in +// the given RangeTable. +func In(rt *unicode.RangeTable) Set { + return setFunc(func(r rune) bool { return unicode.Is(rt, r) }) +} + +// In creates a Set with a Contains method that returns true for all runes not +// in the given RangeTable. +func NotIn(rt *unicode.RangeTable) Set { + return setFunc(func(r rune) bool { return !unicode.Is(rt, r) }) +} + +// Predicate creates a Set with a Contains method that returns f(r). +func Predicate(f func(rune) bool) Set { + return setFunc(f) +} + +// Transformer implements the transform.Transformer interface. +type Transformer struct { + t transform.SpanningTransformer +} + +func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + return t.t.Transform(dst, src, atEOF) +} + +func (t Transformer) Span(b []byte, atEOF bool) (n int, err error) { + return t.t.Span(b, atEOF) +} + +func (t Transformer) Reset() { t.t.Reset() } + +// Bytes returns a new byte slice with the result of converting b using t. It +// calls Reset on t. It returns nil if any error was found. This can only happen +// if an error-producing Transformer is passed to If. +func (t Transformer) Bytes(b []byte) []byte { + b, _, err := transform.Bytes(t, b) + if err != nil { + return nil + } + return b +} + +// String returns a string with the result of converting s using t. It calls +// Reset on t. It returns the empty string if any error was found. This can only +// happen if an error-producing Transformer is passed to If. +func (t Transformer) String(s string) string { + s, _, err := transform.String(t, s) + if err != nil { + return "" + } + return s +} + +// TODO: +// - Copy: copying strings and bytes in whole-rune units. +// - Validation (maybe) +// - Well-formed-ness (maybe) + +const runeErrorString = string(utf8.RuneError) + +// Remove returns a Transformer that removes runes r for which s.Contains(r). +// Illegal input bytes are replaced by RuneError before being passed to f. +func Remove(s Set) Transformer { + if f, ok := s.(setFunc); ok { + // This little trick cuts the running time of BenchmarkRemove for sets + // created by Predicate roughly in half. + // TODO: special-case RangeTables as well. + return Transformer{remove(f)} + } + return Transformer{remove(s.Contains)} +} + +// TODO: remove transform.RemoveFunc. + +type remove func(r rune) bool + +func (remove) Reset() {} + +// Span implements transform.Spanner. +func (t remove) Span(src []byte, atEOF bool) (n int, err error) { + for r, size := rune(0), 0; n < len(src); { + if r = rune(src[n]); r < utf8.RuneSelf { + size = 1 + } else if r, size = utf8.DecodeRune(src[n:]); size == 1 { + // Invalid rune. + if !atEOF && !utf8.FullRune(src[n:]) { + err = transform.ErrShortSrc + } else { + err = transform.ErrEndOfSpan + } + break + } + if t(r) { + err = transform.ErrEndOfSpan + break + } + n += size + } + return +} + +// Transform implements transform.Transformer. +func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + for r, size := rune(0), 0; nSrc < len(src); { + if r = rune(src[nSrc]); r < utf8.RuneSelf { + size = 1 + } else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 { + // Invalid rune. + if !atEOF && !utf8.FullRune(src[nSrc:]) { + err = transform.ErrShortSrc + break + } + // We replace illegal bytes with RuneError. Not doing so might + // otherwise turn a sequence of invalid UTF-8 into valid UTF-8. + // The resulting byte sequence may subsequently contain runes + // for which t(r) is true that were passed unnoticed. + if !t(utf8.RuneError) { + if nDst+3 > len(dst) { + err = transform.ErrShortDst + break + } + dst[nDst+0] = runeErrorString[0] + dst[nDst+1] = runeErrorString[1] + dst[nDst+2] = runeErrorString[2] + nDst += 3 + } + nSrc++ + continue + } + if t(r) { + nSrc += size + continue + } + if nDst+size > len(dst) { + err = transform.ErrShortDst + break + } + for i := 0; i < size; i++ { + dst[nDst] = src[nSrc] + nDst++ + nSrc++ + } + } + return +} + +// Map returns a Transformer that maps the runes in the input using the given +// mapping. Illegal bytes in the input are converted to utf8.RuneError before +// being passed to the mapping func. +func Map(mapping func(rune) rune) Transformer { + return Transformer{mapper(mapping)} +} + +type mapper func(rune) rune + +func (mapper) Reset() {} + +// Span implements transform.Spanner. +func (t mapper) Span(src []byte, atEOF bool) (n int, err error) { + for r, size := rune(0), 0; n < len(src); n += size { + if r = rune(src[n]); r < utf8.RuneSelf { + size = 1 + } else if r, size = utf8.DecodeRune(src[n:]); size == 1 { + // Invalid rune. + if !atEOF && !utf8.FullRune(src[n:]) { + err = transform.ErrShortSrc + } else { + err = transform.ErrEndOfSpan + } + break + } + if t(r) != r { + err = transform.ErrEndOfSpan + break + } + } + return n, err +} + +// Transform implements transform.Transformer. +func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + var replacement rune + var b [utf8.UTFMax]byte + + for r, size := rune(0), 0; nSrc < len(src); { + if r = rune(src[nSrc]); r < utf8.RuneSelf { + if replacement = t(r); replacement < utf8.RuneSelf { + if nDst == len(dst) { + err = transform.ErrShortDst + break + } + dst[nDst] = byte(replacement) + nDst++ + nSrc++ + continue + } + size = 1 + } else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 { + // Invalid rune. + if !atEOF && !utf8.FullRune(src[nSrc:]) { + err = transform.ErrShortSrc + break + } + + if replacement = t(utf8.RuneError); replacement == utf8.RuneError { + if nDst+3 > len(dst) { + err = transform.ErrShortDst + break + } + dst[nDst+0] = runeErrorString[0] + dst[nDst+1] = runeErrorString[1] + dst[nDst+2] = runeErrorString[2] + nDst += 3 + nSrc++ + continue + } + } else if replacement = t(r); replacement == r { + if nDst+size > len(dst) { + err = transform.ErrShortDst + break + } + for i := 0; i < size; i++ { + dst[nDst] = src[nSrc] + nDst++ + nSrc++ + } + continue + } + + n := utf8.EncodeRune(b[:], replacement) + + if nDst+n > len(dst) { + err = transform.ErrShortDst + break + } + for i := 0; i < n; i++ { + dst[nDst] = b[i] + nDst++ + } + nSrc += size + } + return +} + +// ReplaceIllFormed returns a transformer that replaces all input bytes that are +// not part of a well-formed UTF-8 code sequence with utf8.RuneError. +func ReplaceIllFormed() Transformer { + return Transformer{&replaceIllFormed{}} +} + +type replaceIllFormed struct{ transform.NopResetter } + +func (t replaceIllFormed) Span(src []byte, atEOF bool) (n int, err error) { + for n < len(src) { + // ASCII fast path. + if src[n] < utf8.RuneSelf { + n++ + continue + } + + r, size := utf8.DecodeRune(src[n:]) + + // Look for a valid non-ASCII rune. + if r != utf8.RuneError || size != 1 { + n += size + continue + } + + // Look for short source data. + if !atEOF && !utf8.FullRune(src[n:]) { + err = transform.ErrShortSrc + break + } + + // We have an invalid rune. + err = transform.ErrEndOfSpan + break + } + return n, err +} + +func (t replaceIllFormed) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + for nSrc < len(src) { + // ASCII fast path. + if r := src[nSrc]; r < utf8.RuneSelf { + if nDst == len(dst) { + err = transform.ErrShortDst + break + } + dst[nDst] = r + nDst++ + nSrc++ + continue + } + + // Look for a valid non-ASCII rune. + if _, size := utf8.DecodeRune(src[nSrc:]); size != 1 { + if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { + err = transform.ErrShortDst + break + } + nDst += size + nSrc += size + continue + } + + // Look for short source data. + if !atEOF && !utf8.FullRune(src[nSrc:]) { + err = transform.ErrShortSrc + break + } + + // We have an invalid rune. + if nDst+3 > len(dst) { + err = transform.ErrShortDst + break + } + dst[nDst+0] = runeErrorString[0] + dst[nDst+1] = runeErrorString[1] + dst[nDst+2] = runeErrorString[2] + nDst += 3 + nSrc++ + } + return nDst, nSrc, err +} diff --git a/vendor/golang.org/x/text/runes/runes_test.go b/vendor/golang.org/x/text/runes/runes_test.go new file mode 100644 index 000000000..23c5bc952 --- /dev/null +++ b/vendor/golang.org/x/text/runes/runes_test.go @@ -0,0 +1,664 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runes + +import ( + "strings" + "testing" + "unicode/utf8" + + "golang.org/x/text/internal/testtext" + "golang.org/x/text/transform" +) + +type transformTest struct { + desc string + szDst int + atEOF bool + repl string + in string + out string // result string of first call to Transform + outFull string // transform of entire input string + err error + errSpan error + nSpan int + + t transform.SpanningTransformer +} + +const large = 10240 + +func (tt *transformTest) check(t *testing.T, i int) { + if tt.t == nil { + return + } + dst := make([]byte, tt.szDst) + src := []byte(tt.in) + nDst, nSrc, err := tt.t.Transform(dst, src, tt.atEOF) + if err != tt.err { + t.Errorf("%d:%s:error: got %v; want %v", i, tt.desc, err, tt.err) + } + if got := string(dst[:nDst]); got != tt.out { + t.Errorf("%d:%s:out: got %q; want %q", i, tt.desc, got, tt.out) + } + + // Calls tt.t.Transform for the remainder of the input. We use this to test + // the nSrc return value. + out := make([]byte, large) + n := copy(out, dst[:nDst]) + nDst, _, _ = tt.t.Transform(out[n:], src[nSrc:], true) + if got, want := string(out[:n+nDst]), tt.outFull; got != want { + t.Errorf("%d:%s:outFull: got %q; want %q", i, tt.desc, got, want) + } + + tt.t.Reset() + p := 0 + for ; p < len(tt.in) && p < len(tt.outFull) && tt.in[p] == tt.outFull[p]; p++ { + } + if tt.nSpan != 0 { + p = tt.nSpan + } + if n, err = tt.t.Span([]byte(tt.in), tt.atEOF); n != p || err != tt.errSpan { + t.Errorf("%d:%s:span: got %d, %v; want %d, %v", i, tt.desc, n, err, p, tt.errSpan) + } +} + +func idem(r rune) rune { return r } + +func TestMap(t *testing.T) { + runes := []rune{'a', 'ç', '中', '\U00012345', 'a'} + // Default mapper used for this test. + rotate := Map(func(r rune) rune { + for i, m := range runes { + if m == r { + return runes[i+1] + } + } + return r + }) + + for i, tt := range []transformTest{{ + desc: "empty", + szDst: large, + atEOF: true, + in: "", + out: "", + outFull: "", + t: rotate, + }, { + desc: "no change", + szDst: 1, + atEOF: true, + in: "b", + out: "b", + outFull: "b", + t: rotate, + }, { + desc: "short dst", + szDst: 2, + atEOF: true, + in: "aaaa", + out: "ç", + outFull: "çççç", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "short dst ascii, no change", + szDst: 2, + atEOF: true, + in: "bbb", + out: "bb", + outFull: "bbb", + err: transform.ErrShortDst, + t: rotate, + }, { + desc: "short dst writing error", + szDst: 2, + atEOF: false, + in: "a\x80", + out: "ç", + outFull: "ç\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "short dst writing incomplete rune", + szDst: 2, + atEOF: true, + in: "a\xc0", + out: "ç", + outFull: "ç\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "short dst, longer", + szDst: 5, + atEOF: true, + in: "Hellø", + out: "Hell", + outFull: "Hellø", + err: transform.ErrShortDst, + t: rotate, + }, { + desc: "short dst, single", + szDst: 1, + atEOF: false, + in: "ø", + out: "", + outFull: "ø", + err: transform.ErrShortDst, + t: Map(idem), + }, { + desc: "short dst, longer, writing error", + szDst: 8, + atEOF: false, + in: "\x80Hello\x80", + out: "\ufffdHello", + outFull: "\ufffdHello\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "short src", + szDst: 2, + atEOF: false, + in: "a\xc2", + out: "ç", + outFull: "ç\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "invalid input, atEOF", + szDst: large, + atEOF: true, + in: "\x80", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "invalid input, !atEOF", + szDst: large, + atEOF: false, + in: "\x80", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "incomplete rune !atEOF", + szDst: large, + atEOF: false, + in: "\xc2", + out: "", + outFull: "\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrShortSrc, + t: rotate, + }, { + desc: "invalid input, incomplete rune atEOF", + szDst: large, + atEOF: true, + in: "\xc2", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "misc correct", + szDst: large, + atEOF: true, + in: "a\U00012345 ç!", + out: "ça 中!", + outFull: "ça 中!", + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "misc correct and invalid", + szDst: large, + atEOF: true, + in: "Hello\x80 w\x80orl\xc0d!\xc0", + out: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", + outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "misc correct and invalid, short src", + szDst: large, + atEOF: false, + in: "Hello\x80 w\x80orl\xc0d!\xc2", + out: "Hello\ufffd w\ufffdorl\ufffdd!", + outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrEndOfSpan, + t: rotate, + }, { + desc: "misc correct and invalid, short src, replacing RuneError", + szDst: large, + atEOF: false, + in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2", + out: "Hel?lo? w?orl?d!", + outFull: "Hel?lo? w?orl?d!?", + errSpan: transform.ErrEndOfSpan, + err: transform.ErrShortSrc, + t: Map(func(r rune) rune { + if r == utf8.RuneError { + return '?' + } + return r + }), + }} { + tt.check(t, i) + } +} + +func TestRemove(t *testing.T) { + remove := Remove(Predicate(func(r rune) bool { + return strings.ContainsRune("aeiou\u0300\uFF24\U00012345", r) + })) + + for i, tt := range []transformTest{ + 0: { + szDst: large, + atEOF: true, + in: "", + out: "", + outFull: "", + t: remove, + }, + 1: { + szDst: 0, + atEOF: true, + in: "aaaa", + out: "", + outFull: "", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 2: { + szDst: 1, + atEOF: true, + in: "aaaa", + out: "", + outFull: "", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 3: { + szDst: 1, + atEOF: true, + in: "baaaa", + out: "b", + outFull: "b", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 4: { + szDst: 2, + atEOF: true, + in: "açaaa", + out: "ç", + outFull: "ç", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 5: { + szDst: 2, + atEOF: true, + in: "aaaç", + out: "ç", + outFull: "ç", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 6: { + szDst: 2, + atEOF: false, + in: "a\x80", + out: "", + outFull: "\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 7: { + szDst: 1, + atEOF: true, + in: "a\xc0", + out: "", + outFull: "\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 8: { + szDst: 1, + atEOF: false, + in: "a\xc2", + out: "", + outFull: "\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 9: { + szDst: large, + atEOF: true, + in: "\x80", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 10: { + szDst: large, + atEOF: false, + in: "\x80", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 11: { + szDst: large, + atEOF: true, + in: "\xc2", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 12: { + szDst: large, + atEOF: false, + in: "\xc2", + out: "", + outFull: "\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrShortSrc, + t: remove, + }, + 13: { + szDst: large, + atEOF: true, + in: "Hello \U00012345world!", + out: "Hll wrld!", + outFull: "Hll wrld!", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 14: { + szDst: large, + atEOF: true, + in: "Hello\x80 w\x80orl\xc0d!\xc0", + out: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd", + outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd", + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 15: { + szDst: large, + atEOF: false, + in: "Hello\x80 w\x80orl\xc0d!\xc2", + out: "Hll\ufffd w\ufffdrl\ufffdd!", + outFull: "Hll\ufffd w\ufffdrl\ufffdd!\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 16: { + szDst: large, + atEOF: false, + in: "Hel\ufffdlo\x80 w\x80orl\xc0d!\xc2", + out: "Hello world!", + outFull: "Hello world!", + err: transform.ErrShortSrc, + errSpan: transform.ErrEndOfSpan, + t: Remove(Predicate(func(r rune) bool { return r == utf8.RuneError })), + }, + 17: { + szDst: 4, + atEOF: true, + in: "Hellø", + out: "Hll", + outFull: "Hllø", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 18: { + szDst: 4, + atEOF: false, + in: "Hellø", + out: "Hll", + outFull: "Hllø", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 19: { + szDst: 8, + atEOF: false, + in: "\x80Hello\uFF24\x80", + out: "\ufffdHll", + outFull: "\ufffdHll\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: remove, + }, + 20: { + szDst: 8, + atEOF: false, + in: "Hllll", + out: "Hllll", + outFull: "Hllll", + t: remove, + }} { + tt.check(t, i) + } +} + +func TestReplaceIllFormed(t *testing.T) { + replace := ReplaceIllFormed() + + for i, tt := range []transformTest{ + 0: { + szDst: large, + atEOF: true, + in: "", + out: "", + outFull: "", + t: replace, + }, + 1: { + szDst: 1, + atEOF: true, + in: "aa", + out: "a", + outFull: "aa", + err: transform.ErrShortDst, + t: replace, + }, + 2: { + szDst: 1, + atEOF: true, + in: "a\x80", + out: "a", + outFull: "a\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 3: { + szDst: 1, + atEOF: true, + in: "a\xc2", + out: "a", + outFull: "a\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 4: { + szDst: large, + atEOF: true, + in: "\x80", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 5: { + szDst: large, + atEOF: false, + in: "\x80", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 6: { + szDst: large, + atEOF: true, + in: "\xc2", + out: "\ufffd", + outFull: "\ufffd", + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 7: { + szDst: large, + atEOF: false, + in: "\xc2", + out: "", + outFull: "\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrShortSrc, + t: replace, + }, + 8: { + szDst: large, + atEOF: true, + in: "Hello world!", + out: "Hello world!", + outFull: "Hello world!", + t: replace, + }, + 9: { + szDst: large, + atEOF: true, + in: "Hello\x80 w\x80orl\xc2d!\xc2", + out: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", + outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 10: { + szDst: large, + atEOF: false, + in: "Hello\x80 w\x80orl\xc2d!\xc2", + out: "Hello\ufffd w\ufffdorl\ufffdd!", + outFull: "Hello\ufffd w\ufffdorl\ufffdd!\ufffd", + err: transform.ErrShortSrc, + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 16: { + szDst: 10, + atEOF: false, + in: "\x80Hello\x80", + out: "\ufffdHello", + outFull: "\ufffdHello\ufffd", + err: transform.ErrShortDst, + errSpan: transform.ErrEndOfSpan, + t: replace, + }, + 17: { + szDst: 10, + atEOF: false, + in: "\ufffdHello\ufffd", + out: "\ufffdHello", + outFull: "\ufffdHello\ufffd", + err: transform.ErrShortDst, + t: replace, + }, + } { + tt.check(t, i) + } +} + +func TestMapAlloc(t *testing.T) { + if n := testtext.AllocsPerRun(3, func() { + Map(idem).Transform(nil, nil, false) + }); n > 0 { + t.Errorf("got %f; want 0", n) + } +} + +func rmNop(r rune) bool { return false } + +func TestRemoveAlloc(t *testing.T) { + if n := testtext.AllocsPerRun(3, func() { + Remove(Predicate(rmNop)).Transform(nil, nil, false) + }); n > 0 { + t.Errorf("got %f; want 0", n) + } +} + +func TestReplaceIllFormedAlloc(t *testing.T) { + if n := testtext.AllocsPerRun(3, func() { + ReplaceIllFormed().Transform(nil, nil, false) + }); n > 0 { + t.Errorf("got %f; want 0", n) + } +} + +func doBench(b *testing.B, t Transformer) { + for _, bc := range []struct{ name, data string }{ + {"ascii", testtext.ASCII}, + {"3byte", testtext.ThreeByteUTF8}, + } { + dst := make([]byte, 2*len(bc.data)) + src := []byte(bc.data) + + testtext.Bench(b, bc.name+"/transform", func(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + t.Transform(dst, src, true) + } + }) + src = t.Bytes(src) + t.Reset() + testtext.Bench(b, bc.name+"/span", func(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + t.Span(src, true) + } + }) + } +} + +func BenchmarkRemove(b *testing.B) { + doBench(b, Remove(Predicate(func(r rune) bool { return r == 'e' }))) +} + +func BenchmarkMapAll(b *testing.B) { + doBench(b, Map(func(r rune) rune { return 'a' })) +} + +func BenchmarkMapNone(b *testing.B) { + doBench(b, Map(func(r rune) rune { return r })) +} + +func BenchmarkReplaceIllFormed(b *testing.B) { + doBench(b, ReplaceIllFormed()) +} + +var ( + input = strings.Repeat("Thé qüick brøwn føx jumps øver the lazy døg. ", 100) +) -- cgit v1.2.3-1-g7c22