From 2fa7c464f019f67c5c0494aaf5ac0f5ecc1ee7a7 Mon Sep 17 00:00:00 2001 From: Harrison Healey Date: Tue, 16 Jan 2018 12:03:31 -0500 Subject: Updated dependencies and added avct/uasurfer (#8089) * Updated dependencies and added avct/uasurfer * Added uasurfer to NOTICE.txt --- .../golang.org/x/text/message/pipeline/pipeline.go | 422 +++++++++++++++++++++ 1 file changed, 422 insertions(+) create mode 100644 vendor/golang.org/x/text/message/pipeline/pipeline.go (limited to 'vendor/golang.org/x/text/message/pipeline/pipeline.go') diff --git a/vendor/golang.org/x/text/message/pipeline/pipeline.go b/vendor/golang.org/x/text/message/pipeline/pipeline.go new file mode 100644 index 000000000..cafd6f29b --- /dev/null +++ b/vendor/golang.org/x/text/message/pipeline/pipeline.go @@ -0,0 +1,422 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pipeline provides tools for creating translation pipelines. +// +// NOTE: UNDER DEVELOPMENT. API MAY CHANGE. +package pipeline + +import ( + "bytes" + "encoding/json" + "fmt" + "go/build" + "go/parser" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "text/template" + "unicode" + + "golang.org/x/text/internal" + "golang.org/x/text/language" + "golang.org/x/text/runes" + "golang.org/x/tools/go/loader" +) + +const ( + extractFile = "extracted.gotext.json" + outFile = "out.gotext.json" + gotextSuffix = "gotext.json" +) + +// Config contains configuration for the translation pipeline. +type Config struct { + // Supported indicates the languages for which data should be generated. + // The default is to support all locales for which there are matching + // translation files. + Supported []language.Tag + + // --- Extraction + + SourceLanguage language.Tag + + Packages []string + + // --- File structure + + // Dir is the root dir for all operations. + Dir string + + // TranslationsPattern is a regular expression to match incoming translation + // files. These files may appear in any directory rooted at Dir. + // language for the translation files is determined as follows: + // 1. From the Language field in the file. + // 2. If not present, from a valid language tag in the filename, separated + // by dots (e.g. "en-US.json" or "incoming.pt_PT.xmb"). + // 3. If not present, from a the closest subdirectory in which the file + // is contained that parses as a valid language tag. + TranslationsPattern string + + // OutPattern defines the location for translation files for a certain + // language. The default is "{{.Dir}}/{{.Language}}/out.{{.Ext}}" + OutPattern string + + // Format defines the file format for generated translation files. + // The default is XMB. Alternatives are GetText, XLIFF, L20n, GoText. + Format string + + Ext string + + // TODO: + // Actions are additional actions to be performed after the initial extract + // and merge. + // Actions []struct { + // Name string + // Options map[string]string + // } + + // --- Generation + + // GenFile may be in a different package. It is not defined, it will + // be written to stdout. + GenFile string + + // GenPackage is the package or relative path into which to generate the + // file. If not specified it is relative to the current directory. + GenPackage string + + // DeclareVar defines a variable to which to assing the generated Catalog. + DeclareVar string + + // SetDefault determines whether to assign the generated Catalog to + // message.DefaultCatalog. The default for this is true if DeclareVar is + // not defined, false otherwise. + SetDefault bool + + // TODO: + // - Printf-style configuration + // - Template-style configuration + // - Extraction options + // - Rewrite options + // - Generation options +} + +// Operations: +// - extract: get the strings +// - disambiguate: find messages with the same key, but possible different meaning. +// - create out: create a list of messages that need translations +// - load trans: load the list of current translations +// - merge: assign list of translations as done +// - (action)expand: analyze features and create example sentences for each version. +// - (action)googletrans: pre-populate messages with automatic translations. +// - (action)export: send out messages somewhere non-standard +// - (action)import: load messages from somewhere non-standard +// - vet program: don't pass "foo" + var + "bar" strings. Not using funcs for translated strings. +// - vet trans: coverage: all translations/ all features. +// - generate: generate Go code + +// State holds all accumulated information on translations during processing. +type State struct { + Config Config + + Package string + program *loader.Program + + Extracted Messages `json:"messages"` + + // Messages includes all messages for which there need to be translations. + // Duplicates may be eliminated. Generation will be done from these messages + // (usually after merging). + Messages []Messages + + // Translations are incoming translations for the application messages. + Translations []Messages +} + +func (s *State) dir() string { + if d := s.Config.Dir; d != "" { + return d + } + return "./locales" +} + +func outPattern(s *State) (string, error) { + c := s.Config + pat := c.OutPattern + if pat == "" { + pat = "{{.Dir}}/{{.Language}}/out.{{.Ext}}" + } + + ext := c.Ext + if ext == "" { + ext = c.Format + } + if ext == "" { + ext = gotextSuffix + } + t, err := template.New("").Parse(pat) + if err != nil { + return "", wrap(err, "error parsing template") + } + buf := bytes.Buffer{} + err = t.Execute(&buf, map[string]string{ + "Dir": s.dir(), + "Language": "%s", + "Ext": ext, + }) + return filepath.FromSlash(buf.String()), wrap(err, "incorrect OutPattern") +} + +var transRE = regexp.MustCompile(`.*\.` + gotextSuffix) + +// Import loads existing translation files. +func (s *State) Import() error { + outPattern, err := outPattern(s) + if err != nil { + return err + } + re := transRE + if pat := s.Config.TranslationsPattern; pat != "" { + if re, err = regexp.Compile(pat); err != nil { + return wrapf(err, "error parsing regexp %q", s.Config.TranslationsPattern) + } + } + x := importer{s, outPattern, re} + return x.walkImport(s.dir(), s.Config.SourceLanguage) +} + +type importer struct { + state *State + outPattern string + transFile *regexp.Regexp +} + +func (i *importer) walkImport(path string, tag language.Tag) error { + files, err := ioutil.ReadDir(path) + if err != nil { + return nil + } + for _, f := range files { + name := f.Name() + tag := tag + if f.IsDir() { + if t, err := language.Parse(name); err == nil { + tag = t + } + // We ignore errors + if err := i.walkImport(filepath.Join(path, name), tag); err != nil { + return err + } + continue + } + for _, l := range strings.Split(name, ".") { + if t, err := language.Parse(l); err == nil { + tag = t + } + } + file := filepath.Join(path, name) + // TODO: Should we skip files that match output files? + if fmt.Sprintf(i.outPattern, tag) == file { + continue + } + // TODO: handle different file formats. + if !i.transFile.MatchString(name) { + continue + } + b, err := ioutil.ReadFile(file) + if err != nil { + return wrap(err, "read file failed") + } + var translations Messages + if err := json.Unmarshal(b, &translations); err != nil { + return wrap(err, "parsing translation file failed") + } + i.state.Translations = append(i.state.Translations, translations) + } + return nil +} + +// Merge merges the extracted messages with the existing translations. +func (s *State) Merge() error { + if s.Messages != nil { + panic("already merged") + } + // Create an index for each unique message. + // Duplicates are okay as long as the substitution arguments are okay as + // well. + // Top-level messages are okay to appear in multiple substitution points. + + // Collect key equivalence. + msgs := []*Message{} + keyToIDs := map[string]*Message{} + for _, m := range s.Extracted.Messages { + m := m + if prev, ok := keyToIDs[m.Key]; ok { + if err := checkEquivalence(&m, prev); err != nil { + warnf("Key %q matches conflicting messages: %v and %v", m.Key, prev.ID, m.ID) + // TODO: track enough information so that the rewriter can + // suggest/disambiguate messages. + } + // TODO: add position to message. + continue + } + i := len(msgs) + msgs = append(msgs, &m) + keyToIDs[m.Key] = msgs[i] + } + + // Messages with different keys may still refer to the same translated + // message (e.g. different whitespace). Filter these. + idMap := map[string]bool{} + filtered := []*Message{} + for _, m := range msgs { + found := false + for _, id := range m.ID { + found = found || idMap[id] + } + if !found { + filtered = append(filtered, m) + } + for _, id := range m.ID { + idMap[id] = true + } + } + + // Build index of translations. + translations := map[language.Tag]map[string]Message{} + languages := append([]language.Tag{}, s.Config.Supported...) + + for _, t := range s.Translations { + tag := t.Language + if _, ok := translations[tag]; !ok { + translations[tag] = map[string]Message{} + languages = append(languages, tag) + } + for _, m := range t.Messages { + if !m.Translation.IsEmpty() { + for _, id := range m.ID { + if _, ok := translations[tag][id]; ok { + warnf("Duplicate translation in locale %q for message %q", tag, id) + } + translations[tag][id] = m + } + } + } + } + languages = internal.UniqueTags(languages) + + for _, tag := range languages { + ms := Messages{Language: tag} + for _, orig := range filtered { + m := *orig + m.Key = "" + m.Position = "" + + for _, id := range m.ID { + if t, ok := translations[tag][id]; ok { + m.Translation = t.Translation + if t.TranslatorComment != "" { + m.TranslatorComment = t.TranslatorComment + m.Fuzzy = t.Fuzzy + } + break + } + } + if tag == s.Config.SourceLanguage && m.Translation.IsEmpty() { + m.Translation = m.Message + if m.TranslatorComment == "" { + m.TranslatorComment = "Copied from source." + m.Fuzzy = true + } + } + // TODO: if translation is empty: pre-expand based on available + // linguistic features. This may also be done as a plugin. + ms.Messages = append(ms.Messages, m) + } + s.Messages = append(s.Messages, ms) + } + return nil +} + +// Export writes out the messages to translation out files. +func (s *State) Export() error { + path, err := outPattern(s) + if err != nil { + return wrap(err, "export failed") + } + for _, out := range s.Messages { + // TODO: inject translations from existing files to avoid retranslation. + data, err := json.MarshalIndent(out, "", " ") + if err != nil { + return wrap(err, "JSON marshal failed") + } + file := fmt.Sprintf(path, out.Language) + if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { + return wrap(err, "dir create failed") + } + if err := ioutil.WriteFile(file, data, 0644); err != nil { + return wrap(err, "write failed") + } + } + return nil +} + +var ( + ws = runes.In(unicode.White_Space).Contains + notWS = runes.NotIn(unicode.White_Space).Contains +) + +func trimWS(s string) (trimmed, leadWS, trailWS string) { + trimmed = strings.TrimRightFunc(s, ws) + trailWS = s[len(trimmed):] + if i := strings.IndexFunc(trimmed, notWS); i > 0 { + leadWS = trimmed[:i] + trimmed = trimmed[i:] + } + return trimmed, leadWS, trailWS +} + +// NOTE: The command line tool already prefixes with "gotext:". +var ( + wrap = func(err error, msg string) error { + if err == nil { + return nil + } + return fmt.Errorf("%s: %v", msg, err) + } + wrapf = func(err error, msg string, args ...interface{}) error { + if err == nil { + return nil + } + return wrap(err, fmt.Sprintf(msg, args...)) + } + errorf = fmt.Errorf +) + +func warnf(format string, args ...interface{}) { + // TODO: don't log. + log.Printf(format, args...) +} + +func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) { + if len(args) == 0 { + args = []string{"."} + } + + conf.Build = &build.Default + conf.ParserMode = parser.ParseComments + + // Use the initial packages from the command line. + args, err := conf.FromArgs(args, false) + if err != nil { + return nil, wrap(err, "loading packages failed") + } + + // Load, parse and type-check the whole program. + return conf.Load() +} -- cgit v1.2.3-1-g7c22