diff options
Diffstat (limited to 'vendor/golang.org/x/text/message/pipeline/pipeline.go')
| -rw-r--r-- | vendor/golang.org/x/text/message/pipeline/pipeline.go | 422 |
1 files changed, 0 insertions, 422 deletions
diff --git a/vendor/golang.org/x/text/message/pipeline/pipeline.go b/vendor/golang.org/x/text/message/pipeline/pipeline.go deleted file mode 100644 index cafd6f29b..000000000 --- a/vendor/golang.org/x/text/message/pipeline/pipeline.go +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package pipeline provides tools for creating translation pipelines. -// -// NOTE: UNDER DEVELOPMENT. API MAY CHANGE. -package pipeline - -import ( - "bytes" - "encoding/json" - "fmt" - "go/build" - "go/parser" - "io/ioutil" - "log" - "os" - "path/filepath" - "regexp" - "strings" - "text/template" - "unicode" - - "golang.org/x/text/internal" - "golang.org/x/text/language" - "golang.org/x/text/runes" - "golang.org/x/tools/go/loader" -) - -const ( - extractFile = "extracted.gotext.json" - outFile = "out.gotext.json" - gotextSuffix = "gotext.json" -) - -// Config contains configuration for the translation pipeline. -type Config struct { - // Supported indicates the languages for which data should be generated. - // The default is to support all locales for which there are matching - // translation files. - Supported []language.Tag - - // --- Extraction - - SourceLanguage language.Tag - - Packages []string - - // --- File structure - - // Dir is the root dir for all operations. - Dir string - - // TranslationsPattern is a regular expression to match incoming translation - // files. These files may appear in any directory rooted at Dir. - // language for the translation files is determined as follows: - // 1. From the Language field in the file. - // 2. If not present, from a valid language tag in the filename, separated - // by dots (e.g. "en-US.json" or "incoming.pt_PT.xmb"). - // 3. If not present, from a the closest subdirectory in which the file - // is contained that parses as a valid language tag. - TranslationsPattern string - - // OutPattern defines the location for translation files for a certain - // language. The default is "{{.Dir}}/{{.Language}}/out.{{.Ext}}" - OutPattern string - - // Format defines the file format for generated translation files. - // The default is XMB. Alternatives are GetText, XLIFF, L20n, GoText. - Format string - - Ext string - - // TODO: - // Actions are additional actions to be performed after the initial extract - // and merge. - // Actions []struct { - // Name string - // Options map[string]string - // } - - // --- Generation - - // GenFile may be in a different package. It is not defined, it will - // be written to stdout. - GenFile string - - // GenPackage is the package or relative path into which to generate the - // file. If not specified it is relative to the current directory. - GenPackage string - - // DeclareVar defines a variable to which to assing the generated Catalog. - DeclareVar string - - // SetDefault determines whether to assign the generated Catalog to - // message.DefaultCatalog. The default for this is true if DeclareVar is - // not defined, false otherwise. - SetDefault bool - - // TODO: - // - Printf-style configuration - // - Template-style configuration - // - Extraction options - // - Rewrite options - // - Generation options -} - -// Operations: -// - extract: get the strings -// - disambiguate: find messages with the same key, but possible different meaning. -// - create out: create a list of messages that need translations -// - load trans: load the list of current translations -// - merge: assign list of translations as done -// - (action)expand: analyze features and create example sentences for each version. -// - (action)googletrans: pre-populate messages with automatic translations. -// - (action)export: send out messages somewhere non-standard -// - (action)import: load messages from somewhere non-standard -// - vet program: don't pass "foo" + var + "bar" strings. Not using funcs for translated strings. -// - vet trans: coverage: all translations/ all features. -// - generate: generate Go code - -// State holds all accumulated information on translations during processing. -type State struct { - Config Config - - Package string - program *loader.Program - - Extracted Messages `json:"messages"` - - // Messages includes all messages for which there need to be translations. - // Duplicates may be eliminated. Generation will be done from these messages - // (usually after merging). - Messages []Messages - - // Translations are incoming translations for the application messages. - Translations []Messages -} - -func (s *State) dir() string { - if d := s.Config.Dir; d != "" { - return d - } - return "./locales" -} - -func outPattern(s *State) (string, error) { - c := s.Config - pat := c.OutPattern - if pat == "" { - pat = "{{.Dir}}/{{.Language}}/out.{{.Ext}}" - } - - ext := c.Ext - if ext == "" { - ext = c.Format - } - if ext == "" { - ext = gotextSuffix - } - t, err := template.New("").Parse(pat) - if err != nil { - return "", wrap(err, "error parsing template") - } - buf := bytes.Buffer{} - err = t.Execute(&buf, map[string]string{ - "Dir": s.dir(), - "Language": "%s", - "Ext": ext, - }) - return filepath.FromSlash(buf.String()), wrap(err, "incorrect OutPattern") -} - -var transRE = regexp.MustCompile(`.*\.` + gotextSuffix) - -// Import loads existing translation files. -func (s *State) Import() error { - outPattern, err := outPattern(s) - if err != nil { - return err - } - re := transRE - if pat := s.Config.TranslationsPattern; pat != "" { - if re, err = regexp.Compile(pat); err != nil { - return wrapf(err, "error parsing regexp %q", s.Config.TranslationsPattern) - } - } - x := importer{s, outPattern, re} - return x.walkImport(s.dir(), s.Config.SourceLanguage) -} - -type importer struct { - state *State - outPattern string - transFile *regexp.Regexp -} - -func (i *importer) walkImport(path string, tag language.Tag) error { - files, err := ioutil.ReadDir(path) - if err != nil { - return nil - } - for _, f := range files { - name := f.Name() - tag := tag - if f.IsDir() { - if t, err := language.Parse(name); err == nil { - tag = t - } - // We ignore errors - if err := i.walkImport(filepath.Join(path, name), tag); err != nil { - return err - } - continue - } - for _, l := range strings.Split(name, ".") { - if t, err := language.Parse(l); err == nil { - tag = t - } - } - file := filepath.Join(path, name) - // TODO: Should we skip files that match output files? - if fmt.Sprintf(i.outPattern, tag) == file { - continue - } - // TODO: handle different file formats. - if !i.transFile.MatchString(name) { - continue - } - b, err := ioutil.ReadFile(file) - if err != nil { - return wrap(err, "read file failed") - } - var translations Messages - if err := json.Unmarshal(b, &translations); err != nil { - return wrap(err, "parsing translation file failed") - } - i.state.Translations = append(i.state.Translations, translations) - } - return nil -} - -// Merge merges the extracted messages with the existing translations. -func (s *State) Merge() error { - if s.Messages != nil { - panic("already merged") - } - // Create an index for each unique message. - // Duplicates are okay as long as the substitution arguments are okay as - // well. - // Top-level messages are okay to appear in multiple substitution points. - - // Collect key equivalence. - msgs := []*Message{} - keyToIDs := map[string]*Message{} - for _, m := range s.Extracted.Messages { - m := m - if prev, ok := keyToIDs[m.Key]; ok { - if err := checkEquivalence(&m, prev); err != nil { - warnf("Key %q matches conflicting messages: %v and %v", m.Key, prev.ID, m.ID) - // TODO: track enough information so that the rewriter can - // suggest/disambiguate messages. - } - // TODO: add position to message. - continue - } - i := len(msgs) - msgs = append(msgs, &m) - keyToIDs[m.Key] = msgs[i] - } - - // Messages with different keys may still refer to the same translated - // message (e.g. different whitespace). Filter these. - idMap := map[string]bool{} - filtered := []*Message{} - for _, m := range msgs { - found := false - for _, id := range m.ID { - found = found || idMap[id] - } - if !found { - filtered = append(filtered, m) - } - for _, id := range m.ID { - idMap[id] = true - } - } - - // Build index of translations. - translations := map[language.Tag]map[string]Message{} - languages := append([]language.Tag{}, s.Config.Supported...) - - for _, t := range s.Translations { - tag := t.Language - if _, ok := translations[tag]; !ok { - translations[tag] = map[string]Message{} - languages = append(languages, tag) - } - for _, m := range t.Messages { - if !m.Translation.IsEmpty() { - for _, id := range m.ID { - if _, ok := translations[tag][id]; ok { - warnf("Duplicate translation in locale %q for message %q", tag, id) - } - translations[tag][id] = m - } - } - } - } - languages = internal.UniqueTags(languages) - - for _, tag := range languages { - ms := Messages{Language: tag} - for _, orig := range filtered { - m := *orig - m.Key = "" - m.Position = "" - - for _, id := range m.ID { - if t, ok := translations[tag][id]; ok { - m.Translation = t.Translation - if t.TranslatorComment != "" { - m.TranslatorComment = t.TranslatorComment - m.Fuzzy = t.Fuzzy - } - break - } - } - if tag == s.Config.SourceLanguage && m.Translation.IsEmpty() { - m.Translation = m.Message - if m.TranslatorComment == "" { - m.TranslatorComment = "Copied from source." - m.Fuzzy = true - } - } - // TODO: if translation is empty: pre-expand based on available - // linguistic features. This may also be done as a plugin. - ms.Messages = append(ms.Messages, m) - } - s.Messages = append(s.Messages, ms) - } - return nil -} - -// Export writes out the messages to translation out files. -func (s *State) Export() error { - path, err := outPattern(s) - if err != nil { - return wrap(err, "export failed") - } - for _, out := range s.Messages { - // TODO: inject translations from existing files to avoid retranslation. - data, err := json.MarshalIndent(out, "", " ") - if err != nil { - return wrap(err, "JSON marshal failed") - } - file := fmt.Sprintf(path, out.Language) - if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { - return wrap(err, "dir create failed") - } - if err := ioutil.WriteFile(file, data, 0644); err != nil { - return wrap(err, "write failed") - } - } - return nil -} - -var ( - ws = runes.In(unicode.White_Space).Contains - notWS = runes.NotIn(unicode.White_Space).Contains -) - -func trimWS(s string) (trimmed, leadWS, trailWS string) { - trimmed = strings.TrimRightFunc(s, ws) - trailWS = s[len(trimmed):] - if i := strings.IndexFunc(trimmed, notWS); i > 0 { - leadWS = trimmed[:i] - trimmed = trimmed[i:] - } - return trimmed, leadWS, trailWS -} - -// NOTE: The command line tool already prefixes with "gotext:". -var ( - wrap = func(err error, msg string) error { - if err == nil { - return nil - } - return fmt.Errorf("%s: %v", msg, err) - } - wrapf = func(err error, msg string, args ...interface{}) error { - if err == nil { - return nil - } - return wrap(err, fmt.Sprintf(msg, args...)) - } - errorf = fmt.Errorf -) - -func warnf(format string, args ...interface{}) { - // TODO: don't log. - log.Printf(format, args...) -} - -func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) { - if len(args) == 0 { - args = []string{"."} - } - - conf.Build = &build.Default - conf.ParserMode = parser.ParseComments - - // Use the initial packages from the command line. - args, err := conf.FromArgs(args, false) - if err != nil { - return nil, wrap(err, "loading packages failed") - } - - // Load, parse and type-check the whole program. - return conf.Load() -} |