From 961c04cae992eadb42d286d2f85f8a675bdc68c8 Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Mon, 29 Jan 2018 14:17:40 -0800 Subject: Upgrading server dependancies (#8154) --- .../google.golang.org/appengine/search/search.go | 1189 ++++++++++++++++++++ 1 file changed, 1189 insertions(+) create mode 100644 vendor/google.golang.org/appengine/search/search.go (limited to 'vendor/google.golang.org/appengine/search/search.go') diff --git a/vendor/google.golang.org/appengine/search/search.go b/vendor/google.golang.org/appengine/search/search.go new file mode 100644 index 000000000..35a567d62 --- /dev/null +++ b/vendor/google.golang.org/appengine/search/search.go @@ -0,0 +1,1189 @@ +// Copyright 2012 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package search // import "google.golang.org/appengine/search" + +// TODO: let Put specify the document language: "en", "fr", etc. Also: order_id?? storage?? +// TODO: Index.GetAll (or Iterator.GetAll)? +// TODO: struct <-> protobuf tests. +// TODO: enforce Python's MIN_NUMBER_VALUE and MIN_DATE (which would disallow a zero +// time.Time)? _MAXIMUM_STRING_LENGTH? + +import ( + "errors" + "fmt" + "math" + "reflect" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + + "google.golang.org/appengine" + "google.golang.org/appengine/internal" + pb "google.golang.org/appengine/internal/search" +) + +const maxDocumentsPerPutDelete = 200 + +var ( + // ErrInvalidDocumentType is returned when methods like Put, Get or Next + // are passed a dst or src argument of invalid type. + ErrInvalidDocumentType = errors.New("search: invalid document type") + + // ErrNoSuchDocument is returned when no document was found for a given ID. + ErrNoSuchDocument = errors.New("search: no such document") + + // ErrTooManyDocuments is returned when the user passes too many documents to + // PutMulti or DeleteMulti. + ErrTooManyDocuments = fmt.Errorf("search: too many documents given to put or delete (max is %d)", maxDocumentsPerPutDelete) +) + +// Atom is a document field whose contents are indexed as a single indivisible +// string. +type Atom string + +// HTML is a document field whose contents are indexed as HTML. Only text nodes +// are indexed: "foobar" will be treated as "foobar". +type HTML string + +// validIndexNameOrDocID is the Go equivalent of Python's +// _ValidateVisiblePrintableAsciiNotReserved. +func validIndexNameOrDocID(s string) bool { + if strings.HasPrefix(s, "!") { + return false + } + for _, c := range s { + if c < 0x21 || 0x7f <= c { + return false + } + } + return true +} + +var ( + fieldNameRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) + languageRE = regexp.MustCompile(`^[a-z]{2}$`) +) + +// validFieldName is the Go equivalent of Python's _CheckFieldName. It checks +// the validity of both field and facet names. +func validFieldName(s string) bool { + return len(s) <= 500 && fieldNameRE.MatchString(s) +} + +// validDocRank checks that the ranks is in the range [0, 2^31). +func validDocRank(r int) bool { + return 0 <= r && r <= (1<<31-1) +} + +// validLanguage checks that a language looks like ISO 639-1. +func validLanguage(s string) bool { + return languageRE.MatchString(s) +} + +// validFloat checks that f is in the range [-2147483647, 2147483647]. +func validFloat(f float64) bool { + return -(1<<31-1) <= f && f <= (1<<31-1) +} + +// Index is an index of documents. +type Index struct { + spec pb.IndexSpec +} + +// orderIDEpoch forms the basis for populating OrderId on documents. +var orderIDEpoch = time.Date(2011, 1, 1, 0, 0, 0, 0, time.UTC) + +// Open opens the index with the given name. The index is created if it does +// not already exist. +// +// The name is a human-readable ASCII string. It must contain no whitespace +// characters and not start with "!". +func Open(name string) (*Index, error) { + if !validIndexNameOrDocID(name) { + return nil, fmt.Errorf("search: invalid index name %q", name) + } + return &Index{ + spec: pb.IndexSpec{ + Name: &name, + }, + }, nil +} + +// Put saves src to the index. If id is empty, a new ID is allocated by the +// service and returned. If id is not empty, any existing index entry for that +// ID is replaced. +// +// The ID is a human-readable ASCII string. It must contain no whitespace +// characters and not start with "!". +// +// src must be a non-nil struct pointer or implement the FieldLoadSaver +// interface. +func (x *Index) Put(c context.Context, id string, src interface{}) (string, error) { + ids, err := x.PutMulti(c, []string{id}, []interface{}{src}) + if err != nil { + return "", err + } + return ids[0], nil +} + +// PutMulti is like Put, but is more efficient for adding multiple documents to +// the index at once. +// +// Up to 200 documents can be added at once. ErrTooManyDocuments is returned if +// you try to add more. +// +// ids can either be an empty slice (which means new IDs will be allocated for +// each of the documents added) or a slice the same size as srcs. +// +// The error may be an instance of appengine.MultiError, in which case it will +// be the same size as srcs and the individual errors inside will correspond +// with the items in srcs. +func (x *Index) PutMulti(c context.Context, ids []string, srcs []interface{}) ([]string, error) { + if len(ids) != 0 && len(srcs) != len(ids) { + return nil, fmt.Errorf("search: PutMulti expects ids and srcs slices of the same length") + } + if len(srcs) > maxDocumentsPerPutDelete { + return nil, ErrTooManyDocuments + } + + docs := make([]*pb.Document, len(srcs)) + for i, s := range srcs { + var err error + docs[i], err = saveDoc(s) + if err != nil { + return nil, err + } + + if len(ids) != 0 && ids[i] != "" { + if !validIndexNameOrDocID(ids[i]) { + return nil, fmt.Errorf("search: invalid ID %q", ids[i]) + } + docs[i].Id = proto.String(ids[i]) + } + } + + // spec is modified by Call when applying the current Namespace, so copy it to + // avoid retaining the namespace beyond the scope of the Call. + spec := x.spec + req := &pb.IndexDocumentRequest{ + Params: &pb.IndexDocumentParams{ + Document: docs, + IndexSpec: &spec, + }, + } + res := &pb.IndexDocumentResponse{} + if err := internal.Call(c, "search", "IndexDocument", req, res); err != nil { + return nil, err + } + multiErr, hasErr := make(appengine.MultiError, len(res.Status)), false + for i, s := range res.Status { + if s.GetCode() != pb.SearchServiceError_OK { + multiErr[i] = fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail()) + hasErr = true + } + } + if hasErr { + return res.DocId, multiErr + } + + if len(res.Status) != len(docs) || len(res.DocId) != len(docs) { + return nil, fmt.Errorf("search: internal error: wrong number of results (%d Statuses, %d DocIDs, expected %d)", + len(res.Status), len(res.DocId), len(docs)) + } + return res.DocId, nil +} + +// Get loads the document with the given ID into dst. +// +// The ID is a human-readable ASCII string. It must be non-empty, contain no +// whitespace characters and not start with "!". +// +// dst must be a non-nil struct pointer or implement the FieldLoadSaver +// interface. +// +// ErrFieldMismatch is returned when a field is to be loaded into a different +// type than the one it was stored from, or when a field is missing or +// unexported in the destination struct. ErrFieldMismatch is only returned if +// dst is a struct pointer. It is up to the callee to decide whether this error +// is fatal, recoverable, or ignorable. +func (x *Index) Get(c context.Context, id string, dst interface{}) error { + if id == "" || !validIndexNameOrDocID(id) { + return fmt.Errorf("search: invalid ID %q", id) + } + req := &pb.ListDocumentsRequest{ + Params: &pb.ListDocumentsParams{ + IndexSpec: &x.spec, + StartDocId: proto.String(id), + Limit: proto.Int32(1), + }, + } + res := &pb.ListDocumentsResponse{} + if err := internal.Call(c, "search", "ListDocuments", req, res); err != nil { + return err + } + if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK { + return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail()) + } + if len(res.Document) != 1 || res.Document[0].GetId() != id { + return ErrNoSuchDocument + } + return loadDoc(dst, res.Document[0], nil) +} + +// Delete deletes a document from the index. +func (x *Index) Delete(c context.Context, id string) error { + return x.DeleteMulti(c, []string{id}) +} + +// DeleteMulti deletes multiple documents from the index. +// +// The returned error may be an instance of appengine.MultiError, in which case +// it will be the same size as srcs and the individual errors inside will +// correspond with the items in srcs. +func (x *Index) DeleteMulti(c context.Context, ids []string) error { + if len(ids) > maxDocumentsPerPutDelete { + return ErrTooManyDocuments + } + + req := &pb.DeleteDocumentRequest{ + Params: &pb.DeleteDocumentParams{ + DocId: ids, + IndexSpec: &x.spec, + }, + } + res := &pb.DeleteDocumentResponse{} + if err := internal.Call(c, "search", "DeleteDocument", req, res); err != nil { + return err + } + if len(res.Status) != len(ids) { + return fmt.Errorf("search: internal error: wrong number of results (%d, expected %d)", + len(res.Status), len(ids)) + } + multiErr, hasErr := make(appengine.MultiError, len(ids)), false + for i, s := range res.Status { + if s.GetCode() != pb.SearchServiceError_OK { + multiErr[i] = fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail()) + hasErr = true + } + } + if hasErr { + return multiErr + } + return nil +} + +// List lists all of the documents in an index. The documents are returned in +// increasing ID order. +func (x *Index) List(c context.Context, opts *ListOptions) *Iterator { + t := &Iterator{ + c: c, + index: x, + count: -1, + listInclusive: true, + more: moreList, + } + if opts != nil { + t.listStartID = opts.StartID + t.limit = opts.Limit + t.idsOnly = opts.IDsOnly + } + return t +} + +func moreList(t *Iterator) error { + req := &pb.ListDocumentsRequest{ + Params: &pb.ListDocumentsParams{ + IndexSpec: &t.index.spec, + }, + } + if t.listStartID != "" { + req.Params.StartDocId = &t.listStartID + req.Params.IncludeStartDoc = &t.listInclusive + } + if t.limit > 0 { + req.Params.Limit = proto.Int32(int32(t.limit)) + } + if t.idsOnly { + req.Params.KeysOnly = &t.idsOnly + } + + res := &pb.ListDocumentsResponse{} + if err := internal.Call(t.c, "search", "ListDocuments", req, res); err != nil { + return err + } + if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK { + return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail()) + } + t.listRes = res.Document + t.listStartID, t.listInclusive, t.more = "", false, nil + if len(res.Document) != 0 && t.limit <= 0 { + if id := res.Document[len(res.Document)-1].GetId(); id != "" { + t.listStartID, t.more = id, moreList + } + } + return nil +} + +// ListOptions are the options for listing documents in an index. Passing a nil +// *ListOptions is equivalent to using the default values. +type ListOptions struct { + // StartID is the inclusive lower bound for the ID of the returned + // documents. The zero value means all documents will be returned. + StartID string + + // Limit is the maximum number of documents to return. The zero value + // indicates no limit. + Limit int + + // IDsOnly indicates that only document IDs should be returned for the list + // operation; no document fields are populated. + IDsOnly bool +} + +// Search searches the index for the given query. +func (x *Index) Search(c context.Context, query string, opts *SearchOptions) *Iterator { + t := &Iterator{ + c: c, + index: x, + searchQuery: query, + more: moreSearch, + } + if opts != nil { + if opts.Cursor != "" { + if opts.Offset != 0 { + return errIter("at most one of Cursor and Offset may be specified") + } + t.searchCursor = proto.String(string(opts.Cursor)) + } + t.limit = opts.Limit + t.fields = opts.Fields + t.idsOnly = opts.IDsOnly + t.sort = opts.Sort + t.exprs = opts.Expressions + t.refinements = opts.Refinements + t.facetOpts = opts.Facets + t.searchOffset = opts.Offset + t.countAccuracy = opts.CountAccuracy + } + return t +} + +func moreSearch(t *Iterator) error { + // We use per-result (rather than single/per-page) cursors since this + // lets us return a Cursor for every iterator document. The two cursor + // types are largely interchangeable: a page cursor is the same as the + // last per-result cursor in a given search response. + req := &pb.SearchRequest{ + Params: &pb.SearchParams{ + IndexSpec: &t.index.spec, + Query: &t.searchQuery, + Cursor: t.searchCursor, + CursorType: pb.SearchParams_PER_RESULT.Enum(), + FieldSpec: &pb.FieldSpec{ + Name: t.fields, + }, + }, + } + if t.limit > 0 { + req.Params.Limit = proto.Int32(int32(t.limit)) + } + if t.searchOffset > 0 { + req.Params.Offset = proto.Int32(int32(t.searchOffset)) + t.searchOffset = 0 + } + if t.countAccuracy > 0 { + req.Params.MatchedCountAccuracy = proto.Int32(int32(t.countAccuracy)) + } + if t.idsOnly { + req.Params.KeysOnly = &t.idsOnly + } + if t.sort != nil { + if err := sortToProto(t.sort, req.Params); err != nil { + return err + } + } + if t.refinements != nil { + if err := refinementsToProto(t.refinements, req.Params); err != nil { + return err + } + } + for _, e := range t.exprs { + req.Params.FieldSpec.Expression = append(req.Params.FieldSpec.Expression, &pb.FieldSpec_Expression{ + Name: proto.String(e.Name), + Expression: proto.String(e.Expr), + }) + } + for _, f := range t.facetOpts { + if err := f.setParams(req.Params); err != nil { + return fmt.Errorf("bad FacetSearchOption: %v", err) + } + } + // Don't repeat facet search. + t.facetOpts = nil + + res := &pb.SearchResponse{} + if err := internal.Call(t.c, "search", "Search", req, res); err != nil { + return err + } + if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK { + return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail()) + } + t.searchRes = res.Result + if len(res.FacetResult) > 0 { + t.facetRes = res.FacetResult + } + t.count = int(*res.MatchedCount) + if t.limit > 0 { + t.more = nil + } else { + t.more = moreSearch + } + return nil +} + +// SearchOptions are the options for searching an index. Passing a nil +// *SearchOptions is equivalent to using the default values. +type SearchOptions struct { + // Limit is the maximum number of documents to return. The zero value + // indicates no limit. + Limit int + + // IDsOnly indicates that only document IDs should be returned for the search + // operation; no document fields are populated. + IDsOnly bool + + // Sort controls the ordering of search results. + Sort *SortOptions + + // Fields specifies which document fields to include in the results. If omitted, + // all document fields are returned. No more than 100 fields may be specified. + Fields []string + + // Expressions specifies additional computed fields to add to each returned + // document. + Expressions []FieldExpression + + // Facets controls what facet information is returned for these search results. + // If no options are specified, no facet results will be returned. + Facets []FacetSearchOption + + // Refinements filters the returned documents by requiring them to contain facets + // with specific values. Refinements are applied in conjunction for facets with + // different names, and in disjunction otherwise. + Refinements []Facet + + // Cursor causes the results to commence with the first document after + // the document associated with the cursor. + Cursor Cursor + + // Offset specifies the number of documents to skip over before returning results. + // When specified, Cursor must be nil. + Offset int + + // CountAccuracy specifies the maximum result count that can be expected to + // be accurate. If zero, the count accuracy defaults to 20. + CountAccuracy int +} + +// Cursor represents an iterator's position. +// +// The string value of a cursor is web-safe. It can be saved and restored +// for later use. +type Cursor string + +// FieldExpression defines a custom expression to evaluate for each result. +type FieldExpression struct { + // Name is the name to use for the computed field. + Name string + + // Expr is evaluated to provide a custom content snippet for each document. + // See https://cloud.google.com/appengine/docs/standard/go/search/options for + // the supported expression syntax. + Expr string +} + +// FacetSearchOption controls what facet information is returned in search results. +type FacetSearchOption interface { + setParams(*pb.SearchParams) error +} + +// AutoFacetDiscovery returns a FacetSearchOption which enables automatic facet +// discovery for the search. Automatic facet discovery looks for the facets +// which appear the most often in the aggregate in the matched documents. +// +// The maximum number of facets returned is controlled by facetLimit, and the +// maximum number of values per facet by facetLimit. A limit of zero indicates +// a default limit should be used. +func AutoFacetDiscovery(facetLimit, valueLimit int) FacetSearchOption { + return &autoFacetOpt{facetLimit, valueLimit} +} + +type autoFacetOpt struct { + facetLimit, valueLimit int +} + +const defaultAutoFacetLimit = 10 // As per python runtime search.py. + +func (o *autoFacetOpt) setParams(params *pb.SearchParams) error { + lim := int32(o.facetLimit) + if lim == 0 { + lim = defaultAutoFacetLimit + } + params.AutoDiscoverFacetCount = &lim + if o.valueLimit > 0 { + params.FacetAutoDetectParam = &pb.FacetAutoDetectParam{ + ValueLimit: proto.Int32(int32(o.valueLimit)), + } + } + return nil +} + +// FacetDiscovery returns a FacetSearchOption which selects a facet to be +// returned with the search results. By default, the most frequently +// occurring values for that facet will be returned. However, you can also +// specify a list of particular Atoms or specific Ranges to return. +func FacetDiscovery(name string, value ...interface{}) FacetSearchOption { + return &facetOpt{name, value} +} + +type facetOpt struct { + name string + values []interface{} +} + +func (o *facetOpt) setParams(params *pb.SearchParams) error { + req := &pb.FacetRequest{Name: &o.name} + params.IncludeFacet = append(params.IncludeFacet, req) + if len(o.values) == 0 { + return nil + } + vtype := reflect.TypeOf(o.values[0]) + reqParam := &pb.FacetRequestParam{} + for _, v := range o.values { + if reflect.TypeOf(v) != vtype { + return errors.New("values must all be Atom, or must all be Range") + } + switch v := v.(type) { + case Atom: + reqParam.ValueConstraint = append(reqParam.ValueConstraint, string(v)) + case Range: + rng, err := rangeToProto(v) + if err != nil { + return fmt.Errorf("invalid range: %v", err) + } + reqParam.Range = append(reqParam.Range, rng) + default: + return fmt.Errorf("unsupported value type %T", v) + } + } + req.Params = reqParam + return nil +} + +// FacetDocumentDepth returns a FacetSearchOption which controls the number of +// documents to be evaluated with preparing facet results. +func FacetDocumentDepth(depth int) FacetSearchOption { + return facetDepthOpt(depth) +} + +type facetDepthOpt int + +func (o facetDepthOpt) setParams(params *pb.SearchParams) error { + params.FacetDepth = proto.Int32(int32(o)) + return nil +} + +// FacetResult represents the number of times a particular facet and value +// appeared in the documents matching a search request. +type FacetResult struct { + Facet + + // Count is the number of times this specific facet and value appeared in the + // matching documents. + Count int +} + +// Range represents a numeric range with inclusive start and exclusive end. +// Start may be specified as math.Inf(-1) to indicate there is no minimum +// value, and End may similarly be specified as math.Inf(1); at least one of +// Start or End must be a finite number. +type Range struct { + Start, End float64 +} + +var ( + negInf = math.Inf(-1) + posInf = math.Inf(1) +) + +// AtLeast returns a Range matching any value greater than, or equal to, min. +func AtLeast(min float64) Range { + return Range{Start: min, End: posInf} +} + +// LessThan returns a Range matching any value less than max. +func LessThan(max float64) Range { + return Range{Start: negInf, End: max} +} + +// SortOptions control the ordering and scoring of search results. +type SortOptions struct { + // Expressions is a slice of expressions representing a multi-dimensional + // sort. + Expressions []SortExpression + + // Scorer, when specified, will cause the documents to be scored according to + // search term frequency. + Scorer Scorer + + // Limit is the maximum number of objects to score and/or sort. Limit cannot + // be more than 10,000. The zero value indicates a default limit. + Limit int +} + +// SortExpression defines a single dimension for sorting a document. +type SortExpression struct { + // Expr is evaluated to provide a sorting value for each document. + // See https://cloud.google.com/appengine/docs/standard/go/search/options for + // the supported expression syntax. + Expr string + + // Reverse causes the documents to be sorted in ascending order. + Reverse bool + + // The default value to use when no field is present or the expresion + // cannot be calculated for a document. For text sorts, Default must + // be of type string; for numeric sorts, float64. + Default interface{} +} + +// A Scorer defines how a document is scored. +type Scorer interface { + toProto(*pb.ScorerSpec) +} + +type enumScorer struct { + enum pb.ScorerSpec_Scorer +} + +func (e enumScorer) toProto(spec *pb.ScorerSpec) { + spec.Scorer = e.enum.Enum() +} + +var ( + // MatchScorer assigns a score based on term frequency in a document. + MatchScorer Scorer = enumScorer{pb.ScorerSpec_MATCH_SCORER} + + // RescoringMatchScorer assigns a score based on the quality of the query + // match. It is similar to a MatchScorer but uses a more complex scoring + // algorithm based on match term frequency and other factors like field type. + // Please be aware that this algorithm is continually refined and can change + // over time without notice. This means that the ordering of search results + // that use this scorer can also change without notice. + RescoringMatchScorer Scorer = enumScorer{pb.ScorerSpec_RESCORING_MATCH_SCORER} +) + +func sortToProto(sort *SortOptions, params *pb.SearchParams) error { + for _, e := range sort.Expressions { + spec := &pb.SortSpec{ + SortExpression: proto.String(e.Expr), + } + if e.Reverse { + spec.SortDescending = proto.Bool(false) + } + if e.Default != nil { + switch d := e.Default.(type) { + case float64: + spec.DefaultValueNumeric = &d + case string: + spec.DefaultValueText = &d + default: + return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr) + } + } + params.SortSpec = append(params.SortSpec, spec) + } + + spec := &pb.ScorerSpec{} + if sort.Limit > 0 { + spec.Limit = proto.Int32(int32(sort.Limit)) + params.ScorerSpec = spec + } + if sort.Scorer != nil { + sort.Scorer.toProto(spec) + params.ScorerSpec = spec + } + + return nil +} + +func refinementsToProto(refinements []Facet, params *pb.SearchParams) error { + for _, r := range refinements { + ref := &pb.FacetRefinement{ + Name: proto.String(r.Name), + } + switch v := r.Value.(type) { + case Atom: + ref.Value = proto.String(string(v)) + case Range: + rng, err := rangeToProto(v) + if err != nil { + return fmt.Errorf("search: refinement for facet %q: %v", r.Name, err) + } + // Unfortunately there are two identical messages for identify Facet ranges. + ref.Range = &pb.FacetRefinement_Range{Start: rng.Start, End: rng.End} + default: + return fmt.Errorf("search: unsupported refinement for facet %q of type %T", r.Name, v) + } + params.FacetRefinement = append(params.FacetRefinement, ref) + } + return nil +} + +func rangeToProto(r Range) (*pb.FacetRange, error) { + rng := &pb.FacetRange{} + if r.Start != negInf { + if !validFloat(r.Start) { + return nil, errors.New("invalid value for Start") + } + rng.Start = proto.String(strconv.FormatFloat(r.Start, 'e', -1, 64)) + } else if r.End == posInf { + return nil, errors.New("either Start or End must be finite") + } + if r.End != posInf { + if !validFloat(r.End) { + return nil, errors.New("invalid value for End") + } + rng.End = proto.String(strconv.FormatFloat(r.End, 'e', -1, 64)) + } + return rng, nil +} + +func protoToRange(rng *pb.FacetRefinement_Range) Range { + r := Range{Start: negInf, End: posInf} + if x, err := strconv.ParseFloat(rng.GetStart(), 64); err != nil { + r.Start = x + } + if x, err := strconv.ParseFloat(rng.GetEnd(), 64); err != nil { + r.End = x + } + return r +} + +// Iterator is the result of searching an index for a query or listing an +// index. +type Iterator struct { + c context.Context + index *Index + err error + + listRes []*pb.Document + listStartID string + listInclusive bool + + searchRes []*pb.SearchResult + facetRes []*pb.FacetResult + searchQuery string + searchCursor *string + searchOffset int + sort *SortOptions + + fields []string + exprs []FieldExpression + refinements []Facet + facetOpts []FacetSearchOption + + more func(*Iterator) error + + count int + countAccuracy int + limit int // items left to return; 0 for unlimited. + idsOnly bool +} + +// errIter returns an iterator that only returns the given error. +func errIter(err string) *Iterator { + return &Iterator{ + err: errors.New(err), + } +} + +// Done is returned when a query iteration has completed. +var Done = errors.New("search: query has no more results") + +// Count returns an approximation of the number of documents matched by the +// query. It is only valid to call for iterators returned by Search. +func (t *Iterator) Count() int { return t.count } + +// fetchMore retrieves more results, if there are no errors or pending results. +func (t *Iterator) fetchMore() { + if t.err == nil && len(t.listRes)+len(t.searchRes) == 0 && t.more != nil { + t.err = t.more(t) + } +} + +// Next returns the ID of the next result. When there are no more results, +// Done is returned as the error. +// +// dst must be a non-nil struct pointer, implement the FieldLoadSaver +// interface, or be a nil interface value. If a non-nil dst is provided, it +// will be filled with the indexed fields. dst is ignored if this iterator was +// created with an IDsOnly option. +func (t *Iterator) Next(dst interface{}) (string, error) { + t.fetchMore() + if t.err != nil { + return "", t.err + } + + var doc *pb.Document + var exprs []*pb.Field + switch { + case len(t.listRes) != 0: + doc = t.listRes[0] + t.listRes = t.listRes[1:] + case len(t.searchRes) != 0: + doc = t.searchRes[0].Document + exprs = t.searchRes[0].Expression + t.searchCursor = t.searchRes[0].Cursor + t.searchRes = t.searchRes[1:] + default: + return "", Done + } + if doc == nil { + return "", errors.New("search: internal error: no document returned") + } + if !t.idsOnly && dst != nil { + if err := loadDoc(dst, doc, exprs); err != nil { + return "", err + } + } + return doc.GetId(), nil +} + +// Cursor returns the cursor associated with the current document (that is, +// the document most recently returned by a call to Next). +// +// Passing this cursor in a future call to Search will cause those results +// to commence with the first document after the current document. +func (t *Iterator) Cursor() Cursor { + if t.searchCursor == nil { + return "" + } + return Cursor(*t.searchCursor) +} + +// Facets returns the facets found within the search results, if any facets +// were requested in the SearchOptions. +func (t *Iterator) Facets() ([][]FacetResult, error) { + t.fetchMore() + if t.err != nil && t.err != Done { + return nil, t.err + } + + var facets [][]FacetResult + for _, f := range t.facetRes { + fres := make([]FacetResult, 0, len(f.Value)) + for _, v := range f.Value { + ref := v.Refinement + facet := FacetResult{ + Facet: Facet{Name: ref.GetName()}, + Count: int(v.GetCount()), + } + if ref.Value != nil { + facet.Value = Atom(*ref.Value) + } else { + facet.Value = protoToRange(ref.Range) + } + fres = append(fres, facet) + } + facets = append(facets, fres) + } + return facets, nil +} + +// saveDoc converts from a struct pointer or +// FieldLoadSaver/FieldMetadataLoadSaver to the Document protobuf. +func saveDoc(src interface{}) (*pb.Document, error) { + var err error + var fields []Field + var meta *DocumentMetadata + switch x := src.(type) { + case FieldLoadSaver: + fields, meta, err = x.Save() + default: + fields, meta, err = saveStructWithMeta(src) + } + if err != nil { + return nil, err + } + + fieldsProto, err := fieldsToProto(fields) + if err != nil { + return nil, err + } + d := &pb.Document{ + Field: fieldsProto, + OrderId: proto.Int32(int32(time.Since(orderIDEpoch).Seconds())), + OrderIdSource: pb.Document_DEFAULTED.Enum(), + } + if meta != nil { + if meta.Rank != 0 { + if !validDocRank(meta.Rank) { + return nil, fmt.Errorf("search: invalid rank %d, must be [0, 2^31)", meta.Rank) + } + *d.OrderId = int32(meta.Rank) + d.OrderIdSource = pb.Document_SUPPLIED.Enum() + } + if len(meta.Facets) > 0 { + facets, err := facetsToProto(meta.Facets) + if err != nil { + return nil, err + } + d.Facet = facets + } + } + return d, nil +} + +func fieldsToProto(src []Field) ([]*pb.Field, error) { + // Maps to catch duplicate time or numeric fields. + timeFields, numericFields := make(map[string]bool), make(map[string]bool) + dst := make([]*pb.Field, 0, len(src)) + for _, f := range src { + if !validFieldName(f.Name) { + return nil, fmt.Errorf("search: invalid field name %q", f.Name) + } + fieldValue := &pb.FieldValue{} + switch x := f.Value.(type) { + case string: + fieldValue.Type = pb.FieldValue_TEXT.Enum() + fieldValue.StringValue = proto.String(x) + case Atom: + fieldValue.Type = pb.FieldValue_ATOM.Enum() + fieldValue.StringValue = proto.String(string(x)) + case HTML: + fieldValue.Type = pb.FieldValue_HTML.Enum() + fieldValue.StringValue = proto.String(string(x)) + case time.Time: + if timeFields[f.Name] { + return nil, fmt.Errorf("search: duplicate time field %q", f.Name) + } + timeFields[f.Name] = true + fieldValue.Type = pb.FieldValue_DATE.Enum() + fieldValue.StringValue = proto.String(strconv.FormatInt(x.UnixNano()/1e6, 10)) + case float64: + if numericFields[f.Name] { + return nil, fmt.Errorf("search: duplicate numeric field %q", f.Name) + } + if !validFloat(x) { + return nil, fmt.Errorf("search: numeric field %q with invalid value %f", f.Name, x) + } + numericFields[f.Name] = true + fieldValue.Type = pb.FieldValue_NUMBER.Enum() + fieldValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64)) + case appengine.GeoPoint: + if !x.Valid() { + return nil, fmt.Errorf( + "search: GeoPoint field %q with invalid value %v", + f.Name, x) + } + fieldValue.Type = pb.FieldValue_GEO.Enum() + fieldValue.Geo = &pb.FieldValue_Geo{ + Lat: proto.Float64(x.Lat), + Lng: proto.Float64(x.Lng), + } + default: + return nil, fmt.Errorf("search: unsupported field type: %v", reflect.TypeOf(f.Value)) + } + if f.Language != "" { + switch f.Value.(type) { + case string, HTML: + if !validLanguage(f.Language) { + return nil, fmt.Errorf("search: invalid language for field %q: %q", f.Name, f.Language) + } + fieldValue.Language = proto.String(f.Language) + default: + return nil, fmt.Errorf("search: setting language not supported for field %q of type %T", f.Name, f.Value) + } + } + if p := fieldValue.StringValue; p != nil && !utf8.ValidString(*p) { + return nil, fmt.Errorf("search: %q field is invalid UTF-8: %q", f.Name, *p) + } + dst = append(dst, &pb.Field{ + Name: proto.String(f.Name), + Value: fieldValue, + }) + } + return dst, nil +} + +func facetsToProto(src []Facet) ([]*pb.Facet, error) { + dst := make([]*pb.Facet, 0, len(src)) + for _, f := range src { + if !validFieldName(f.Name) { + return nil, fmt.Errorf("search: invalid facet name %q", f.Name) + } + facetValue := &pb.FacetValue{} + switch x := f.Value.(type) { + case Atom: + if !utf8.ValidString(string(x)) { + return nil, fmt.Errorf("search: %q facet is invalid UTF-8: %q", f.Name, x) + } + facetValue.Type = pb.FacetValue_ATOM.Enum() + facetValue.StringValue = proto.String(string(x)) + case float64: + if !validFloat(x) { + return nil, fmt.Errorf("search: numeric facet %q with invalid value %f", f.Name, x) + } + facetValue.Type = pb.FacetValue_NUMBER.Enum() + facetValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64)) + default: + return nil, fmt.Errorf("search: unsupported facet type: %v", reflect.TypeOf(f.Value)) + } + dst = append(dst, &pb.Facet{ + Name: proto.String(f.Name), + Value: facetValue, + }) + } + return dst, nil +} + +// loadDoc converts from protobufs to a struct pointer or +// FieldLoadSaver/FieldMetadataLoadSaver. The src param provides the document's +// stored fields and facets, and any document metadata. An additional slice of +// fields, exprs, may optionally be provided to contain any derived expressions +// requested by the developer. +func loadDoc(dst interface{}, src *pb.Document, exprs []*pb.Field) (err error) { + fields, err := protoToFields(src.Field) + if err != nil { + return err + } + facets, err := protoToFacets(src.Facet) + if err != nil { + return err + } + if len(exprs) > 0 { + exprFields, err := protoToFields(exprs) + if err != nil { + return err + } + // Mark each field as derived. + for i := range exprFields { + exprFields[i].Derived = true + } + fields = append(fields, exprFields...) + } + meta := &DocumentMetadata{ + Rank: int(src.GetOrderId()), + Facets: facets, + } + switch x := dst.(type) { + case FieldLoadSaver: + return x.Load(fields, meta) + default: + return loadStructWithMeta(dst, fields, meta) + } +} + +func protoToFields(fields []*pb.Field) ([]Field, error) { + dst := make([]Field, 0, len(fields)) + for _, field := range fields { + fieldValue := field.GetValue() + f := Field{ + Name: field.GetName(), + } + switch fieldValue.GetType() { + case pb.FieldValue_TEXT: + f.Value = fieldValue.GetStringValue() + f.Language = fieldValue.GetLanguage() + case pb.FieldValue_ATOM: + f.Value = Atom(fieldValue.GetStringValue()) + case pb.FieldValue_HTML: + f.Value = HTML(fieldValue.GetStringValue()) + f.Language = fieldValue.GetLanguage() + case pb.FieldValue_DATE: + sv := fieldValue.GetStringValue() + millis, err := strconv.ParseInt(sv, 10, 64) + if err != nil { + return nil, fmt.Errorf("search: internal error: bad time.Time encoding %q: %v", sv, err) + } + f.Value = time.Unix(0, millis*1e6) + case pb.FieldValue_NUMBER: + sv := fieldValue.GetStringValue() + x, err := strconv.ParseFloat(sv, 64) + if err != nil { + return nil, err + } + f.Value = x + case pb.FieldValue_GEO: + geoValue := fieldValue.GetGeo() + geoPoint := appengine.GeoPoint{geoValue.GetLat(), geoValue.GetLng()} + if !geoPoint.Valid() { + return nil, fmt.Errorf("search: internal error: invalid GeoPoint encoding: %v", geoPoint) + } + f.Value = geoPoint + default: + return nil, fmt.Errorf("search: internal error: unknown data type %s", fieldValue.GetType()) + } + dst = append(dst, f) + } + return dst, nil +} + +func protoToFacets(facets []*pb.Facet) ([]Facet, error) { + if len(facets) == 0 { + return nil, nil + } + dst := make([]Facet, 0, len(facets)) + for _, facet := range facets { + facetValue := facet.GetValue() + f := Facet{ + Name: facet.GetName(), + } + switch facetValue.GetType() { + case pb.FacetValue_ATOM: + f.Value = Atom(facetValue.GetStringValue()) + case pb.FacetValue_NUMBER: + sv := facetValue.GetStringValue() + x, err := strconv.ParseFloat(sv, 64) + if err != nil { + return nil, err + } + f.Value = x + default: + return nil, fmt.Errorf("search: internal error: unknown data type %s", facetValue.GetType()) + } + dst = append(dst, f) + } + return dst, nil +} + +func namespaceMod(m proto.Message, namespace string) { + set := func(s **string) { + if *s == nil { + *s = &namespace + } + } + switch m := m.(type) { + case *pb.IndexDocumentRequest: + set(&m.Params.IndexSpec.Namespace) + case *pb.ListDocumentsRequest: + set(&m.Params.IndexSpec.Namespace) + case *pb.DeleteDocumentRequest: + set(&m.Params.IndexSpec.Namespace) + case *pb.SearchRequest: + set(&m.Params.IndexSpec.Namespace) + } +} + +func init() { + internal.RegisterErrorCodeMap("search", pb.SearchServiceError_ErrorCode_name) + internal.NamespaceMods["search"] = namespaceMod +} -- cgit v1.2.3-1-g7c22