summaryrefslogtreecommitdiffstats
path: root/vendor/google.golang.org/appengine/search/search.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/google.golang.org/appengine/search/search.go')
-rw-r--r--vendor/google.golang.org/appengine/search/search.go1189
1 files changed, 1189 insertions, 0 deletions
diff --git a/vendor/google.golang.org/appengine/search/search.go b/vendor/google.golang.org/appengine/search/search.go
new file mode 100644
index 000000000..35a567d62
--- /dev/null
+++ b/vendor/google.golang.org/appengine/search/search.go
@@ -0,0 +1,1189 @@
+// Copyright 2012 Google Inc. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package search // import "google.golang.org/appengine/search"
+
+// TODO: let Put specify the document language: "en", "fr", etc. Also: order_id?? storage??
+// TODO: Index.GetAll (or Iterator.GetAll)?
+// TODO: struct <-> protobuf tests.
+// TODO: enforce Python's MIN_NUMBER_VALUE and MIN_DATE (which would disallow a zero
+// time.Time)? _MAXIMUM_STRING_LENGTH?
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "reflect"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/golang/protobuf/proto"
+ "golang.org/x/net/context"
+
+ "google.golang.org/appengine"
+ "google.golang.org/appengine/internal"
+ pb "google.golang.org/appengine/internal/search"
+)
+
+const maxDocumentsPerPutDelete = 200
+
+var (
+ // ErrInvalidDocumentType is returned when methods like Put, Get or Next
+ // are passed a dst or src argument of invalid type.
+ ErrInvalidDocumentType = errors.New("search: invalid document type")
+
+ // ErrNoSuchDocument is returned when no document was found for a given ID.
+ ErrNoSuchDocument = errors.New("search: no such document")
+
+ // ErrTooManyDocuments is returned when the user passes too many documents to
+ // PutMulti or DeleteMulti.
+ ErrTooManyDocuments = fmt.Errorf("search: too many documents given to put or delete (max is %d)", maxDocumentsPerPutDelete)
+)
+
+// Atom is a document field whose contents are indexed as a single indivisible
+// string.
+type Atom string
+
+// HTML is a document field whose contents are indexed as HTML. Only text nodes
+// are indexed: "foo<b>bar" will be treated as "foobar".
+type HTML string
+
+// validIndexNameOrDocID is the Go equivalent of Python's
+// _ValidateVisiblePrintableAsciiNotReserved.
+func validIndexNameOrDocID(s string) bool {
+ if strings.HasPrefix(s, "!") {
+ return false
+ }
+ for _, c := range s {
+ if c < 0x21 || 0x7f <= c {
+ return false
+ }
+ }
+ return true
+}
+
+var (
+ fieldNameRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`)
+ languageRE = regexp.MustCompile(`^[a-z]{2}$`)
+)
+
+// validFieldName is the Go equivalent of Python's _CheckFieldName. It checks
+// the validity of both field and facet names.
+func validFieldName(s string) bool {
+ return len(s) <= 500 && fieldNameRE.MatchString(s)
+}
+
+// validDocRank checks that the ranks is in the range [0, 2^31).
+func validDocRank(r int) bool {
+ return 0 <= r && r <= (1<<31-1)
+}
+
+// validLanguage checks that a language looks like ISO 639-1.
+func validLanguage(s string) bool {
+ return languageRE.MatchString(s)
+}
+
+// validFloat checks that f is in the range [-2147483647, 2147483647].
+func validFloat(f float64) bool {
+ return -(1<<31-1) <= f && f <= (1<<31-1)
+}
+
+// Index is an index of documents.
+type Index struct {
+ spec pb.IndexSpec
+}
+
+// orderIDEpoch forms the basis for populating OrderId on documents.
+var orderIDEpoch = time.Date(2011, 1, 1, 0, 0, 0, 0, time.UTC)
+
+// Open opens the index with the given name. The index is created if it does
+// not already exist.
+//
+// The name is a human-readable ASCII string. It must contain no whitespace
+// characters and not start with "!".
+func Open(name string) (*Index, error) {
+ if !validIndexNameOrDocID(name) {
+ return nil, fmt.Errorf("search: invalid index name %q", name)
+ }
+ return &Index{
+ spec: pb.IndexSpec{
+ Name: &name,
+ },
+ }, nil
+}
+
+// Put saves src to the index. If id is empty, a new ID is allocated by the
+// service and returned. If id is not empty, any existing index entry for that
+// ID is replaced.
+//
+// The ID is a human-readable ASCII string. It must contain no whitespace
+// characters and not start with "!".
+//
+// src must be a non-nil struct pointer or implement the FieldLoadSaver
+// interface.
+func (x *Index) Put(c context.Context, id string, src interface{}) (string, error) {
+ ids, err := x.PutMulti(c, []string{id}, []interface{}{src})
+ if err != nil {
+ return "", err
+ }
+ return ids[0], nil
+}
+
+// PutMulti is like Put, but is more efficient for adding multiple documents to
+// the index at once.
+//
+// Up to 200 documents can be added at once. ErrTooManyDocuments is returned if
+// you try to add more.
+//
+// ids can either be an empty slice (which means new IDs will be allocated for
+// each of the documents added) or a slice the same size as srcs.
+//
+// The error may be an instance of appengine.MultiError, in which case it will
+// be the same size as srcs and the individual errors inside will correspond
+// with the items in srcs.
+func (x *Index) PutMulti(c context.Context, ids []string, srcs []interface{}) ([]string, error) {
+ if len(ids) != 0 && len(srcs) != len(ids) {
+ return nil, fmt.Errorf("search: PutMulti expects ids and srcs slices of the same length")
+ }
+ if len(srcs) > maxDocumentsPerPutDelete {
+ return nil, ErrTooManyDocuments
+ }
+
+ docs := make([]*pb.Document, len(srcs))
+ for i, s := range srcs {
+ var err error
+ docs[i], err = saveDoc(s)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(ids) != 0 && ids[i] != "" {
+ if !validIndexNameOrDocID(ids[i]) {
+ return nil, fmt.Errorf("search: invalid ID %q", ids[i])
+ }
+ docs[i].Id = proto.String(ids[i])
+ }
+ }
+
+ // spec is modified by Call when applying the current Namespace, so copy it to
+ // avoid retaining the namespace beyond the scope of the Call.
+ spec := x.spec
+ req := &pb.IndexDocumentRequest{
+ Params: &pb.IndexDocumentParams{
+ Document: docs,
+ IndexSpec: &spec,
+ },
+ }
+ res := &pb.IndexDocumentResponse{}
+ if err := internal.Call(c, "search", "IndexDocument", req, res); err != nil {
+ return nil, err
+ }
+ multiErr, hasErr := make(appengine.MultiError, len(res.Status)), false
+ for i, s := range res.Status {
+ if s.GetCode() != pb.SearchServiceError_OK {
+ multiErr[i] = fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail())
+ hasErr = true
+ }
+ }
+ if hasErr {
+ return res.DocId, multiErr
+ }
+
+ if len(res.Status) != len(docs) || len(res.DocId) != len(docs) {
+ return nil, fmt.Errorf("search: internal error: wrong number of results (%d Statuses, %d DocIDs, expected %d)",
+ len(res.Status), len(res.DocId), len(docs))
+ }
+ return res.DocId, nil
+}
+
+// Get loads the document with the given ID into dst.
+//
+// The ID is a human-readable ASCII string. It must be non-empty, contain no
+// whitespace characters and not start with "!".
+//
+// dst must be a non-nil struct pointer or implement the FieldLoadSaver
+// interface.
+//
+// ErrFieldMismatch is returned when a field is to be loaded into a different
+// type than the one it was stored from, or when a field is missing or
+// unexported in the destination struct. ErrFieldMismatch is only returned if
+// dst is a struct pointer. It is up to the callee to decide whether this error
+// is fatal, recoverable, or ignorable.
+func (x *Index) Get(c context.Context, id string, dst interface{}) error {
+ if id == "" || !validIndexNameOrDocID(id) {
+ return fmt.Errorf("search: invalid ID %q", id)
+ }
+ req := &pb.ListDocumentsRequest{
+ Params: &pb.ListDocumentsParams{
+ IndexSpec: &x.spec,
+ StartDocId: proto.String(id),
+ Limit: proto.Int32(1),
+ },
+ }
+ res := &pb.ListDocumentsResponse{}
+ if err := internal.Call(c, "search", "ListDocuments", req, res); err != nil {
+ return err
+ }
+ if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
+ return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
+ }
+ if len(res.Document) != 1 || res.Document[0].GetId() != id {
+ return ErrNoSuchDocument
+ }
+ return loadDoc(dst, res.Document[0], nil)
+}
+
+// Delete deletes a document from the index.
+func (x *Index) Delete(c context.Context, id string) error {
+ return x.DeleteMulti(c, []string{id})
+}
+
+// DeleteMulti deletes multiple documents from the index.
+//
+// The returned error may be an instance of appengine.MultiError, in which case
+// it will be the same size as srcs and the individual errors inside will
+// correspond with the items in srcs.
+func (x *Index) DeleteMulti(c context.Context, ids []string) error {
+ if len(ids) > maxDocumentsPerPutDelete {
+ return ErrTooManyDocuments
+ }
+
+ req := &pb.DeleteDocumentRequest{
+ Params: &pb.DeleteDocumentParams{
+ DocId: ids,
+ IndexSpec: &x.spec,
+ },
+ }
+ res := &pb.DeleteDocumentResponse{}
+ if err := internal.Call(c, "search", "DeleteDocument", req, res); err != nil {
+ return err
+ }
+ if len(res.Status) != len(ids) {
+ return fmt.Errorf("search: internal error: wrong number of results (%d, expected %d)",
+ len(res.Status), len(ids))
+ }
+ multiErr, hasErr := make(appengine.MultiError, len(ids)), false
+ for i, s := range res.Status {
+ if s.GetCode() != pb.SearchServiceError_OK {
+ multiErr[i] = fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail())
+ hasErr = true
+ }
+ }
+ if hasErr {
+ return multiErr
+ }
+ return nil
+}
+
+// List lists all of the documents in an index. The documents are returned in
+// increasing ID order.
+func (x *Index) List(c context.Context, opts *ListOptions) *Iterator {
+ t := &Iterator{
+ c: c,
+ index: x,
+ count: -1,
+ listInclusive: true,
+ more: moreList,
+ }
+ if opts != nil {
+ t.listStartID = opts.StartID
+ t.limit = opts.Limit
+ t.idsOnly = opts.IDsOnly
+ }
+ return t
+}
+
+func moreList(t *Iterator) error {
+ req := &pb.ListDocumentsRequest{
+ Params: &pb.ListDocumentsParams{
+ IndexSpec: &t.index.spec,
+ },
+ }
+ if t.listStartID != "" {
+ req.Params.StartDocId = &t.listStartID
+ req.Params.IncludeStartDoc = &t.listInclusive
+ }
+ if t.limit > 0 {
+ req.Params.Limit = proto.Int32(int32(t.limit))
+ }
+ if t.idsOnly {
+ req.Params.KeysOnly = &t.idsOnly
+ }
+
+ res := &pb.ListDocumentsResponse{}
+ if err := internal.Call(t.c, "search", "ListDocuments", req, res); err != nil {
+ return err
+ }
+ if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
+ return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
+ }
+ t.listRes = res.Document
+ t.listStartID, t.listInclusive, t.more = "", false, nil
+ if len(res.Document) != 0 && t.limit <= 0 {
+ if id := res.Document[len(res.Document)-1].GetId(); id != "" {
+ t.listStartID, t.more = id, moreList
+ }
+ }
+ return nil
+}
+
+// ListOptions are the options for listing documents in an index. Passing a nil
+// *ListOptions is equivalent to using the default values.
+type ListOptions struct {
+ // StartID is the inclusive lower bound for the ID of the returned
+ // documents. The zero value means all documents will be returned.
+ StartID string
+
+ // Limit is the maximum number of documents to return. The zero value
+ // indicates no limit.
+ Limit int
+
+ // IDsOnly indicates that only document IDs should be returned for the list
+ // operation; no document fields are populated.
+ IDsOnly bool
+}
+
+// Search searches the index for the given query.
+func (x *Index) Search(c context.Context, query string, opts *SearchOptions) *Iterator {
+ t := &Iterator{
+ c: c,
+ index: x,
+ searchQuery: query,
+ more: moreSearch,
+ }
+ if opts != nil {
+ if opts.Cursor != "" {
+ if opts.Offset != 0 {
+ return errIter("at most one of Cursor and Offset may be specified")
+ }
+ t.searchCursor = proto.String(string(opts.Cursor))
+ }
+ t.limit = opts.Limit
+ t.fields = opts.Fields
+ t.idsOnly = opts.IDsOnly
+ t.sort = opts.Sort
+ t.exprs = opts.Expressions
+ t.refinements = opts.Refinements
+ t.facetOpts = opts.Facets
+ t.searchOffset = opts.Offset
+ t.countAccuracy = opts.CountAccuracy
+ }
+ return t
+}
+
+func moreSearch(t *Iterator) error {
+ // We use per-result (rather than single/per-page) cursors since this
+ // lets us return a Cursor for every iterator document. The two cursor
+ // types are largely interchangeable: a page cursor is the same as the
+ // last per-result cursor in a given search response.
+ req := &pb.SearchRequest{
+ Params: &pb.SearchParams{
+ IndexSpec: &t.index.spec,
+ Query: &t.searchQuery,
+ Cursor: t.searchCursor,
+ CursorType: pb.SearchParams_PER_RESULT.Enum(),
+ FieldSpec: &pb.FieldSpec{
+ Name: t.fields,
+ },
+ },
+ }
+ if t.limit > 0 {
+ req.Params.Limit = proto.Int32(int32(t.limit))
+ }
+ if t.searchOffset > 0 {
+ req.Params.Offset = proto.Int32(int32(t.searchOffset))
+ t.searchOffset = 0
+ }
+ if t.countAccuracy > 0 {
+ req.Params.MatchedCountAccuracy = proto.Int32(int32(t.countAccuracy))
+ }
+ if t.idsOnly {
+ req.Params.KeysOnly = &t.idsOnly
+ }
+ if t.sort != nil {
+ if err := sortToProto(t.sort, req.Params); err != nil {
+ return err
+ }
+ }
+ if t.refinements != nil {
+ if err := refinementsToProto(t.refinements, req.Params); err != nil {
+ return err
+ }
+ }
+ for _, e := range t.exprs {
+ req.Params.FieldSpec.Expression = append(req.Params.FieldSpec.Expression, &pb.FieldSpec_Expression{
+ Name: proto.String(e.Name),
+ Expression: proto.String(e.Expr),
+ })
+ }
+ for _, f := range t.facetOpts {
+ if err := f.setParams(req.Params); err != nil {
+ return fmt.Errorf("bad FacetSearchOption: %v", err)
+ }
+ }
+ // Don't repeat facet search.
+ t.facetOpts = nil
+
+ res := &pb.SearchResponse{}
+ if err := internal.Call(t.c, "search", "Search", req, res); err != nil {
+ return err
+ }
+ if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
+ return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
+ }
+ t.searchRes = res.Result
+ if len(res.FacetResult) > 0 {
+ t.facetRes = res.FacetResult
+ }
+ t.count = int(*res.MatchedCount)
+ if t.limit > 0 {
+ t.more = nil
+ } else {
+ t.more = moreSearch
+ }
+ return nil
+}
+
+// SearchOptions are the options for searching an index. Passing a nil
+// *SearchOptions is equivalent to using the default values.
+type SearchOptions struct {
+ // Limit is the maximum number of documents to return. The zero value
+ // indicates no limit.
+ Limit int
+
+ // IDsOnly indicates that only document IDs should be returned for the search
+ // operation; no document fields are populated.
+ IDsOnly bool
+
+ // Sort controls the ordering of search results.
+ Sort *SortOptions
+
+ // Fields specifies which document fields to include in the results. If omitted,
+ // all document fields are returned. No more than 100 fields may be specified.
+ Fields []string
+
+ // Expressions specifies additional computed fields to add to each returned
+ // document.
+ Expressions []FieldExpression
+
+ // Facets controls what facet information is returned for these search results.
+ // If no options are specified, no facet results will be returned.
+ Facets []FacetSearchOption
+
+ // Refinements filters the returned documents by requiring them to contain facets
+ // with specific values. Refinements are applied in conjunction for facets with
+ // different names, and in disjunction otherwise.
+ Refinements []Facet
+
+ // Cursor causes the results to commence with the first document after
+ // the document associated with the cursor.
+ Cursor Cursor
+
+ // Offset specifies the number of documents to skip over before returning results.
+ // When specified, Cursor must be nil.
+ Offset int
+
+ // CountAccuracy specifies the maximum result count that can be expected to
+ // be accurate. If zero, the count accuracy defaults to 20.
+ CountAccuracy int
+}
+
+// Cursor represents an iterator's position.
+//
+// The string value of a cursor is web-safe. It can be saved and restored
+// for later use.
+type Cursor string
+
+// FieldExpression defines a custom expression to evaluate for each result.
+type FieldExpression struct {
+ // Name is the name to use for the computed field.
+ Name string
+
+ // Expr is evaluated to provide a custom content snippet for each document.
+ // See https://cloud.google.com/appengine/docs/standard/go/search/options for
+ // the supported expression syntax.
+ Expr string
+}
+
+// FacetSearchOption controls what facet information is returned in search results.
+type FacetSearchOption interface {
+ setParams(*pb.SearchParams) error
+}
+
+// AutoFacetDiscovery returns a FacetSearchOption which enables automatic facet
+// discovery for the search. Automatic facet discovery looks for the facets
+// which appear the most often in the aggregate in the matched documents.
+//
+// The maximum number of facets returned is controlled by facetLimit, and the
+// maximum number of values per facet by facetLimit. A limit of zero indicates
+// a default limit should be used.
+func AutoFacetDiscovery(facetLimit, valueLimit int) FacetSearchOption {
+ return &autoFacetOpt{facetLimit, valueLimit}
+}
+
+type autoFacetOpt struct {
+ facetLimit, valueLimit int
+}
+
+const defaultAutoFacetLimit = 10 // As per python runtime search.py.
+
+func (o *autoFacetOpt) setParams(params *pb.SearchParams) error {
+ lim := int32(o.facetLimit)
+ if lim == 0 {
+ lim = defaultAutoFacetLimit
+ }
+ params.AutoDiscoverFacetCount = &lim
+ if o.valueLimit > 0 {
+ params.FacetAutoDetectParam = &pb.FacetAutoDetectParam{
+ ValueLimit: proto.Int32(int32(o.valueLimit)),
+ }
+ }
+ return nil
+}
+
+// FacetDiscovery returns a FacetSearchOption which selects a facet to be
+// returned with the search results. By default, the most frequently
+// occurring values for that facet will be returned. However, you can also
+// specify a list of particular Atoms or specific Ranges to return.
+func FacetDiscovery(name string, value ...interface{}) FacetSearchOption {
+ return &facetOpt{name, value}
+}
+
+type facetOpt struct {
+ name string
+ values []interface{}
+}
+
+func (o *facetOpt) setParams(params *pb.SearchParams) error {
+ req := &pb.FacetRequest{Name: &o.name}
+ params.IncludeFacet = append(params.IncludeFacet, req)
+ if len(o.values) == 0 {
+ return nil
+ }
+ vtype := reflect.TypeOf(o.values[0])
+ reqParam := &pb.FacetRequestParam{}
+ for _, v := range o.values {
+ if reflect.TypeOf(v) != vtype {
+ return errors.New("values must all be Atom, or must all be Range")
+ }
+ switch v := v.(type) {
+ case Atom:
+ reqParam.ValueConstraint = append(reqParam.ValueConstraint, string(v))
+ case Range:
+ rng, err := rangeToProto(v)
+ if err != nil {
+ return fmt.Errorf("invalid range: %v", err)
+ }
+ reqParam.Range = append(reqParam.Range, rng)
+ default:
+ return fmt.Errorf("unsupported value type %T", v)
+ }
+ }
+ req.Params = reqParam
+ return nil
+}
+
+// FacetDocumentDepth returns a FacetSearchOption which controls the number of
+// documents to be evaluated with preparing facet results.
+func FacetDocumentDepth(depth int) FacetSearchOption {
+ return facetDepthOpt(depth)
+}
+
+type facetDepthOpt int
+
+func (o facetDepthOpt) setParams(params *pb.SearchParams) error {
+ params.FacetDepth = proto.Int32(int32(o))
+ return nil
+}
+
+// FacetResult represents the number of times a particular facet and value
+// appeared in the documents matching a search request.
+type FacetResult struct {
+ Facet
+
+ // Count is the number of times this specific facet and value appeared in the
+ // matching documents.
+ Count int
+}
+
+// Range represents a numeric range with inclusive start and exclusive end.
+// Start may be specified as math.Inf(-1) to indicate there is no minimum
+// value, and End may similarly be specified as math.Inf(1); at least one of
+// Start or End must be a finite number.
+type Range struct {
+ Start, End float64
+}
+
+var (
+ negInf = math.Inf(-1)
+ posInf = math.Inf(1)
+)
+
+// AtLeast returns a Range matching any value greater than, or equal to, min.
+func AtLeast(min float64) Range {
+ return Range{Start: min, End: posInf}
+}
+
+// LessThan returns a Range matching any value less than max.
+func LessThan(max float64) Range {
+ return Range{Start: negInf, End: max}
+}
+
+// SortOptions control the ordering and scoring of search results.
+type SortOptions struct {
+ // Expressions is a slice of expressions representing a multi-dimensional
+ // sort.
+ Expressions []SortExpression
+
+ // Scorer, when specified, will cause the documents to be scored according to
+ // search term frequency.
+ Scorer Scorer
+
+ // Limit is the maximum number of objects to score and/or sort. Limit cannot
+ // be more than 10,000. The zero value indicates a default limit.
+ Limit int
+}
+
+// SortExpression defines a single dimension for sorting a document.
+type SortExpression struct {
+ // Expr is evaluated to provide a sorting value for each document.
+ // See https://cloud.google.com/appengine/docs/standard/go/search/options for
+ // the supported expression syntax.
+ Expr string
+
+ // Reverse causes the documents to be sorted in ascending order.
+ Reverse bool
+
+ // The default value to use when no field is present or the expresion
+ // cannot be calculated for a document. For text sorts, Default must
+ // be of type string; for numeric sorts, float64.
+ Default interface{}
+}
+
+// A Scorer defines how a document is scored.
+type Scorer interface {
+ toProto(*pb.ScorerSpec)
+}
+
+type enumScorer struct {
+ enum pb.ScorerSpec_Scorer
+}
+
+func (e enumScorer) toProto(spec *pb.ScorerSpec) {
+ spec.Scorer = e.enum.Enum()
+}
+
+var (
+ // MatchScorer assigns a score based on term frequency in a document.
+ MatchScorer Scorer = enumScorer{pb.ScorerSpec_MATCH_SCORER}
+
+ // RescoringMatchScorer assigns a score based on the quality of the query
+ // match. It is similar to a MatchScorer but uses a more complex scoring
+ // algorithm based on match term frequency and other factors like field type.
+ // Please be aware that this algorithm is continually refined and can change
+ // over time without notice. This means that the ordering of search results
+ // that use this scorer can also change without notice.
+ RescoringMatchScorer Scorer = enumScorer{pb.ScorerSpec_RESCORING_MATCH_SCORER}
+)
+
+func sortToProto(sort *SortOptions, params *pb.SearchParams) error {
+ for _, e := range sort.Expressions {
+ spec := &pb.SortSpec{
+ SortExpression: proto.String(e.Expr),
+ }
+ if e.Reverse {
+ spec.SortDescending = proto.Bool(false)
+ }
+ if e.Default != nil {
+ switch d := e.Default.(type) {
+ case float64:
+ spec.DefaultValueNumeric = &d
+ case string:
+ spec.DefaultValueText = &d
+ default:
+ return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr)
+ }
+ }
+ params.SortSpec = append(params.SortSpec, spec)
+ }
+
+ spec := &pb.ScorerSpec{}
+ if sort.Limit > 0 {
+ spec.Limit = proto.Int32(int32(sort.Limit))
+ params.ScorerSpec = spec
+ }
+ if sort.Scorer != nil {
+ sort.Scorer.toProto(spec)
+ params.ScorerSpec = spec
+ }
+
+ return nil
+}
+
+func refinementsToProto(refinements []Facet, params *pb.SearchParams) error {
+ for _, r := range refinements {
+ ref := &pb.FacetRefinement{
+ Name: proto.String(r.Name),
+ }
+ switch v := r.Value.(type) {
+ case Atom:
+ ref.Value = proto.String(string(v))
+ case Range:
+ rng, err := rangeToProto(v)
+ if err != nil {
+ return fmt.Errorf("search: refinement for facet %q: %v", r.Name, err)
+ }
+ // Unfortunately there are two identical messages for identify Facet ranges.
+ ref.Range = &pb.FacetRefinement_Range{Start: rng.Start, End: rng.End}
+ default:
+ return fmt.Errorf("search: unsupported refinement for facet %q of type %T", r.Name, v)
+ }
+ params.FacetRefinement = append(params.FacetRefinement, ref)
+ }
+ return nil
+}
+
+func rangeToProto(r Range) (*pb.FacetRange, error) {
+ rng := &pb.FacetRange{}
+ if r.Start != negInf {
+ if !validFloat(r.Start) {
+ return nil, errors.New("invalid value for Start")
+ }
+ rng.Start = proto.String(strconv.FormatFloat(r.Start, 'e', -1, 64))
+ } else if r.End == posInf {
+ return nil, errors.New("either Start or End must be finite")
+ }
+ if r.End != posInf {
+ if !validFloat(r.End) {
+ return nil, errors.New("invalid value for End")
+ }
+ rng.End = proto.String(strconv.FormatFloat(r.End, 'e', -1, 64))
+ }
+ return rng, nil
+}
+
+func protoToRange(rng *pb.FacetRefinement_Range) Range {
+ r := Range{Start: negInf, End: posInf}
+ if x, err := strconv.ParseFloat(rng.GetStart(), 64); err != nil {
+ r.Start = x
+ }
+ if x, err := strconv.ParseFloat(rng.GetEnd(), 64); err != nil {
+ r.End = x
+ }
+ return r
+}
+
+// Iterator is the result of searching an index for a query or listing an
+// index.
+type Iterator struct {
+ c context.Context
+ index *Index
+ err error
+
+ listRes []*pb.Document
+ listStartID string
+ listInclusive bool
+
+ searchRes []*pb.SearchResult
+ facetRes []*pb.FacetResult
+ searchQuery string
+ searchCursor *string
+ searchOffset int
+ sort *SortOptions
+
+ fields []string
+ exprs []FieldExpression
+ refinements []Facet
+ facetOpts []FacetSearchOption
+
+ more func(*Iterator) error
+
+ count int
+ countAccuracy int
+ limit int // items left to return; 0 for unlimited.
+ idsOnly bool
+}
+
+// errIter returns an iterator that only returns the given error.
+func errIter(err string) *Iterator {
+ return &Iterator{
+ err: errors.New(err),
+ }
+}
+
+// Done is returned when a query iteration has completed.
+var Done = errors.New("search: query has no more results")
+
+// Count returns an approximation of the number of documents matched by the
+// query. It is only valid to call for iterators returned by Search.
+func (t *Iterator) Count() int { return t.count }
+
+// fetchMore retrieves more results, if there are no errors or pending results.
+func (t *Iterator) fetchMore() {
+ if t.err == nil && len(t.listRes)+len(t.searchRes) == 0 && t.more != nil {
+ t.err = t.more(t)
+ }
+}
+
+// Next returns the ID of the next result. When there are no more results,
+// Done is returned as the error.
+//
+// dst must be a non-nil struct pointer, implement the FieldLoadSaver
+// interface, or be a nil interface value. If a non-nil dst is provided, it
+// will be filled with the indexed fields. dst is ignored if this iterator was
+// created with an IDsOnly option.
+func (t *Iterator) Next(dst interface{}) (string, error) {
+ t.fetchMore()
+ if t.err != nil {
+ return "", t.err
+ }
+
+ var doc *pb.Document
+ var exprs []*pb.Field
+ switch {
+ case len(t.listRes) != 0:
+ doc = t.listRes[0]
+ t.listRes = t.listRes[1:]
+ case len(t.searchRes) != 0:
+ doc = t.searchRes[0].Document
+ exprs = t.searchRes[0].Expression
+ t.searchCursor = t.searchRes[0].Cursor
+ t.searchRes = t.searchRes[1:]
+ default:
+ return "", Done
+ }
+ if doc == nil {
+ return "", errors.New("search: internal error: no document returned")
+ }
+ if !t.idsOnly && dst != nil {
+ if err := loadDoc(dst, doc, exprs); err != nil {
+ return "", err
+ }
+ }
+ return doc.GetId(), nil
+}
+
+// Cursor returns the cursor associated with the current document (that is,
+// the document most recently returned by a call to Next).
+//
+// Passing this cursor in a future call to Search will cause those results
+// to commence with the first document after the current document.
+func (t *Iterator) Cursor() Cursor {
+ if t.searchCursor == nil {
+ return ""
+ }
+ return Cursor(*t.searchCursor)
+}
+
+// Facets returns the facets found within the search results, if any facets
+// were requested in the SearchOptions.
+func (t *Iterator) Facets() ([][]FacetResult, error) {
+ t.fetchMore()
+ if t.err != nil && t.err != Done {
+ return nil, t.err
+ }
+
+ var facets [][]FacetResult
+ for _, f := range t.facetRes {
+ fres := make([]FacetResult, 0, len(f.Value))
+ for _, v := range f.Value {
+ ref := v.Refinement
+ facet := FacetResult{
+ Facet: Facet{Name: ref.GetName()},
+ Count: int(v.GetCount()),
+ }
+ if ref.Value != nil {
+ facet.Value = Atom(*ref.Value)
+ } else {
+ facet.Value = protoToRange(ref.Range)
+ }
+ fres = append(fres, facet)
+ }
+ facets = append(facets, fres)
+ }
+ return facets, nil
+}
+
+// saveDoc converts from a struct pointer or
+// FieldLoadSaver/FieldMetadataLoadSaver to the Document protobuf.
+func saveDoc(src interface{}) (*pb.Document, error) {
+ var err error
+ var fields []Field
+ var meta *DocumentMetadata
+ switch x := src.(type) {
+ case FieldLoadSaver:
+ fields, meta, err = x.Save()
+ default:
+ fields, meta, err = saveStructWithMeta(src)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ fieldsProto, err := fieldsToProto(fields)
+ if err != nil {
+ return nil, err
+ }
+ d := &pb.Document{
+ Field: fieldsProto,
+ OrderId: proto.Int32(int32(time.Since(orderIDEpoch).Seconds())),
+ OrderIdSource: pb.Document_DEFAULTED.Enum(),
+ }
+ if meta != nil {
+ if meta.Rank != 0 {
+ if !validDocRank(meta.Rank) {
+ return nil, fmt.Errorf("search: invalid rank %d, must be [0, 2^31)", meta.Rank)
+ }
+ *d.OrderId = int32(meta.Rank)
+ d.OrderIdSource = pb.Document_SUPPLIED.Enum()
+ }
+ if len(meta.Facets) > 0 {
+ facets, err := facetsToProto(meta.Facets)
+ if err != nil {
+ return nil, err
+ }
+ d.Facet = facets
+ }
+ }
+ return d, nil
+}
+
+func fieldsToProto(src []Field) ([]*pb.Field, error) {
+ // Maps to catch duplicate time or numeric fields.
+ timeFields, numericFields := make(map[string]bool), make(map[string]bool)
+ dst := make([]*pb.Field, 0, len(src))
+ for _, f := range src {
+ if !validFieldName(f.Name) {
+ return nil, fmt.Errorf("search: invalid field name %q", f.Name)
+ }
+ fieldValue := &pb.FieldValue{}
+ switch x := f.Value.(type) {
+ case string:
+ fieldValue.Type = pb.FieldValue_TEXT.Enum()
+ fieldValue.StringValue = proto.String(x)
+ case Atom:
+ fieldValue.Type = pb.FieldValue_ATOM.Enum()
+ fieldValue.StringValue = proto.String(string(x))
+ case HTML:
+ fieldValue.Type = pb.FieldValue_HTML.Enum()
+ fieldValue.StringValue = proto.String(string(x))
+ case time.Time:
+ if timeFields[f.Name] {
+ return nil, fmt.Errorf("search: duplicate time field %q", f.Name)
+ }
+ timeFields[f.Name] = true
+ fieldValue.Type = pb.FieldValue_DATE.Enum()
+ fieldValue.StringValue = proto.String(strconv.FormatInt(x.UnixNano()/1e6, 10))
+ case float64:
+ if numericFields[f.Name] {
+ return nil, fmt.Errorf("search: duplicate numeric field %q", f.Name)
+ }
+ if !validFloat(x) {
+ return nil, fmt.Errorf("search: numeric field %q with invalid value %f", f.Name, x)
+ }
+ numericFields[f.Name] = true
+ fieldValue.Type = pb.FieldValue_NUMBER.Enum()
+ fieldValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
+ case appengine.GeoPoint:
+ if !x.Valid() {
+ return nil, fmt.Errorf(
+ "search: GeoPoint field %q with invalid value %v",
+ f.Name, x)
+ }
+ fieldValue.Type = pb.FieldValue_GEO.Enum()
+ fieldValue.Geo = &pb.FieldValue_Geo{
+ Lat: proto.Float64(x.Lat),
+ Lng: proto.Float64(x.Lng),
+ }
+ default:
+ return nil, fmt.Errorf("search: unsupported field type: %v", reflect.TypeOf(f.Value))
+ }
+ if f.Language != "" {
+ switch f.Value.(type) {
+ case string, HTML:
+ if !validLanguage(f.Language) {
+ return nil, fmt.Errorf("search: invalid language for field %q: %q", f.Name, f.Language)
+ }
+ fieldValue.Language = proto.String(f.Language)
+ default:
+ return nil, fmt.Errorf("search: setting language not supported for field %q of type %T", f.Name, f.Value)
+ }
+ }
+ if p := fieldValue.StringValue; p != nil && !utf8.ValidString(*p) {
+ return nil, fmt.Errorf("search: %q field is invalid UTF-8: %q", f.Name, *p)
+ }
+ dst = append(dst, &pb.Field{
+ Name: proto.String(f.Name),
+ Value: fieldValue,
+ })
+ }
+ return dst, nil
+}
+
+func facetsToProto(src []Facet) ([]*pb.Facet, error) {
+ dst := make([]*pb.Facet, 0, len(src))
+ for _, f := range src {
+ if !validFieldName(f.Name) {
+ return nil, fmt.Errorf("search: invalid facet name %q", f.Name)
+ }
+ facetValue := &pb.FacetValue{}
+ switch x := f.Value.(type) {
+ case Atom:
+ if !utf8.ValidString(string(x)) {
+ return nil, fmt.Errorf("search: %q facet is invalid UTF-8: %q", f.Name, x)
+ }
+ facetValue.Type = pb.FacetValue_ATOM.Enum()
+ facetValue.StringValue = proto.String(string(x))
+ case float64:
+ if !validFloat(x) {
+ return nil, fmt.Errorf("search: numeric facet %q with invalid value %f", f.Name, x)
+ }
+ facetValue.Type = pb.FacetValue_NUMBER.Enum()
+ facetValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
+ default:
+ return nil, fmt.Errorf("search: unsupported facet type: %v", reflect.TypeOf(f.Value))
+ }
+ dst = append(dst, &pb.Facet{
+ Name: proto.String(f.Name),
+ Value: facetValue,
+ })
+ }
+ return dst, nil
+}
+
+// loadDoc converts from protobufs to a struct pointer or
+// FieldLoadSaver/FieldMetadataLoadSaver. The src param provides the document's
+// stored fields and facets, and any document metadata. An additional slice of
+// fields, exprs, may optionally be provided to contain any derived expressions
+// requested by the developer.
+func loadDoc(dst interface{}, src *pb.Document, exprs []*pb.Field) (err error) {
+ fields, err := protoToFields(src.Field)
+ if err != nil {
+ return err
+ }
+ facets, err := protoToFacets(src.Facet)
+ if err != nil {
+ return err
+ }
+ if len(exprs) > 0 {
+ exprFields, err := protoToFields(exprs)
+ if err != nil {
+ return err
+ }
+ // Mark each field as derived.
+ for i := range exprFields {
+ exprFields[i].Derived = true
+ }
+ fields = append(fields, exprFields...)
+ }
+ meta := &DocumentMetadata{
+ Rank: int(src.GetOrderId()),
+ Facets: facets,
+ }
+ switch x := dst.(type) {
+ case FieldLoadSaver:
+ return x.Load(fields, meta)
+ default:
+ return loadStructWithMeta(dst, fields, meta)
+ }
+}
+
+func protoToFields(fields []*pb.Field) ([]Field, error) {
+ dst := make([]Field, 0, len(fields))
+ for _, field := range fields {
+ fieldValue := field.GetValue()
+ f := Field{
+ Name: field.GetName(),
+ }
+ switch fieldValue.GetType() {
+ case pb.FieldValue_TEXT:
+ f.Value = fieldValue.GetStringValue()
+ f.Language = fieldValue.GetLanguage()
+ case pb.FieldValue_ATOM:
+ f.Value = Atom(fieldValue.GetStringValue())
+ case pb.FieldValue_HTML:
+ f.Value = HTML(fieldValue.GetStringValue())
+ f.Language = fieldValue.GetLanguage()
+ case pb.FieldValue_DATE:
+ sv := fieldValue.GetStringValue()
+ millis, err := strconv.ParseInt(sv, 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("search: internal error: bad time.Time encoding %q: %v", sv, err)
+ }
+ f.Value = time.Unix(0, millis*1e6)
+ case pb.FieldValue_NUMBER:
+ sv := fieldValue.GetStringValue()
+ x, err := strconv.ParseFloat(sv, 64)
+ if err != nil {
+ return nil, err
+ }
+ f.Value = x
+ case pb.FieldValue_GEO:
+ geoValue := fieldValue.GetGeo()
+ geoPoint := appengine.GeoPoint{geoValue.GetLat(), geoValue.GetLng()}
+ if !geoPoint.Valid() {
+ return nil, fmt.Errorf("search: internal error: invalid GeoPoint encoding: %v", geoPoint)
+ }
+ f.Value = geoPoint
+ default:
+ return nil, fmt.Errorf("search: internal error: unknown data type %s", fieldValue.GetType())
+ }
+ dst = append(dst, f)
+ }
+ return dst, nil
+}
+
+func protoToFacets(facets []*pb.Facet) ([]Facet, error) {
+ if len(facets) == 0 {
+ return nil, nil
+ }
+ dst := make([]Facet, 0, len(facets))
+ for _, facet := range facets {
+ facetValue := facet.GetValue()
+ f := Facet{
+ Name: facet.GetName(),
+ }
+ switch facetValue.GetType() {
+ case pb.FacetValue_ATOM:
+ f.Value = Atom(facetValue.GetStringValue())
+ case pb.FacetValue_NUMBER:
+ sv := facetValue.GetStringValue()
+ x, err := strconv.ParseFloat(sv, 64)
+ if err != nil {
+ return nil, err
+ }
+ f.Value = x
+ default:
+ return nil, fmt.Errorf("search: internal error: unknown data type %s", facetValue.GetType())
+ }
+ dst = append(dst, f)
+ }
+ return dst, nil
+}
+
+func namespaceMod(m proto.Message, namespace string) {
+ set := func(s **string) {
+ if *s == nil {
+ *s = &namespace
+ }
+ }
+ switch m := m.(type) {
+ case *pb.IndexDocumentRequest:
+ set(&m.Params.IndexSpec.Namespace)
+ case *pb.ListDocumentsRequest:
+ set(&m.Params.IndexSpec.Namespace)
+ case *pb.DeleteDocumentRequest:
+ set(&m.Params.IndexSpec.Namespace)
+ case *pb.SearchRequest:
+ set(&m.Params.IndexSpec.Namespace)
+ }
+}
+
+func init() {
+ internal.RegisterErrorCodeMap("search", pb.SearchServiceError_ErrorCode_name)
+ internal.NamespaceMods["search"] = namespaceMod
+}
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-05-20 19:21:03 +0000