diff options
author | Christopher Speller <crspeller@gmail.com> | 2018-01-29 14:17:40 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-01-29 14:17:40 -0800 |
commit | 961c04cae992eadb42d286d2f85f8a675bdc68c8 (patch) | |
tree | 3408f2d06f847e966c53485e2d54c692cdd037c1 /vendor/gopkg.in/olivere/elastic.v5/bulk_test.go | |
parent | 8d66523ba7d9a77129844be476732ebfd5272d64 (diff) | |
download | chat-961c04cae992eadb42d286d2f85f8a675bdc68c8.tar.gz chat-961c04cae992eadb42d286d2f85f8a675bdc68c8.tar.bz2 chat-961c04cae992eadb42d286d2f85f8a675bdc68c8.zip |
Upgrading server dependancies (#8154)
Diffstat (limited to 'vendor/gopkg.in/olivere/elastic.v5/bulk_test.go')
-rw-r--r-- | vendor/gopkg.in/olivere/elastic.v5/bulk_test.go | 188 |
1 files changed, 127 insertions, 61 deletions
diff --git a/vendor/gopkg.in/olivere/elastic.v5/bulk_test.go b/vendor/gopkg.in/olivere/elastic.v5/bulk_test.go index 394439630..f31ed6613 100644 --- a/vendor/gopkg.in/olivere/elastic.v5/bulk_test.go +++ b/vendor/gopkg.in/olivere/elastic.v5/bulk_test.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "math/rand" "net/http" "net/http/httptest" "testing" @@ -19,9 +20,9 @@ func TestBulk(t *testing.T) { tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."} tweet2 := tweet{User: "sandrae", Message: "Dancing all night long. Yeah."} - index1Req := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("1").Doc(tweet1) - index2Req := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("2").Doc(tweet2) - delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("tweet").Id("1") + index1Req := NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("1").Doc(tweet1) + index2Req := NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("2").Doc(tweet2) + delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("doc").Id("1") bulkRequest := client.Bulk() bulkRequest = bulkRequest.Add(index1Req) @@ -45,7 +46,7 @@ func TestBulk(t *testing.T) { } // Document with Id="1" should not exist - exists, err := client.Exists().Index(testIndexName).Type("tweet").Id("1").Do(context.TODO()) + exists, err := client.Exists().Index(testIndexName).Type("doc").Id("1").Do(context.TODO()) if err != nil { t.Fatal(err) } @@ -54,7 +55,7 @@ func TestBulk(t *testing.T) { } // Document with Id="2" should exist - exists, err = client.Exists().Index(testIndexName).Type("tweet").Id("2").Do(context.TODO()) + exists, err = client.Exists().Index(testIndexName).Type("doc").Id("2").Do(context.TODO()) if err != nil { t.Fatal(err) } @@ -68,7 +69,7 @@ func TestBulk(t *testing.T) { }{ 42, } - update1Req := NewBulkUpdateRequest().Index(testIndexName).Type("tweet").Id("2").Doc(&updateDoc) + update1Req := NewBulkUpdateRequest().Index(testIndexName).Type("doc").Id("2").Doc(&updateDoc) bulkRequest = client.Bulk() bulkRequest = bulkRequest.Add(update1Req) @@ -89,7 +90,7 @@ func TestBulk(t *testing.T) { } // Document with Id="1" should have a retweets count of 42 - doc, err := client.Get().Index(testIndexName).Type("tweet").Id("2").Do(context.TODO()) + doc, err := client.Get().Index(testIndexName).Type("doc").Id("2").Do(context.TODO()) if err != nil { t.Fatal(err) } @@ -112,7 +113,7 @@ func TestBulk(t *testing.T) { } // Update with script - update2Req := NewBulkUpdateRequest().Index(testIndexName).Type("tweet").Id("2"). + update2Req := NewBulkUpdateRequest().Index(testIndexName).Type("doc").Id("2"). RetryOnConflict(3). Script(NewScript("ctx._source.retweets += params.v").Param("v", 1)) bulkRequest = client.Bulk() @@ -133,7 +134,7 @@ func TestBulk(t *testing.T) { } // Document with Id="1" should have a retweets count of 43 - doc, err = client.Get().Index(testIndexName).Type("tweet").Id("2").Do(context.TODO()) + doc, err = client.Get().Index(testIndexName).Type("doc").Id("2").Do(context.TODO()) if err != nil { t.Fatal(err) } @@ -161,11 +162,11 @@ func TestBulkWithIndexSetOnClient(t *testing.T) { tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."} tweet2 := tweet{User: "sandrae", Message: "Dancing all night long. Yeah."} - index1Req := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("1").Doc(tweet1) - index2Req := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("2").Doc(tweet2) - delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("tweet").Id("1") + index1Req := NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("1").Doc(tweet1).Routing("1") + index2Req := NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("2").Doc(tweet2) + delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("doc").Id("1") - bulkRequest := client.Bulk().Index(testIndexName).Type("tweet") + bulkRequest := client.Bulk().Index(testIndexName).Type("doc") bulkRequest = bulkRequest.Add(index1Req) bulkRequest = bulkRequest.Add(index2Req) bulkRequest = bulkRequest.Add(delete1Req) @@ -183,7 +184,7 @@ func TestBulkWithIndexSetOnClient(t *testing.T) { } // Document with Id="1" should not exist - exists, err := client.Exists().Index(testIndexName).Type("tweet").Id("1").Do(context.TODO()) + exists, err := client.Exists().Index(testIndexName).Type("doc").Id("1").Do(context.TODO()) if err != nil { t.Fatal(err) } @@ -192,7 +193,7 @@ func TestBulkWithIndexSetOnClient(t *testing.T) { } // Document with Id="2" should exist - exists, err = client.Exists().Index(testIndexName).Type("tweet").Id("2").Do(context.TODO()) + exists, err = client.Exists().Index(testIndexName).Type("doc").Id("2").Do(context.TODO()) if err != nil { t.Fatal(err) } @@ -201,16 +202,18 @@ func TestBulkWithIndexSetOnClient(t *testing.T) { } } -func TestBulkRequestsSerialization(t *testing.T) { +func TestBulkIndexDeleteUpdate(t *testing.T) { client := setupTestClientAndCreateIndex(t) + //client := setupTestClientAndCreateIndexAndLog(t) tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."} tweet2 := tweet{User: "sandrae", Message: "Dancing all night long. Yeah."} - index1Req := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("1").Doc(tweet1) - index2Req := NewBulkIndexRequest().OpType("create").Index(testIndexName).Type("tweet").Id("2").Doc(tweet2) - delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("tweet").Id("1") - update2Req := NewBulkUpdateRequest().Index(testIndexName).Type("tweet").Id("2"). + index1Req := NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("1").Doc(tweet1) + index2Req := NewBulkIndexRequest().OpType("create").Index(testIndexName).Type("doc").Id("2").Doc(tweet2) + delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("doc").Id("1") + update2Req := NewBulkUpdateRequest().Index(testIndexName).Type("doc").Id("2"). + ReturnSource(true). Doc(struct { Retweets int `json:"retweets"` }{ @@ -227,13 +230,13 @@ func TestBulkRequestsSerialization(t *testing.T) { t.Errorf("expected bulkRequest.NumberOfActions %d; got %d", 4, bulkRequest.NumberOfActions()) } - expected := `{"index":{"_id":"1","_index":"` + testIndexName + `","_type":"tweet"}} + expected := `{"index":{"_index":"` + testIndexName + `","_id":"1","_type":"doc"}} {"user":"olivere","message":"Welcome to Golang and Elasticsearch.","retweets":0,"created":"0001-01-01T00:00:00Z"} -{"create":{"_id":"2","_index":"` + testIndexName + `","_type":"tweet"}} +{"create":{"_index":"` + testIndexName + `","_id":"2","_type":"doc"}} {"user":"sandrae","message":"Dancing all night long. Yeah.","retweets":0,"created":"0001-01-01T00:00:00Z"} -{"delete":{"_id":"1","_index":"` + testIndexName + `","_type":"tweet"}} -{"update":{"_id":"2","_index":"` + testIndexName + `","_type":"tweet"}} -{"doc":{"retweets":42}} +{"delete":{"_index":"` + testIndexName + `","_type":"doc","_id":"1"}} +{"update":{"_index":"` + testIndexName + `","_type":"doc","_id":"2"}} +{"doc":{"retweets":42},"_source":true} ` got, err := bulkRequest.bodyAsString() if err != nil { @@ -244,7 +247,7 @@ func TestBulkRequestsSerialization(t *testing.T) { } // Run the bulk request - bulkResponse, err := bulkRequest.Do(context.TODO()) + bulkResponse, err := bulkRequest.Pretty(true).Do(context.TODO()) if err != nil { t.Fatal(err) } @@ -290,6 +293,9 @@ func TestBulkRequestsSerialization(t *testing.T) { if created[0].Status != 201 { t.Errorf("expected created[0].Status == %d; got %d", 201, created[0].Status) } + if want, have := "created", created[0].Result; want != have { + t.Errorf("expected created[0].Result == %q; got %q", want, have) + } // Deleted actions deleted := bulkResponse.Deleted() @@ -305,8 +311,8 @@ func TestBulkRequestsSerialization(t *testing.T) { if deleted[0].Status != 200 { t.Errorf("expected deleted[0].Status == %d; got %d", 200, deleted[0].Status) } - if !deleted[0].Found { - t.Errorf("expected deleted[0].Found == %v; got %v", true, deleted[0].Found) + if want, have := "deleted", deleted[0].Result; want != have { + t.Errorf("expected deleted[0].Result == %q; got %q", want, have) } // Updated actions @@ -326,6 +332,25 @@ func TestBulkRequestsSerialization(t *testing.T) { if updated[0].Version != 2 { t.Errorf("expected updated[0].Version == %d; got %d", 2, updated[0].Version) } + if want, have := "updated", updated[0].Result; want != have { + t.Errorf("expected updated[0].Result == %q; got %q", want, have) + } + if updated[0].GetResult == nil { + t.Fatalf("expected updated[0].GetResult to be != nil; got nil") + } + if updated[0].GetResult.Source == nil { + t.Fatalf("expected updated[0].GetResult.Source to be != nil; got nil") + } + if want, have := true, updated[0].GetResult.Found; want != have { + t.Fatalf("expected updated[0].GetResult.Found to be != %v; got %v", want, have) + } + var doc tweet + if err := json.Unmarshal(*updated[0].GetResult.Source, &doc); err != nil { + t.Fatalf("expected to unmarshal updated[0].GetResult.Source; got %v", err) + } + if want, have := 42, doc.Retweets; want != have { + t.Fatalf("expected updated tweet to have Retweets = %v; got %v", want, have) + } // Succeeded actions succeeded := bulkResponse.Succeeded() @@ -371,7 +396,7 @@ func TestFailedBulkRequests(t *testing.T) { "items" : [ { "index" : { "_index" : "elastic-test", - "_type" : "tweet", + "_type" : "doc", "_id" : "1", "_version" : 1, "status" : 201 @@ -379,7 +404,7 @@ func TestFailedBulkRequests(t *testing.T) { }, { "create" : { "_index" : "elastic-test", - "_type" : "tweet", + "_type" : "doc", "_id" : "2", "_version" : 1, "status" : 423, @@ -391,7 +416,7 @@ func TestFailedBulkRequests(t *testing.T) { }, { "delete" : { "_index" : "elastic-test", - "_type" : "tweet", + "_type" : "doc", "_id" : "1", "_version" : 2, "status" : 404, @@ -400,7 +425,7 @@ func TestFailedBulkRequests(t *testing.T) { }, { "update" : { "_index" : "elastic-test", - "_type" : "tweet", + "_type" : "doc", "_id" : "2", "_version" : 2, "status" : 200 @@ -425,10 +450,10 @@ func TestBulkEstimatedSizeInBytes(t *testing.T) { tweet1 := tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."} tweet2 := tweet{User: "sandrae", Message: "Dancing all night long. Yeah."} - index1Req := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("1").Doc(tweet1) - index2Req := NewBulkIndexRequest().OpType("create").Index(testIndexName).Type("tweet").Id("2").Doc(tweet2) - delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("tweet").Id("1") - update2Req := NewBulkUpdateRequest().Index(testIndexName).Type("tweet").Id("2"). + index1Req := NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("1").Doc(tweet1) + index2Req := NewBulkIndexRequest().OpType("create").Index(testIndexName).Type("doc").Id("2").Doc(tweet2) + delete1Req := NewBulkDeleteRequest().Index(testIndexName).Type("doc").Id("1") + update2Req := NewBulkUpdateRequest().Index(testIndexName).Type("doc").Id("2"). Doc(struct { Retweets int `json:"retweets"` }{ @@ -468,13 +493,39 @@ func TestBulkEstimatedSizeInBytes(t *testing.T) { func TestBulkEstimateSizeInBytesLength(t *testing.T) { client := setupTestClientAndCreateIndex(t) s := client.Bulk() - r := NewBulkDeleteRequest().Index(testIndexName).Type("tweet").Id("1") + r := NewBulkDeleteRequest().Index(testIndexName).Type("doc").Id("1") s = s.Add(r) if got, want := s.estimateSizeInBytes(r), int64(1+len(r.String())); got != want { t.Fatalf("expected %d; got: %d", want, got) } } +func TestBulkContentType(t *testing.T) { + var header http.Header + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + header = r.Header + fmt.Fprintln(w, `{}`) + })) + defer ts.Close() + + client, err := NewSimpleClient(SetURL(ts.URL)) + if err != nil { + t.Fatal(err) + } + indexReq := NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("1").Doc(tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."}) + if _, err := client.Bulk().Add(indexReq).Do(context.Background()); err != nil { + t.Fatal(err) + } + if header == nil { + t.Fatalf("expected header, got %v", header) + } + if want, have := "application/x-ndjson", header.Get("Content-Type"); want != have { + t.Fatalf("Content-Type: want %q, have %q", want, have) + } +} + +// -- Benchmarks -- + var benchmarkBulkEstimatedSizeInBytes int64 func BenchmarkBulkEstimatedSizeInBytesWith1Request(b *testing.B) { @@ -482,9 +533,9 @@ func BenchmarkBulkEstimatedSizeInBytesWith1Request(b *testing.B) { s := client.Bulk() var result int64 for n := 0; n < b.N; n++ { - s = s.Add(NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("1").Doc(struct{ A string }{"1"})) - s = s.Add(NewBulkUpdateRequest().Index(testIndexName).Type("tweet").Id("1").Doc(struct{ A string }{"2"})) - s = s.Add(NewBulkDeleteRequest().Index(testIndexName).Type("tweet").Id("1")) + s = s.Add(NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("1").Doc(struct{ A string }{"1"})) + s = s.Add(NewBulkUpdateRequest().Index(testIndexName).Type("doc").Id("1").Doc(struct{ A string }{"2"})) + s = s.Add(NewBulkDeleteRequest().Index(testIndexName).Type("doc").Id("1")) result = s.EstimatedSizeInBytes() s.reset() } @@ -498,9 +549,9 @@ func BenchmarkBulkEstimatedSizeInBytesWith100Requests(b *testing.B) { var result int64 for n := 0; n < b.N; n++ { for i := 0; i < 100; i++ { - s = s.Add(NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("1").Doc(struct{ A string }{"1"})) - s = s.Add(NewBulkUpdateRequest().Index(testIndexName).Type("tweet").Id("1").Doc(struct{ A string }{"2"})) - s = s.Add(NewBulkDeleteRequest().Index(testIndexName).Type("tweet").Id("1")) + s = s.Add(NewBulkIndexRequest().Index(testIndexName).Type("doc").Id("1").Doc(struct{ A string }{"1"})) + s = s.Add(NewBulkUpdateRequest().Index(testIndexName).Type("doc").Id("1").Doc(struct{ A string }{"2"})) + s = s.Add(NewBulkDeleteRequest().Index(testIndexName).Type("doc").Id("1")) } result = s.EstimatedSizeInBytes() s.reset() @@ -509,26 +560,41 @@ func BenchmarkBulkEstimatedSizeInBytesWith100Requests(b *testing.B) { benchmarkBulkEstimatedSizeInBytes = result // ensure the compiler doesn't optimize } -func TestBulkContentType(t *testing.T) { - var header http.Header - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - header = r.Header - fmt.Fprintln(w, `{}`) - })) - defer ts.Close() +func BenchmarkBulkAllocs(b *testing.B) { + b.Run("1000 docs with 64 byte", func(b *testing.B) { benchmarkBulkAllocs(b, 64, 1000) }) + b.Run("1000 docs with 1 KiB", func(b *testing.B) { benchmarkBulkAllocs(b, 1024, 1000) }) + b.Run("1000 docs with 4 KiB", func(b *testing.B) { benchmarkBulkAllocs(b, 4096, 1000) }) + b.Run("1000 docs with 16 KiB", func(b *testing.B) { benchmarkBulkAllocs(b, 16*1024, 1000) }) + b.Run("1000 docs with 64 KiB", func(b *testing.B) { benchmarkBulkAllocs(b, 64*1024, 1000) }) + b.Run("1000 docs with 256 KiB", func(b *testing.B) { benchmarkBulkAllocs(b, 256*1024, 1000) }) + b.Run("1000 docs with 1 MiB", func(b *testing.B) { benchmarkBulkAllocs(b, 1024*1024, 1000) }) +} - client, err := NewSimpleClient(SetURL(ts.URL)) - if err != nil { - t.Fatal(err) - } - indexReq := NewBulkIndexRequest().Index(testIndexName).Type("tweet").Id("1").Doc(tweet{User: "olivere", Message: "Welcome to Golang and Elasticsearch."}) - if _, err := client.Bulk().Add(indexReq).Do(context.Background()); err != nil { - t.Fatal(err) +const ( + charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" +) + +func benchmarkBulkAllocs(b *testing.B, size, num int) { + buf := make([]byte, size) + for i := range buf { + buf[i] = charset[rand.Intn(len(charset))] } - if header == nil { - t.Fatalf("expected header, got %v", header) + + s := &BulkService{} + n := 0 + for { + n++ + s = s.Add(NewBulkIndexRequest().Index("test").Type("doc").Id("1").Doc(struct { + S string `json:"s"` + }{ + S: string(buf), + })) + if n >= num { + break + } } - if want, have := "application/x-ndjson", header.Get("Content-Type"); want != have { - t.Fatalf("Content-Type: want %q, have %q", want, have) + for i := 0; i < b.N; i++ { + s.bodyAsString() } + b.ReportAllocs() } |