1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
|
// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.
// BulkProcessor runs a bulk processing job that fills an index
// given certain criteria like flush interval etc.
//
// Example
//
// bulk_processor -url=http://127.0.0.1:9200/bulk-processor-test?sniff=false -n=100000 -flush-interval=1s
//
package main
import (
"context"
"flag"
"fmt"
"log"
"math/rand"
"os"
"os/signal"
"sync/atomic"
"syscall"
"time"
"github.com/google/uuid"
"github.com/olivere/elastic"
"github.com/olivere/elastic/config"
)
func main() {
var (
url = flag.String("url", "http://localhost:9200/bulk-processor-test", "Elasticsearch URL")
numWorkers = flag.Int("num-workers", 4, "Number of workers")
n = flag.Int64("n", -1, "Number of documents to process (-1 for unlimited)")
flushInterval = flag.Duration("flush-interval", 1*time.Second, "Flush interval")
bulkActions = flag.Int("bulk-actions", 0, "Number of bulk actions before committing")
bulkSize = flag.Int("bulk-size", 0, "Size of bulk requests before committing")
)
flag.Parse()
log.SetFlags(0)
rand.Seed(time.Now().UnixNano())
// Parse configuration from URL
cfg, err := config.Parse(*url)
if err != nil {
log.Fatal(err)
}
// Create an Elasticsearch client from the parsed config
client, err := elastic.NewClientFromConfig(cfg)
if err != nil {
log.Fatal(err)
}
// Drop old index
exists, err := client.IndexExists(cfg.Index).Do(context.Background())
if err != nil {
log.Fatal(err)
}
if exists {
_, err = client.DeleteIndex(cfg.Index).Do(context.Background())
if err != nil {
log.Fatal(err)
}
}
// Create processor
bulkp := elastic.NewBulkProcessorService(client).
Name("bulk-test-processor").
Stats(true).
Backoff(elastic.StopBackoff{}).
FlushInterval(*flushInterval).
Workers(*numWorkers)
if *bulkActions > 0 {
bulkp = bulkp.BulkActions(*bulkActions)
}
if *bulkSize > 0 {
bulkp = bulkp.BulkSize(*bulkSize)
}
p, err := bulkp.Do(context.Background())
if err != nil {
log.Fatal(err)
}
var created int64
errc := make(chan error, 1)
go func() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
<-c
errc <- nil
}()
go func() {
defer func() {
if err := p.Close(); err != nil {
errc <- err
}
}()
type Doc struct {
Timestamp time.Time `json:"@timestamp"`
}
for {
current := atomic.AddInt64(&created, 1)
if *n > 0 && current >= *n {
errc <- nil
return
}
r := elastic.NewBulkIndexRequest().
Index(cfg.Index).
Type("doc").
Id(uuid.New().String()).
Doc(Doc{Timestamp: time.Now()})
p.Add(r)
time.Sleep(time.Duration(rand.Intn(1000)) * time.Microsecond)
}
}()
go func() {
t := time.NewTicker(1 * time.Second)
defer t.Stop()
for range t.C {
stats := p.Stats()
written := atomic.LoadInt64(&created)
var queued int64
for _, w := range stats.Workers {
queued += w.Queued
}
fmt.Printf("Queued=%5d Written=%8d Succeeded=%8d Failed=%8d Comitted=%6d Flushed=%6d\n",
queued,
written,
stats.Succeeded,
stats.Failed,
stats.Committed,
stats.Flushed,
)
}
}()
if err := <-errc; err != nil {
log.Fatal(err)
}
}
|
