PLT-7302: Aggregate Elasticsearch indexes over a certain age. (#7224)

* PLT-7302: Aggregate Elasticsearch indexes over a certain age.

This is done by a scheduled daily job, in order to keep the shard count
to a sensible level in Elasticsearch.

* Use map[string]string instead of StringMap

2 files changed, 49 insertions, 24 deletions

diff --git a/model/config.go b/model/config.go
index 933c643f2..9906723e7 100644
--- a/model/config.go
+++ b/model/config.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"time"
 )
 
 const (
@@ -123,11 +124,13 @@ const (
 	ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_COLOR      = "#f2a93b"
 	ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_TEXT_COLOR = "#333333"
 
-	ELASTICSEARCH_SETTINGS_DEFAULT_CONNECTION_URL      = ""
-	ELASTICSEARCH_SETTINGS_DEFAULT_USERNAME            = ""
-	ELASTICSEARCH_SETTINGS_DEFAULT_PASSWORD            = ""
-	ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_REPLICAS = 1
-	ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS   = 1
+	ELASTICSEARCH_SETTINGS_DEFAULT_CONNECTION_URL                  = ""
+	ELASTICSEARCH_SETTINGS_DEFAULT_USERNAME                        = ""
+	ELASTICSEARCH_SETTINGS_DEFAULT_PASSWORD                        = ""
+	ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_REPLICAS             = 1
+	ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS               = 1
+	ELASTICSEARCH_SETTINGS_DEFAULT_AGGREGATE_POSTS_AFTER_DAYS      = 365
+	ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME = "03:00"
 )
 
 type ServiceSettings struct {
@@ -441,14 +444,16 @@ type WebrtcSettings struct {
 }
 
 type ElasticsearchSettings struct {
-	ConnectionUrl     *string
-	Username          *string
-	Password          *string
-	EnableIndexing    *bool
-	EnableSearching   *bool
-	Sniff             *bool
-	PostIndexReplicas *int
-	PostIndexShards   *int
+	ConnectionUrl               *string
+	Username                    *string
+	Password                    *string
+	EnableIndexing              *bool
+	EnableSearching             *bool
+	Sniff                       *bool
+	PostIndexReplicas           *int
+	PostIndexShards             *int
+	AggregatePostsAfterDays     *int
+	PostsAggregatorJobStartTime *string
 }
 
 type DataRetentionSettings struct {
@@ -1452,6 +1457,16 @@ func (o *Config) SetDefaults() {
 		*o.ElasticsearchSettings.PostIndexShards = ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS
 	}
 
+	if o.ElasticsearchSettings.AggregatePostsAfterDays == nil {
+		o.ElasticsearchSettings.AggregatePostsAfterDays = new(int)
+		*o.ElasticsearchSettings.AggregatePostsAfterDays = ELASTICSEARCH_SETTINGS_DEFAULT_AGGREGATE_POSTS_AFTER_DAYS
+	}
+
+	if o.ElasticsearchSettings.PostsAggregatorJobStartTime == nil {
+		o.ElasticsearchSettings.PostsAggregatorJobStartTime = new(string)
+		*o.ElasticsearchSettings.PostsAggregatorJobStartTime = ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME
+	}
+
 	if o.DataRetentionSettings.Enable == nil {
 		o.DataRetentionSettings.Enable = new(bool)
 		*o.DataRetentionSettings.Enable = false
@@ -1700,6 +1715,14 @@ func (o *Config) IsValid() *AppError {
 		return NewLocAppError("Config.IsValid", "model.config.is_valid.elastic_search.enable_searching.app_error", nil, "")
 	}
 
+	if *o.ElasticsearchSettings.AggregatePostsAfterDays < 1 {
+		return NewAppError("Config.IsValid", "model.config.is_valid.elastic_search.aggregate_posts_after_days.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	if _, err := time.Parse("03:04", *o.ElasticsearchSettings.PostsAggregatorJobStartTime); err != nil {
+		return NewAppError("Config.IsValid", "model.config.is_valid.elastic_search.posts_aggregator_job_start_time.app_error", nil, err.Error(), http.StatusBadRequest)
+	}
+
 	return nil
 }
 
diff --git a/model/job.go b/model/job.go
index 004331a1f..258fa2bd3 100644
--- a/model/job.go
+++ b/model/job.go
@@ -10,8 +10,9 @@ import (
 )
 
 const (
-	JOB_TYPE_DATA_RETENTION              = "data_retention"
-	JOB_TYPE_ELASTICSEARCH_POST_INDEXING = "elasticsearch_post_indexing"
+	JOB_TYPE_DATA_RETENTION                 = "data_retention"
+	JOB_TYPE_ELASTICSEARCH_POST_INDEXING    = "elasticsearch_post_indexing"
+	JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION = "elasticsearch_post_aggregation"
 
 	JOB_STATUS_PENDING          = "pending"
 	JOB_STATUS_IN_PROGRESS      = "in_progress"
@@ -22,15 +23,15 @@ const (
 )
 
 type Job struct {
-	Id             string                 `json:"id"`
-	Type           string                 `json:"type"`
-	Priority       int64                  `json:"priority"`
-	CreateAt       int64                  `json:"create_at"`
-	StartAt        int64                  `json:"start_at"`
-	LastActivityAt int64                  `json:"last_activity_at"`
-	Status         string                 `json:"status"`
-	Progress       int64                  `json:"progress"`
-	Data           map[string]interface{} `json:"data"`
+	Id             string            `json:"id"`
+	Type           string            `json:"type"`
+	Priority       int64             `json:"priority"`
+	CreateAt       int64             `json:"create_at"`
+	StartAt        int64             `json:"start_at"`
+	LastActivityAt int64             `json:"last_activity_at"`
+	Status         string            `json:"status"`
+	Progress       int64             `json:"progress"`
+	Data           map[string]string `json:"data"`
 }
 
 func (j *Job) IsValid() *AppError {
@@ -45,6 +46,7 @@ func (j *Job) IsValid() *AppError {
 	switch j.Type {
 	case JOB_TYPE_DATA_RETENTION:
 	case JOB_TYPE_ELASTICSEARCH_POST_INDEXING:
+	case JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION:
 	default:
 		return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)
 	}