PLT-7934: Make query for bulk elasticsearch indexing more efficient. (#7664)

1 files changed, 34 insertions, 14 deletions

diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go
index a1b25b5c5..d8f93d2bc 100644
--- a/store/sqlstore/post_store.go
+++ b/store/sqlstore/post_store.go
@@ -1067,29 +1067,37 @@ func (s SqlPostStore) GetPostsByIds(postIds []string) store.StoreChannel {
 	})
 }
 
-func (s SqlPostStore) GetPostsBatchForIndexing(startTime int64, limit int) store.StoreChannel {
+func (s SqlPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) store.StoreChannel {
 	return store.Do(func(result *store.StoreResult) {
 		var posts []*model.PostForIndexing
 		_, err1 := s.GetSearchReplica().Select(&posts,
-			`(SELECT
-			    Posts.*,
-			    Channels.TeamId,
-			    ParentPosts.CreateAt ParentCreateAt
-			FROM
-				Posts
+			`SELECT
+				PostsQuery.*, Channels.TeamId, ParentPosts.CreateAt ParentCreateAt
+			FROM (
+				SELECT
+					*
+				FROM
+					Posts
+				WHERE
+					Posts.CreateAt >= :StartTime
+				AND
+					Posts.CreateAt < :EndTime
+				ORDER BY
+					CreateAt ASC
+				LIMIT
+					1000
+				)
+			AS
+				PostsQuery
 			LEFT JOIN
 				Channels
 			ON
-				Posts.ChannelId = Channels.Id
+				PostsQuery.ChannelId = Channels.Id
 			LEFT JOIN
 				Posts ParentPosts
 			ON
-				Posts.RootId = ParentPosts.Id
-			WHERE
-				Posts.CreateAt >= :StartTime
-			ORDER BY CreateAt ASC
-			LIMIT :NumPosts)`,
-			map[string]interface{}{"StartTime": startTime, "NumPosts": limit})
+				PostsQuery.RootId = ParentPosts.Id`,
+			map[string]interface{}{"StartTime": startTime, "EndTime": endTime, "NumPosts": limit})
 
 		if err1 != nil {
 			result.Err = model.NewAppError("SqlPostStore.GetPostContext", "store.sql_post.get_posts_batch_for_indexing.get.app_error", nil, err1.Error(), http.StatusInternalServerError)
@@ -1122,3 +1130,15 @@ func (s SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) store.Sto
 		}
 	})
 }
+
+func (s SqlPostStore) GetOldest() store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		var post model.Post
+		err := s.GetReplica().SelectOne(&post, "SELECT * FROM Posts ORDER BY CreateAt LIMIT 1")
+		if err != nil {
+			result.Err = model.NewAppError("SqlPostStore.GetOldest", "store.sql_post.get.app_error", nil, err.Error(), http.StatusNotFound)
+		}
+
+		result.Data = &post
+	})
+}