PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)

* Created message export scheduled task

* Added CLI command to immediately kick off an export job

* Added email addresses for users joining and leaving the channel to the export

* Added support for both MySQL and PostgreSQL

* Fixing gofmt error

* Added a new ChannelMemberHistory store and associated tests

* Updating the ChannelMemberHistory channel as users create/join/leave channels

* Added user email to the message export object so it can be included in the actiance export xml

* Don't fail to log a leave event if a corresponding join event wasn't logged

* Adding copyright notices

* Adding message export settings to daily diagnostics report

* Added System Console integration for message export

* Cleaned up TODOs

* Made batch size configurable

* Added export from timestamp to CLI command

* Made ChannelMemberHistory table updates best effort

* Added a context-based timeout option to the message export CLI

* Minor PR updates/improvements

* Removed unnecessary fields from MessageExport object to reduce query overhead

* Removed JSON functions from the message export query in an effort to optimize performance

* Changed the way that channel member history queries and purges work to better account for edge cases

* Fixing a test I missed with the last refactor

* Added file copy functionality to file backend, improved config validation, added default config values

* Fixed file copy tests

* More concise use of the testing libraries

* Fixed context leak error

* Changed default export path to correctly place an 'export' directory under the 'data' directory

* Can't delete records from a read replica

* Fixed copy file tests

* Start job workers when license is applied, if configured to do so

* Suggestions from the PR

* Moar unit tests

* Fixed test imports

1 files changed, 117 insertions, 0 deletions

diff --git a/store/storetest/compliance_store.go b/store/storetest/compliance_store.go
index 514910f6f..c5bd60f05 100644
--- a/store/storetest/compliance_store.go
+++ b/store/storetest/compliance_store.go
@@ -9,12 +9,14 @@ import (
 
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestComplianceStore(t *testing.T, ss store.Store) {
 	t.Run("", func(t *testing.T) { testComplianceStore(t, ss) })
 	t.Run("ComplianceExport", func(t *testing.T) { testComplianceExport(t, ss) })
 	t.Run("ComplianceExportDirectMessages", func(t *testing.T) { testComplianceExportDirectMessages(t, ss) })
+	t.Run("MessageExport", func(t *testing.T) { testComplianceMessageExport(t, ss) })
 }
 
 func testComplianceStore(t *testing.T, ss store.Store) {
@@ -316,3 +318,118 @@ func testComplianceExportDirectMessages(t *testing.T, ss store.Store) {
 		}
 	}
 }
+
+func testComplianceMessageExport(t *testing.T, ss store.Store) {
+	// get the starting number of message export entries
+	startTime := model.GetMillis()
+	var numMessageExports = 0
+	if r1 := <-ss.Compliance().MessageExport(startTime-10, 10); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		messages := r1.Data.([]*model.MessageExport)
+		numMessageExports = len(messages)
+	}
+
+	// need a team
+	team := &model.Team{
+		DisplayName: "DisplayName",
+		Name:        "zz" + model.NewId() + "b",
+		Email:       model.NewId() + "@nowhere.com",
+		Type:        model.TEAM_OPEN,
+	}
+	team = store.Must(ss.Team().Save(team)).(*model.Team)
+
+	// and two users that are a part of that team
+	user1 := &model.User{
+		Email: model.NewId(),
+	}
+	user1 = store.Must(ss.User().Save(user1)).(*model.User)
+	store.Must(ss.Team().SaveMember(&model.TeamMember{
+		TeamId: team.Id,
+		UserId: user1.Id,
+	}, -1))
+
+	user2 := &model.User{
+		Email: model.NewId(),
+	}
+	user2 = store.Must(ss.User().Save(user2)).(*model.User)
+	store.Must(ss.Team().SaveMember(&model.TeamMember{
+		TeamId: team.Id,
+		UserId: user2.Id,
+	}, -1))
+
+	// need a public channel as well as a DM channel between the two users
+	channel := &model.Channel{
+		TeamId:      team.Id,
+		Name:        model.NewId(),
+		DisplayName: "Channel2",
+		Type:        model.CHANNEL_OPEN,
+	}
+	channel = store.Must(ss.Channel().Save(channel, -1)).(*model.Channel)
+	directMessageChannel := store.Must(ss.Channel().CreateDirectChannel(user1.Id, user2.Id)).(*model.Channel)
+
+	// user1 posts twice in the public channel
+	post1 := &model.Post{
+		ChannelId: channel.Id,
+		UserId:    user1.Id,
+		CreateAt:  startTime,
+		Message:   "zz" + model.NewId() + "a",
+	}
+	post1 = store.Must(ss.Post().Save(post1)).(*model.Post)
+
+	post2 := &model.Post{
+		ChannelId: channel.Id,
+		UserId:    user1.Id,
+		CreateAt:  startTime + 10,
+		Message:   "zz" + model.NewId() + "b",
+	}
+	post2 = store.Must(ss.Post().Save(post2)).(*model.Post)
+
+	// user1 also sends a DM to user2
+	post3 := &model.Post{
+		ChannelId: directMessageChannel.Id,
+		UserId:    user1.Id,
+		CreateAt:  startTime + 20,
+		Message:   "zz" + model.NewId() + "c",
+	}
+	post3 = store.Must(ss.Post().Save(post3)).(*model.Post)
+
+	// fetch the message exports for all three posts that user1 sent
+	messageExportMap := map[string]model.MessageExport{}
+	if r1 := <-ss.Compliance().MessageExport(startTime-10, 10); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		messages := r1.Data.([]*model.MessageExport)
+		assert.Equal(t, numMessageExports+3, len(messages))
+
+		for _, v := range messages {
+			messageExportMap[*v.PostId] = *v
+		}
+	}
+
+	// post1 was made by user1 in channel1 and team1
+	assert.Equal(t, post1.Id, *messageExportMap[post1.Id].PostId)
+	assert.Equal(t, post1.CreateAt, *messageExportMap[post1.Id].PostCreateAt)
+	assert.Equal(t, post1.Message, *messageExportMap[post1.Id].PostMessage)
+	assert.Equal(t, channel.Id, *messageExportMap[post1.Id].ChannelId)
+	assert.Equal(t, channel.DisplayName, *messageExportMap[post1.Id].ChannelDisplayName)
+	assert.Equal(t, user1.Id, *messageExportMap[post1.Id].UserId)
+	assert.Equal(t, user1.Email, *messageExportMap[post1.Id].UserEmail)
+
+	// post2 was made by user1 in channel1 and team1
+	assert.Equal(t, post2.Id, *messageExportMap[post2.Id].PostId)
+	assert.Equal(t, post2.CreateAt, *messageExportMap[post2.Id].PostCreateAt)
+	assert.Equal(t, post2.Message, *messageExportMap[post2.Id].PostMessage)
+	assert.Equal(t, channel.Id, *messageExportMap[post2.Id].ChannelId)
+	assert.Equal(t, channel.DisplayName, *messageExportMap[post2.Id].ChannelDisplayName)
+	assert.Equal(t, user1.Id, *messageExportMap[post2.Id].UserId)
+	assert.Equal(t, user1.Email, *messageExportMap[post2.Id].UserEmail)
+
+	// post3 is a DM between user1 and user2
+	assert.Equal(t, post3.Id, *messageExportMap[post3.Id].PostId)
+	assert.Equal(t, post3.CreateAt, *messageExportMap[post3.Id].PostCreateAt)
+	assert.Equal(t, post3.Message, *messageExportMap[post3.Id].PostMessage)
+	assert.Equal(t, directMessageChannel.Id, *messageExportMap[post3.Id].ChannelId)
+	assert.Equal(t, user1.Id, *messageExportMap[post3.Id].UserId)
+	assert.Equal(t, user1.Email, *messageExportMap[post3.Id].UserEmail)
+}