PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)

* Created message export scheduled task

* Added CLI command to immediately kick off an export job

* Added email addresses for users joining and leaving the channel to the export

* Added support for both MySQL and PostgreSQL

* Fixing gofmt error

* Added a new ChannelMemberHistory store and associated tests

* Updating the ChannelMemberHistory channel as users create/join/leave channels

* Added user email to the message export object so it can be included in the actiance export xml

* Don't fail to log a leave event if a corresponding join event wasn't logged

* Adding copyright notices

* Adding message export settings to daily diagnostics report

* Added System Console integration for message export

* Cleaned up TODOs

* Made batch size configurable

* Added export from timestamp to CLI command

* Made ChannelMemberHistory table updates best effort

* Added a context-based timeout option to the message export CLI

* Minor PR updates/improvements

* Removed unnecessary fields from MessageExport object to reduce query overhead

* Removed JSON functions from the message export query in an effort to optimize performance

* Changed the way that channel member history queries and purges work to better account for edge cases

* Fixing a test I missed with the last refactor

* Added file copy functionality to file backend, improved config validation, added default config values

* Fixed file copy tests

* More concise use of the testing libraries

* Fixed context leak error

* Changed default export path to correctly place an 'export' directory under the 'data' directory

* Can't delete records from a read replica

* Fixed copy file tests

* Start job workers when license is applied, if configured to do so

* Suggestions from the PR

* Moar unit tests

* Fixed test imports

1 files changed, 166 insertions, 0 deletions

diff --git a/app/channel_test.go b/app/channel_test.go
index 374b20657..d44af467d 100644
--- a/app/channel_test.go
+++ b/app/channel_test.go
@@ -1,9 +1,14 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
 package app
 
 import (
 	"testing"
 
 	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/store"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestPermanentDeleteChannel(t *testing.T) {
@@ -104,3 +109,164 @@ func TestMoveChannel(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestJoinDefaultChannelsTownSquare(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	// figure out the initial number of users in town square
+	townSquareChannelId := store.Must(th.App.Srv.Store.Channel().GetByName(th.BasicTeam.Id, "town-square", true)).(*model.Channel).Id
+	initialNumTownSquareUsers := len(store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, townSquareChannelId)).([]*model.ChannelMemberHistory))
+
+	// create a new user that joins the default channels
+	user := th.CreateUser()
+	th.App.JoinDefaultChannels(th.BasicTeam.Id, user, model.CHANNEL_USER_ROLE_ID, "")
+
+	// there should be a ChannelMemberHistory record for the user
+	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, townSquareChannelId)).([]*model.ChannelMemberHistory)
+	assert.Len(t, histories, initialNumTownSquareUsers+1)
+
+	found := false
+	for _, history := range histories {
+		if user.Id == history.UserId && townSquareChannelId == history.ChannelId {
+			found = true
+			break
+		}
+	}
+	assert.True(t, found)
+}
+
+func TestJoinDefaultChannelsOffTopic(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	// figure out the initial number of users in off-topic
+	offTopicChannelId := store.Must(th.App.Srv.Store.Channel().GetByName(th.BasicTeam.Id, "off-topic", true)).(*model.Channel).Id
+	initialNumTownSquareUsers := len(store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, offTopicChannelId)).([]*model.ChannelMemberHistory))
+
+	// create a new user that joins the default channels
+	user := th.CreateUser()
+	th.App.JoinDefaultChannels(th.BasicTeam.Id, user, model.CHANNEL_USER_ROLE_ID, "")
+
+	// there should be a ChannelMemberHistory record for the user
+	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, offTopicChannelId)).([]*model.ChannelMemberHistory)
+	assert.Len(t, histories, initialNumTownSquareUsers+1)
+
+	found := false
+	for _, history := range histories {
+		if user.Id == history.UserId && offTopicChannelId == history.ChannelId {
+			found = true
+			break
+		}
+	}
+	assert.True(t, found)
+}
+
+func TestCreateChannelPublic(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	// creates a public channel and adds basic user to it
+	publicChannel := th.createChannel(th.BasicTeam, model.CHANNEL_OPEN)
+
+	// there should be a ChannelMemberHistory record for the user
+	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, publicChannel.Id)).([]*model.ChannelMemberHistory)
+	assert.Len(t, histories, 1)
+	assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
+	assert.Equal(t, publicChannel.Id, histories[0].ChannelId)
+}
+
+func TestCreateChannelPrivate(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	// creates a private channel and adds basic user to it
+	privateChannel := th.createChannel(th.BasicTeam, model.CHANNEL_PRIVATE)
+
+	// there should be a ChannelMemberHistory record for the user
+	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, privateChannel.Id)).([]*model.ChannelMemberHistory)
+	assert.Len(t, histories, 1)
+	assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
+	assert.Equal(t, privateChannel.Id, histories[0].ChannelId)
+}
+
+func TestCreateGroupChannel(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	user1 := th.CreateUser()
+	user2 := th.CreateUser()
+
+	groupUserIds := make([]string, 0)
+	groupUserIds = append(groupUserIds, user1.Id)
+	groupUserIds = append(groupUserIds, user2.Id)
+	groupUserIds = append(groupUserIds, th.BasicUser.Id)
+
+	if channel, err := th.App.CreateGroupChannel(groupUserIds, th.BasicUser.Id); err != nil {
+		t.Fatal("Failed to create group channel. Error: " + err.Message)
+	} else {
+		// there should be a ChannelMemberHistory record for each user
+		histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)).([]*model.ChannelMemberHistory)
+		assert.Len(t, histories, 3)
+
+		channelMemberHistoryUserIds := make([]string, 0)
+		for _, history := range histories {
+			assert.Equal(t, channel.Id, history.ChannelId)
+			channelMemberHistoryUserIds = append(channelMemberHistoryUserIds, history.UserId)
+		}
+		assert.Equal(t, groupUserIds, channelMemberHistoryUserIds)
+	}
+}
+
+func TestAddUserToChannel(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	// create a user and add it to a channel
+	user := th.CreateUser()
+	if _, err := th.App.AddTeamMember(th.BasicTeam.Id, user.Id); err != nil {
+		t.Fatal("Failed to add user to team. Error: " + err.Message)
+	}
+
+	groupUserIds := make([]string, 0)
+	groupUserIds = append(groupUserIds, th.BasicUser.Id)
+	groupUserIds = append(groupUserIds, user.Id)
+
+	channel := th.createChannel(th.BasicTeam, model.CHANNEL_OPEN)
+	if _, err := th.App.AddUserToChannel(user, channel); err != nil {
+		t.Fatal("Failed to add user to channel. Error: " + err.Message)
+	}
+
+	// there should be a ChannelMemberHistory record for the user
+	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, channel.Id)).([]*model.ChannelMemberHistory)
+	assert.Len(t, histories, 2)
+	channelMemberHistoryUserIds := make([]string, 0)
+	for _, history := range histories {
+		assert.Equal(t, channel.Id, history.ChannelId)
+		channelMemberHistoryUserIds = append(channelMemberHistoryUserIds, history.UserId)
+	}
+	assert.Equal(t, groupUserIds, channelMemberHistoryUserIds)
+}
+
+func TestRemoveUserFromChannel(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	// a user creates a channel
+	publicChannel := th.createChannel(th.BasicTeam, model.CHANNEL_OPEN)
+	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, publicChannel.Id)).([]*model.ChannelMemberHistory)
+	assert.Len(t, histories, 1)
+	assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
+	assert.Equal(t, publicChannel.Id, histories[0].ChannelId)
+	assert.Nil(t, histories[0].LeaveTime)
+
+	// the user leaves that channel
+	if err := th.App.LeaveChannel(publicChannel.Id, th.BasicUser.Id); err != nil {
+		t.Fatal("Failed to remove user from channel. Error: " + err.Message)
+	}
+	histories = store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, publicChannel.Id)).([]*model.ChannelMemberHistory)
+	assert.Len(t, histories, 1)
+	assert.Equal(t, th.BasicUser.Id, histories[0].UserId)
+	assert.Equal(t, publicChannel.Id, histories[0].ChannelId)
+	assert.NotNil(t, histories[0].LeaveTime)
+}