PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)

* Created message export scheduled task

* Added CLI command to immediately kick off an export job

* Added email addresses for users joining and leaving the channel to the export

* Added support for both MySQL and PostgreSQL

* Fixing gofmt error

* Added a new ChannelMemberHistory store and associated tests

* Updating the ChannelMemberHistory channel as users create/join/leave channels

* Added user email to the message export object so it can be included in the actiance export xml

* Don't fail to log a leave event if a corresponding join event wasn't logged

* Adding copyright notices

* Adding message export settings to daily diagnostics report

* Added System Console integration for message export

* Cleaned up TODOs

* Made batch size configurable

* Added export from timestamp to CLI command

* Made ChannelMemberHistory table updates best effort

* Added a context-based timeout option to the message export CLI

* Minor PR updates/improvements

* Removed unnecessary fields from MessageExport object to reduce query overhead

* Removed JSON functions from the message export query in an effort to optimize performance

* Changed the way that channel member history queries and purges work to better account for edge cases

* Fixing a test I missed with the last refactor

* Added file copy functionality to file backend, improved config validation, added default config values

* Fixed file copy tests

* More concise use of the testing libraries

* Fixed context leak error

* Changed default export path to correctly place an 'export' directory under the 'data' directory

* Can't delete records from a read replica

* Fixed copy file tests

* Start job workers when license is applied, if configured to do so

* Suggestions from the PR

* Moar unit tests

* Fixed test imports

1 files changed, 110 insertions, 0 deletions

diff --git a/model/config_test.go b/model/config_test.go
index 86958458c..58f690165 100644
--- a/model/config_test.go
+++ b/model/config_test.go
@@ -5,6 +5,10 @@ package model
 
 import (
 	"testing"
+
+	"os"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestConfigDefaultFileSettingsDirectory(t *testing.T) {
@@ -33,3 +37,109 @@ func TestConfigDefaultFileSettingsS3SSE(t *testing.T) {
 		t.Fatal("FileSettings.AmazonS3SSE should default to false")
 	}
 }
+
+func TestMessageExportSettingsIsValidEnableExportNotSet(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{}
+
+	// should fail fast because mes.EnableExport is not set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidEnableExportFalse(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport: NewBool(false),
+	}
+
+	// should fail fast because message export isn't enabled
+	require.Nil(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidExportFromTimestampInvalid(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport: NewBool(true),
+	}
+
+	// should fail fast because export from timestamp isn't set
+	require.Error(t, mes.isValid(*fs))
+
+	mes.ExportFromTimestamp = NewInt64(-1)
+
+	// should fail fast because export from timestamp isn't valid
+	require.Error(t, mes.isValid(*fs))
+
+	mes.ExportFromTimestamp = NewInt64(GetMillis() + 10000)
+
+	// should fail fast because export from timestamp is greater than current time
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidDailyRunTimeInvalid(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+	}
+
+	// should fail fast because daily runtime isn't set
+	require.Error(t, mes.isValid(*fs))
+
+	mes.DailyRunTime = NewString("33:33:33")
+
+	// should fail fast because daily runtime is invalid format
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		FileLocation:        NewString("foo"),
+	}
+
+	// should fail fast because batch size isn't set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidFileLocationInvalid(t *testing.T) {
+	fs := &FileSettings{}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should fail fast because FileLocation isn't set
+	require.Error(t, mes.isValid(*fs))
+
+	// if using the local file driver, there are more rules for FileLocation
+	fs.DriverName = NewString(IMAGE_DRIVER_LOCAL)
+	fs.Directory, _ = os.Getwd()
+	mes.FileLocation = NewString("")
+
+	// should fail fast because file location is not relative to basepath
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValid(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		FileLocation:        NewString("foo"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should pass because everything is valid
+	require.Nil(t, mes.isValid(*fs))
+}