From 375c0632fab03e3fb54865e320585888499c076d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 30 Nov 2017 09:07:04 -0500 Subject: PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612) * Created message export scheduled task * Added CLI command to immediately kick off an export job * Added email addresses for users joining and leaving the channel to the export * Added support for both MySQL and PostgreSQL * Fixing gofmt error * Added a new ChannelMemberHistory store and associated tests * Updating the ChannelMemberHistory channel as users create/join/leave channels * Added user email to the message export object so it can be included in the actiance export xml * Don't fail to log a leave event if a corresponding join event wasn't logged * Adding copyright notices * Adding message export settings to daily diagnostics report * Added System Console integration for message export * Cleaned up TODOs * Made batch size configurable * Added export from timestamp to CLI command * Made ChannelMemberHistory table updates best effort * Added a context-based timeout option to the message export CLI * Minor PR updates/improvements * Removed unnecessary fields from MessageExport object to reduce query overhead * Removed JSON functions from the message export query in an effort to optimize performance * Changed the way that channel member history queries and purges work to better account for edge cases * Fixing a test I missed with the last refactor * Added file copy functionality to file backend, improved config validation, added default config values * Fixed file copy tests * More concise use of the testing libraries * Fixed context leak error * Changed default export path to correctly place an 'export' directory under the 'data' directory * Can't delete records from a read replica * Fixed copy file tests * Start job workers when license is applied, if configured to do so * Suggestions from the PR * Moar unit tests * Fixed test imports --- model/config_test.go | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) (limited to 'model/config_test.go') diff --git a/model/config_test.go b/model/config_test.go index 86958458c..58f690165 100644 --- a/model/config_test.go +++ b/model/config_test.go @@ -5,6 +5,10 @@ package model import ( "testing" + + "os" + + "github.com/stretchr/testify/require" ) func TestConfigDefaultFileSettingsDirectory(t *testing.T) { @@ -33,3 +37,109 @@ func TestConfigDefaultFileSettingsS3SSE(t *testing.T) { t.Fatal("FileSettings.AmazonS3SSE should default to false") } } + +func TestMessageExportSettingsIsValidEnableExportNotSet(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{} + + // should fail fast because mes.EnableExport is not set + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidEnableExportFalse(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(false), + } + + // should fail fast because message export isn't enabled + require.Nil(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidExportFromTimestampInvalid(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + } + + // should fail fast because export from timestamp isn't set + require.Error(t, mes.isValid(*fs)) + + mes.ExportFromTimestamp = NewInt64(-1) + + // should fail fast because export from timestamp isn't valid + require.Error(t, mes.isValid(*fs)) + + mes.ExportFromTimestamp = NewInt64(GetMillis() + 10000) + + // should fail fast because export from timestamp is greater than current time + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidDailyRunTimeInvalid(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + } + + // should fail fast because daily runtime isn't set + require.Error(t, mes.isValid(*fs)) + + mes.DailyRunTime = NewString("33:33:33") + + // should fail fast because daily runtime is invalid format + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) { + fs := &FileSettings{ + DriverName: NewString("foo"), // bypass file location check + } + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + DailyRunTime: NewString("15:04"), + FileLocation: NewString("foo"), + } + + // should fail fast because batch size isn't set + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidFileLocationInvalid(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + DailyRunTime: NewString("15:04"), + BatchSize: NewInt(100), + } + + // should fail fast because FileLocation isn't set + require.Error(t, mes.isValid(*fs)) + + // if using the local file driver, there are more rules for FileLocation + fs.DriverName = NewString(IMAGE_DRIVER_LOCAL) + fs.Directory, _ = os.Getwd() + mes.FileLocation = NewString("") + + // should fail fast because file location is not relative to basepath + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValid(t *testing.T) { + fs := &FileSettings{ + DriverName: NewString("foo"), // bypass file location check + } + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + DailyRunTime: NewString("15:04"), + FileLocation: NewString("foo"), + BatchSize: NewInt(100), + } + + // should pass because everything is valid + require.Nil(t, mes.isValid(*fs)) +} -- cgit v1.2.3-1-g7c22