From 375c0632fab03e3fb54865e320585888499c076d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 30 Nov 2017 09:07:04 -0500 Subject: PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612) * Created message export scheduled task * Added CLI command to immediately kick off an export job * Added email addresses for users joining and leaving the channel to the export * Added support for both MySQL and PostgreSQL * Fixing gofmt error * Added a new ChannelMemberHistory store and associated tests * Updating the ChannelMemberHistory channel as users create/join/leave channels * Added user email to the message export object so it can be included in the actiance export xml * Don't fail to log a leave event if a corresponding join event wasn't logged * Adding copyright notices * Adding message export settings to daily diagnostics report * Added System Console integration for message export * Cleaned up TODOs * Made batch size configurable * Added export from timestamp to CLI command * Made ChannelMemberHistory table updates best effort * Added a context-based timeout option to the message export CLI * Minor PR updates/improvements * Removed unnecessary fields from MessageExport object to reduce query overhead * Removed JSON functions from the message export query in an effort to optimize performance * Changed the way that channel member history queries and purges work to better account for edge cases * Fixing a test I missed with the last refactor * Added file copy functionality to file backend, improved config validation, added default config values * Fixed file copy tests * More concise use of the testing libraries * Fixed context leak error * Changed default export path to correctly place an 'export' directory under the 'data' directory * Can't delete records from a read replica * Fixed copy file tests * Start job workers when license is applied, if configured to do so * Suggestions from the PR * Moar unit tests * Fixed test imports --- model/channel_member_history.go | 12 +++++ model/config.go | 66 ++++++++++++++++++++++++ model/config_test.go | 110 ++++++++++++++++++++++++++++++++++++++++ model/job.go | 2 + model/license.go | 6 +++ model/message_export.go | 18 +++++++ 6 files changed, 214 insertions(+) create mode 100644 model/channel_member_history.go create mode 100644 model/message_export.go (limited to 'model') diff --git a/model/channel_member_history.go b/model/channel_member_history.go new file mode 100644 index 000000000..bc71b580a --- /dev/null +++ b/model/channel_member_history.go @@ -0,0 +1,12 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package model + +type ChannelMemberHistory struct { + ChannelId string + UserId string + UserEmail string `db:"Email"` + JoinTime int64 + LeaveTime *int64 +} diff --git a/model/config.go b/model/config.go index e2f05d72e..1f56eb4f5 100644 --- a/model/config.go +++ b/model/config.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "path/filepath" "strings" "time" ) @@ -1508,6 +1509,36 @@ func (s *PluginSettings) SetDefaults() { } } +type MessageExportSettings struct { + EnableExport *bool + DailyRunTime *string + ExportFromTimestamp *int64 + FileLocation *string + BatchSize *int +} + +func (s *MessageExportSettings) SetDefaults() { + if s.EnableExport == nil { + s.EnableExport = NewBool(false) + } + + if s.FileLocation == nil { + s.FileLocation = NewString("export") + } + + if s.DailyRunTime == nil { + s.DailyRunTime = NewString("01:00") + } + + if s.ExportFromTimestamp == nil { + s.ExportFromTimestamp = NewInt64(0) + } + + if s.BatchSize == nil { + s.BatchSize = NewInt(10000) + } +} + type ConfigFunc func() *Config type Config struct { @@ -1538,6 +1569,7 @@ type Config struct { WebrtcSettings WebrtcSettings ElasticsearchSettings ElasticsearchSettings DataRetentionSettings DataRetentionSettings + MessageExportSettings MessageExportSettings JobSettings JobSettings PluginSettings PluginSettings } @@ -1617,6 +1649,7 @@ func (o *Config) SetDefaults() { o.LogSettings.SetDefaults() o.JobSettings.SetDefaults() o.WebrtcSettings.SetDefaults() + o.MessageExportSettings.SetDefaults() } func (o *Config) IsValid() *AppError { @@ -1680,6 +1713,10 @@ func (o *Config) IsValid() *AppError { return err } + if err := o.MessageExportSettings.isValid(o.FileSettings); err != nil { + return err + } + return nil } @@ -1998,6 +2035,35 @@ func (ls *LocalizationSettings) isValid() *AppError { return nil } +func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError { + if mes.EnableExport == nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.enable.app_error", nil, "", http.StatusBadRequest) + } + if *mes.EnableExport { + if mes.ExportFromTimestamp == nil || *mes.ExportFromTimestamp < 0 || *mes.ExportFromTimestamp > time.Now().Unix() { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_from.app_error", nil, "", http.StatusBadRequest) + } else if mes.DailyRunTime == nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, "", http.StatusBadRequest) + } else if _, err := time.Parse("15:04", *mes.DailyRunTime); err != nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest) + } else if mes.FileLocation == nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.app_error", nil, "", http.StatusBadRequest) + } else if mes.BatchSize == nil || *mes.BatchSize < 0 { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest) + } else if *fs.DriverName != IMAGE_DRIVER_LOCAL { + if absFileDir, err := filepath.Abs(fs.Directory); err != nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest) + } else if absMessageExportDir, err := filepath.Abs(*mes.FileLocation); err != nil { + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest) + } else if !strings.HasPrefix(absMessageExportDir, absFileDir) { + // configured export directory must be relative to data directory + return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, "", http.StatusBadRequest) + } + } + } + return nil +} + func (o *Config) GetSanitizeOptions() map[string]bool { options := map[string]bool{} options["fullname"] = o.PrivacySettings.ShowFullName diff --git a/model/config_test.go b/model/config_test.go index 86958458c..58f690165 100644 --- a/model/config_test.go +++ b/model/config_test.go @@ -5,6 +5,10 @@ package model import ( "testing" + + "os" + + "github.com/stretchr/testify/require" ) func TestConfigDefaultFileSettingsDirectory(t *testing.T) { @@ -33,3 +37,109 @@ func TestConfigDefaultFileSettingsS3SSE(t *testing.T) { t.Fatal("FileSettings.AmazonS3SSE should default to false") } } + +func TestMessageExportSettingsIsValidEnableExportNotSet(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{} + + // should fail fast because mes.EnableExport is not set + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidEnableExportFalse(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(false), + } + + // should fail fast because message export isn't enabled + require.Nil(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidExportFromTimestampInvalid(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + } + + // should fail fast because export from timestamp isn't set + require.Error(t, mes.isValid(*fs)) + + mes.ExportFromTimestamp = NewInt64(-1) + + // should fail fast because export from timestamp isn't valid + require.Error(t, mes.isValid(*fs)) + + mes.ExportFromTimestamp = NewInt64(GetMillis() + 10000) + + // should fail fast because export from timestamp is greater than current time + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidDailyRunTimeInvalid(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + } + + // should fail fast because daily runtime isn't set + require.Error(t, mes.isValid(*fs)) + + mes.DailyRunTime = NewString("33:33:33") + + // should fail fast because daily runtime is invalid format + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) { + fs := &FileSettings{ + DriverName: NewString("foo"), // bypass file location check + } + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + DailyRunTime: NewString("15:04"), + FileLocation: NewString("foo"), + } + + // should fail fast because batch size isn't set + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValidFileLocationInvalid(t *testing.T) { + fs := &FileSettings{} + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + DailyRunTime: NewString("15:04"), + BatchSize: NewInt(100), + } + + // should fail fast because FileLocation isn't set + require.Error(t, mes.isValid(*fs)) + + // if using the local file driver, there are more rules for FileLocation + fs.DriverName = NewString(IMAGE_DRIVER_LOCAL) + fs.Directory, _ = os.Getwd() + mes.FileLocation = NewString("") + + // should fail fast because file location is not relative to basepath + require.Error(t, mes.isValid(*fs)) +} + +func TestMessageExportSettingsIsValid(t *testing.T) { + fs := &FileSettings{ + DriverName: NewString("foo"), // bypass file location check + } + mes := &MessageExportSettings{ + EnableExport: NewBool(true), + ExportFromTimestamp: NewInt64(0), + DailyRunTime: NewString("15:04"), + FileLocation: NewString("foo"), + BatchSize: NewInt(100), + } + + // should pass because everything is valid + require.Nil(t, mes.isValid(*fs)) +} diff --git a/model/job.go b/model/job.go index 843d73fad..9a7566025 100644 --- a/model/job.go +++ b/model/job.go @@ -12,6 +12,7 @@ import ( const ( JOB_TYPE_DATA_RETENTION = "data_retention" + JOB_TYPE_MESSAGE_EXPORT = "message_export" JOB_TYPE_ELASTICSEARCH_POST_INDEXING = "elasticsearch_post_indexing" JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION = "elasticsearch_post_aggregation" JOB_TYPE_LDAP_SYNC = "ldap_sync" @@ -50,6 +51,7 @@ func (j *Job) IsValid() *AppError { case JOB_TYPE_ELASTICSEARCH_POST_INDEXING: case JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION: case JOB_TYPE_LDAP_SYNC: + case JOB_TYPE_MESSAGE_EXPORT: default: return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest) } diff --git a/model/license.go b/model/license.go index 3e42a2343..a81f882ca 100644 --- a/model/license.go +++ b/model/license.go @@ -55,6 +55,7 @@ type Features struct { ThemeManagement *bool `json:"theme_management"` EmailNotificationContents *bool `json:"email_notification_contents"` DataRetention *bool `json:"data_retention"` + MessageExport *bool `json:"message_export"` // after we enabled more features for webrtc we'll need to control them with this FutureFeatures *bool `json:"future_features"` @@ -76,6 +77,7 @@ func (f *Features) ToMap() map[string]interface{} { "elastic_search": *f.Elasticsearch, "email_notification_contents": *f.EmailNotificationContents, "data_retention": *f.DataRetention, + "message_export": *f.MessageExport, "future": *f.FutureFeatures, } } @@ -152,6 +154,10 @@ func (f *Features) SetDefaults() { if f.DataRetention == nil { f.DataRetention = NewBool(*f.FutureFeatures) } + + if f.MessageExport == nil { + f.MessageExport = NewBool(*f.FutureFeatures) + } } func (l *License) IsExpired() bool { diff --git a/model/message_export.go b/model/message_export.go new file mode 100644 index 000000000..b59b114d4 --- /dev/null +++ b/model/message_export.go @@ -0,0 +1,18 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package model + +type MessageExport struct { + ChannelId *string + ChannelDisplayName *string + + UserId *string + UserEmail *string + + PostId *string + PostCreateAt *int64 + PostMessage *string + PostType *string + PostFileIds StringArray +} -- cgit v1.2.3-1-g7c22