From 375c0632fab03e3fb54865e320585888499c076d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 30 Nov 2017 09:07:04 -0500 Subject: PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612) * Created message export scheduled task * Added CLI command to immediately kick off an export job * Added email addresses for users joining and leaving the channel to the export * Added support for both MySQL and PostgreSQL * Fixing gofmt error * Added a new ChannelMemberHistory store and associated tests * Updating the ChannelMemberHistory channel as users create/join/leave channels * Added user email to the message export object so it can be included in the actiance export xml * Don't fail to log a leave event if a corresponding join event wasn't logged * Adding copyright notices * Adding message export settings to daily diagnostics report * Added System Console integration for message export * Cleaned up TODOs * Made batch size configurable * Added export from timestamp to CLI command * Made ChannelMemberHistory table updates best effort * Added a context-based timeout option to the message export CLI * Minor PR updates/improvements * Removed unnecessary fields from MessageExport object to reduce query overhead * Removed JSON functions from the message export query in an effort to optimize performance * Changed the way that channel member history queries and purges work to better account for edge cases * Fixing a test I missed with the last refactor * Added file copy functionality to file backend, improved config validation, added default config values * Fixed file copy tests * More concise use of the testing libraries * Fixed context leak error * Changed default export path to correctly place an 'export' directory under the 'data' directory * Can't delete records from a read replica * Fixed copy file tests * Start job workers when license is applied, if configured to do so * Suggestions from the PR * Moar unit tests * Fixed test imports --- cmd/platform/mattermost.go | 2 +- cmd/platform/message_export.go | 79 +++++++++++++++++++++++++++++++++++++ cmd/platform/message_export_test.go | 65 ++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 cmd/platform/message_export.go create mode 100644 cmd/platform/message_export_test.go (limited to 'cmd/platform') diff --git a/cmd/platform/mattermost.go b/cmd/platform/mattermost.go index 6c015c6db..be2ff8164 100644 --- a/cmd/platform/mattermost.go +++ b/cmd/platform/mattermost.go @@ -36,7 +36,7 @@ func init() { resetCmd.Flags().Bool("confirm", false, "Confirm you really want to delete everything and a DB backup has been performed.") - rootCmd.AddCommand(serverCmd, versionCmd, userCmd, teamCmd, licenseCmd, importCmd, resetCmd, channelCmd, rolesCmd, testCmd, ldapCmd, configCmd, jobserverCmd, commandCmd) + rootCmd.AddCommand(serverCmd, versionCmd, userCmd, teamCmd, licenseCmd, importCmd, resetCmd, channelCmd, rolesCmd, testCmd, ldapCmd, configCmd, jobserverCmd, commandCmd, messageExportCmd) } var rootCmd = &cobra.Command{ diff --git a/cmd/platform/message_export.go b/cmd/platform/message_export.go new file mode 100644 index 000000000..fb1f4073b --- /dev/null +++ b/cmd/platform/message_export.go @@ -0,0 +1,79 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package main + +import ( + "errors" + + "context" + + "time" + + "github.com/mattermost/mattermost-server/model" + "github.com/spf13/cobra" +) + +var messageExportCmd = &cobra.Command{ + Use: "export", + Short: "Export data from Mattermost", + Long: "Export data from Mattermost in a format suitable for import into a third-party application", + Example: "export --format=actiance --exportFrom=12345", + RunE: messageExportCmdF, +} + +func init() { + messageExportCmd.Flags().String("format", "actiance", "The format to export data in") + messageExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") + messageExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.") +} + +func messageExportCmdF(cmd *cobra.Command, args []string) error { + a, err := initDBCommandContextCobra(cmd) + if err != nil { + return err + } + + if !*a.Config().MessageExportSettings.EnableExport { + return errors.New("ERROR: The message export feature is not enabled") + } + + // for now, format is hard-coded to actiance. In time, we'll have to support other formats and inject them into job data + if format, err := cmd.Flags().GetString("format"); err != nil { + return errors.New("format flag error") + } else if format != "actiance" { + return errors.New("unsupported export format") + } + + startTime, err := cmd.Flags().GetInt64("exportFrom") + if err != nil { + return errors.New("exportFrom flag error") + } else if startTime < 0 { + return errors.New("exportFrom must be a positive integer") + } + + timeoutSeconds, err := cmd.Flags().GetInt("timeoutSeconds") + if err != nil { + return errors.New("timeoutSeconds error") + } else if timeoutSeconds < 0 { + return errors.New("timeoutSeconds must be a positive integer") + } + + if messageExportI := a.MessageExport; messageExportI != nil { + ctx := context.Background() + if timeoutSeconds > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Second*time.Duration(timeoutSeconds)) + defer cancel() + } + + job, err := messageExportI.StartSynchronizeJob(ctx, startTime) + if err != nil || job.Status == model.JOB_STATUS_ERROR || job.Status == model.JOB_STATUS_CANCELED { + CommandPrintErrorln("ERROR: Message export job failed. Please check the server logs") + } else { + CommandPrettyPrintln("SUCCESS: Message export job complete") + } + } + + return nil +} diff --git a/cmd/platform/message_export_test.go b/cmd/platform/message_export_test.go new file mode 100644 index 000000000..211c1ca3c --- /dev/null +++ b/cmd/platform/message_export_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package main + +import ( + "testing" + + "io/ioutil" + "os" + "path/filepath" + + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils" + "github.com/stretchr/testify/require" +) + +// There are no tests that actually run the Message Export job, because it can take a long time to complete depending +// on the size of the database that the config is pointing to. As such, these tests just ensure that the CLI command +// fails fast if invalid flags are supplied + +func TestMessageExportNotEnabled(t *testing.T) { + configPath := writeTempConfig(t, false) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because the feature isn't enabled + require.Error(t, runCommand(t, "--config", configPath, "export")) +} + +func TestMessageExportInvalidFormat(t *testing.T) { + configPath := writeTempConfig(t, true) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because format isn't supported + require.Error(t, runCommand(t, "--config", configPath, "--format", "not_actiance", "export")) +} + +func TestMessageExportNegativeExportFrom(t *testing.T) { + configPath := writeTempConfig(t, true) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because export from must be a valid timestamp + require.Error(t, runCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export")) +} + +func TestMessageExportNegativeTimeoutSeconds(t *testing.T) { + configPath := writeTempConfig(t, true) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because timeout seconds must be a positive int + require.Error(t, runCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export")) +} + +func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string { + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + + utils.TranslationsPreInit() + config := utils.LoadGlobalConfig("config.json") + config.MessageExportSettings.EnableExport = model.NewBool(isMessageExportEnabled) + configPath := filepath.Join(dir, "foo.json") + require.NoError(t, ioutil.WriteFile(configPath, []byte(config.ToJson()), 0600)) + + return configPath +} -- cgit v1.2.3-1-g7c22