From ab99f0656fabed8a62a8c6340be7d538cc7bf8d9 Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Mon, 17 Sep 2018 15:51:26 +0100 Subject: MM-11781: Basic Data Export Command Line. (#9296) * MM-11781: Basic Data Export Command Line. * ChannelStore new unit tests. * TeamStore new unit tests. * Unit test for new UserStore function. * Unit tests for post store new methods. * Review fixes. * Fix duplicate command name. --- cmd/mattermost/commands/export.go | 188 +++++++++++++++++++++++++ cmd/mattermost/commands/export_test.go | 66 +++++++++ cmd/mattermost/commands/message_export.go | 142 ------------------- cmd/mattermost/commands/message_export_test.go | 66 --------- 4 files changed, 254 insertions(+), 208 deletions(-) create mode 100644 cmd/mattermost/commands/export.go create mode 100644 cmd/mattermost/commands/export_test.go delete mode 100644 cmd/mattermost/commands/message_export.go delete mode 100644 cmd/mattermost/commands/message_export_test.go (limited to 'cmd') diff --git a/cmd/mattermost/commands/export.go b/cmd/mattermost/commands/export.go new file mode 100644 index 000000000..bd311e154 --- /dev/null +++ b/cmd/mattermost/commands/export.go @@ -0,0 +1,188 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package commands + +import ( + "errors" + "os" + + "context" + + "time" + + "github.com/mattermost/mattermost-server/model" + "github.com/spf13/cobra" +) + +var ExportCmd = &cobra.Command{ + Use: "export", + Short: "Export data from Mattermost", + Long: "Export data from Mattermost in a format suitable for import into a third-party application or another Mattermost instance", +} + +var ScheduleExportCmd = &cobra.Command{ + Use: "schedule", + Short: "Schedule an export data job in Mattermost", + Long: "Schedule an export data job in Mattermost (this will run asynchronously via a background worker)", + Example: "export schedule --format=actiance --exportFrom=12345 --timeoutSeconds=12345", + RunE: scheduleExportCmdF, +} + +var CsvExportCmd = &cobra.Command{ + Use: "csv", + Short: "Export data from Mattermost in CSV format", + Long: "Export data from Mattermost in CSV format", + Example: "export csv --exportFrom=12345", + RunE: buildExportCmdF("csv"), +} + +var ActianceExportCmd = &cobra.Command{ + Use: "actiance", + Short: "Export data from Mattermost in Actiance format", + Long: "Export data from Mattermost in Actiance format", + Example: "export actiance --exportFrom=12345", + RunE: buildExportCmdF("actiance"), +} + +var BulkExportCmd = &cobra.Command{ + Use: "bulk [file]", + Short: "Export bulk data.", + Long: "Export data to a file compatible with the Mattermost Bulk Import format.", + Example: " export bulk bulk_data.json", + RunE: bulkExportCmdF, +} + +func init() { + ScheduleExportCmd.Flags().String("format", "actiance", "The format to export data") + ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") + ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.") + + CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") + + ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") + + BulkExportCmd.Flags().Bool("all-teams", false, "Export all teams from the server.") + + ExportCmd.AddCommand(ScheduleExportCmd) + ExportCmd.AddCommand(CsvExportCmd) + ExportCmd.AddCommand(ActianceExportCmd) + ExportCmd.AddCommand(BulkExportCmd) + + RootCmd.AddCommand(ExportCmd) +} + +func scheduleExportCmdF(command *cobra.Command, args []string) error { + a, err := InitDBCommandContextCobra(command) + if err != nil { + return err + } + defer a.Shutdown() + + if !*a.Config().MessageExportSettings.EnableExport { + return errors.New("ERROR: The message export feature is not enabled") + } + + // for now, format is hard-coded to actiance. In time, we'll have to support other formats and inject them into job data + format, err := command.Flags().GetString("format") + if err != nil { + return errors.New("format flag error") + } + if format != "actiance" { + return errors.New("unsupported export format") + } + + startTime, err := command.Flags().GetInt64("exportFrom") + if err != nil { + return errors.New("exportFrom flag error") + } + if startTime < 0 { + return errors.New("exportFrom must be a positive integer") + } + + timeoutSeconds, err := command.Flags().GetInt("timeoutSeconds") + if err != nil { + return errors.New("timeoutSeconds error") + } + if timeoutSeconds < 0 { + return errors.New("timeoutSeconds must be a positive integer") + } + + if messageExportI := a.MessageExport; messageExportI != nil { + ctx := context.Background() + if timeoutSeconds > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Second*time.Duration(timeoutSeconds)) + defer cancel() + } + + job, err := messageExportI.StartSynchronizeJob(ctx, startTime) + if err != nil || job.Status == model.JOB_STATUS_ERROR || job.Status == model.JOB_STATUS_CANCELED { + CommandPrintErrorln("ERROR: Message export job failed. Please check the server logs") + } else { + CommandPrettyPrintln("SUCCESS: Message export job complete") + } + } + + return nil +} + +func buildExportCmdF(format string) func(command *cobra.Command, args []string) error { + return func(command *cobra.Command, args []string) error { + a, err := InitDBCommandContextCobra(command) + if err != nil { + return err + } + defer a.Shutdown() + + startTime, err := command.Flags().GetInt64("exportFrom") + if err != nil { + return errors.New("exportFrom flag error") + } + if startTime < 0 { + return errors.New("exportFrom must be a positive integer") + } + + if a.MessageExport == nil { + CommandPrettyPrintln("MessageExport feature not available") + } + + err2 := a.MessageExport.RunExport(format, startTime) + if err2 != nil { + return err2 + } + CommandPrettyPrintln("SUCCESS: Your data was exported.") + + return nil + } +} + +func bulkExportCmdF(command *cobra.Command, args []string) error { + a, err := InitDBCommandContextCobra(command) + if err != nil { + return err + } + defer a.Shutdown() + + allTeams, err := command.Flags().GetBool("all-teams") + if err != nil { + return errors.New("Apply flag error") + } + + if !allTeams { + return errors.New("Nothing to export. Please specify the --all-teams flag to export all teams.") + } + + fileWriter, err := os.Create(args[0]) + if err != nil { + return err + } + defer fileWriter.Close() + + if err := a.BulkExport(fileWriter); err != nil { + CommandPrettyPrintln(err.Error()) + return err + } + + return nil +} diff --git a/cmd/mattermost/commands/export_test.go b/cmd/mattermost/commands/export_test.go new file mode 100644 index 000000000..89ef45a6a --- /dev/null +++ b/cmd/mattermost/commands/export_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package commands + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils" +) + +// There are no tests that actually run the Message Export job, because it can take a long time to complete depending +// on the size of the database that the config is pointing to. As such, these tests just ensure that the CLI command +// fails fast if invalid flags are supplied + +func TestMessageExportNotEnabled(t *testing.T) { + configPath := writeTempConfig(t, false) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because the feature isn't enabled + require.Error(t, RunCommand(t, "--config", configPath, "export", "schedule")) +} + +func TestMessageExportInvalidFormat(t *testing.T) { + configPath := writeTempConfig(t, true) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because format isn't supported + require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export", "schedule")) +} + +func TestMessageExportNegativeExportFrom(t *testing.T) { + configPath := writeTempConfig(t, true) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because export from must be a valid timestamp + require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export", "schedule")) +} + +func TestMessageExportNegativeTimeoutSeconds(t *testing.T) { + configPath := writeTempConfig(t, true) + defer os.RemoveAll(filepath.Dir(configPath)) + + // should fail fast because timeout seconds must be a positive int + require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule")) +} + +func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string { + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + + utils.TranslationsPreInit() + config, _, _, appErr := utils.LoadConfig("config.json") + require.Nil(t, appErr) + config.MessageExportSettings.EnableExport = model.NewBool(isMessageExportEnabled) + configPath := filepath.Join(dir, "foo.json") + require.NoError(t, ioutil.WriteFile(configPath, []byte(config.ToJson()), 0600)) + + return configPath +} diff --git a/cmd/mattermost/commands/message_export.go b/cmd/mattermost/commands/message_export.go deleted file mode 100644 index 953d4ccba..000000000 --- a/cmd/mattermost/commands/message_export.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package commands - -import ( - "errors" - - "context" - - "time" - - "github.com/mattermost/mattermost-server/model" - "github.com/spf13/cobra" -) - -var MessageExportCmd = &cobra.Command{ - Use: "export", - Short: "Export data from Mattermost", - Long: "Export data from Mattermost in a format suitable for import into a third-party application", -} - -var ScheduleExportCmd = &cobra.Command{ - Use: "schedule", - Short: "Schedule an export data job in Mattermost", - Long: "Schedule an export data job in Mattermost (this will run asynchronously via a background worker)", - Example: "export schedule --format=actiance --exportFrom=12345 --timeoutSeconds=12345", - RunE: scheduleExportCmdF, -} - -var CsvExportCmd = &cobra.Command{ - Use: "csv", - Short: "Export data from Mattermost in CSV format", - Long: "Export data from Mattermost in CSV format", - Example: "export csv --exportFrom=12345", - RunE: buildExportCmdF("csv"), -} - -var ActianceExportCmd = &cobra.Command{ - Use: "actiance", - Short: "Export data from Mattermost in Actiance format", - Long: "Export data from Mattermost in Actiance format", - Example: "export actiance --exportFrom=12345", - RunE: buildExportCmdF("actiance"), -} - -func init() { - ScheduleExportCmd.Flags().String("format", "actiance", "The format to export data") - ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") - ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.") - CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") - ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") - MessageExportCmd.AddCommand(ScheduleExportCmd) - MessageExportCmd.AddCommand(CsvExportCmd) - MessageExportCmd.AddCommand(ActianceExportCmd) - RootCmd.AddCommand(MessageExportCmd) -} - -func scheduleExportCmdF(command *cobra.Command, args []string) error { - a, err := InitDBCommandContextCobra(command) - if err != nil { - return err - } - defer a.Shutdown() - - if !*a.Config().MessageExportSettings.EnableExport { - return errors.New("ERROR: The message export feature is not enabled") - } - - // for now, format is hard-coded to actiance. In time, we'll have to support other formats and inject them into job data - format, err := command.Flags().GetString("format") - if err != nil { - return errors.New("format flag error") - } - if format != "actiance" { - return errors.New("unsupported export format") - } - - startTime, err := command.Flags().GetInt64("exportFrom") - if err != nil { - return errors.New("exportFrom flag error") - } - if startTime < 0 { - return errors.New("exportFrom must be a positive integer") - } - - timeoutSeconds, err := command.Flags().GetInt("timeoutSeconds") - if err != nil { - return errors.New("timeoutSeconds error") - } - if timeoutSeconds < 0 { - return errors.New("timeoutSeconds must be a positive integer") - } - - if messageExportI := a.MessageExport; messageExportI != nil { - ctx := context.Background() - if timeoutSeconds > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Second*time.Duration(timeoutSeconds)) - defer cancel() - } - - job, err := messageExportI.StartSynchronizeJob(ctx, startTime) - if err != nil || job.Status == model.JOB_STATUS_ERROR || job.Status == model.JOB_STATUS_CANCELED { - CommandPrintErrorln("ERROR: Message export job failed. Please check the server logs") - } else { - CommandPrettyPrintln("SUCCESS: Message export job complete") - } - } - - return nil -} - -func buildExportCmdF(format string) func(command *cobra.Command, args []string) error { - return func(command *cobra.Command, args []string) error { - a, err := InitDBCommandContextCobra(command) - if err != nil { - return err - } - defer a.Shutdown() - - startTime, err := command.Flags().GetInt64("exportFrom") - if err != nil { - return errors.New("exportFrom flag error") - } - if startTime < 0 { - return errors.New("exportFrom must be a positive integer") - } - - if a.MessageExport == nil { - CommandPrettyPrintln("MessageExport feature not available") - } - - err2 := a.MessageExport.RunExport(format, startTime) - if err2 != nil { - return err2 - } - CommandPrettyPrintln("SUCCESS: Your data was exported.") - - return nil - } -} diff --git a/cmd/mattermost/commands/message_export_test.go b/cmd/mattermost/commands/message_export_test.go deleted file mode 100644 index 89ef45a6a..000000000 --- a/cmd/mattermost/commands/message_export_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package commands - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" -) - -// There are no tests that actually run the Message Export job, because it can take a long time to complete depending -// on the size of the database that the config is pointing to. As such, these tests just ensure that the CLI command -// fails fast if invalid flags are supplied - -func TestMessageExportNotEnabled(t *testing.T) { - configPath := writeTempConfig(t, false) - defer os.RemoveAll(filepath.Dir(configPath)) - - // should fail fast because the feature isn't enabled - require.Error(t, RunCommand(t, "--config", configPath, "export", "schedule")) -} - -func TestMessageExportInvalidFormat(t *testing.T) { - configPath := writeTempConfig(t, true) - defer os.RemoveAll(filepath.Dir(configPath)) - - // should fail fast because format isn't supported - require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export", "schedule")) -} - -func TestMessageExportNegativeExportFrom(t *testing.T) { - configPath := writeTempConfig(t, true) - defer os.RemoveAll(filepath.Dir(configPath)) - - // should fail fast because export from must be a valid timestamp - require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export", "schedule")) -} - -func TestMessageExportNegativeTimeoutSeconds(t *testing.T) { - configPath := writeTempConfig(t, true) - defer os.RemoveAll(filepath.Dir(configPath)) - - // should fail fast because timeout seconds must be a positive int - require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule")) -} - -func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string { - dir, err := ioutil.TempDir("", "") - require.NoError(t, err) - - utils.TranslationsPreInit() - config, _, _, appErr := utils.LoadConfig("config.json") - require.Nil(t, appErr) - config.MessageExportSettings.EnableExport = model.NewBool(isMessageExportEnabled) - configPath := filepath.Join(dir, "foo.json") - require.NoError(t, ioutil.WriteFile(configPath, []byte(config.ToJson()), 0600)) - - return configPath -} -- cgit v1.2.3-1-g7c22