From ef597f6fe45cae8b3ba405ff89f5b20bfbf349e5 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 7 Dec 2017 15:56:14 -0500 Subject: PLT-8314: Test Message Export Against S3 Bucket (#7957) * Removed export directory config setting, in favour of hard-coding it to an 'export' directory under the local file directory. Improved the local file backend copy implementation to implicitly create the destination directory if it's missing * Fixed the tests --- model/config.go | 21 ++++----------------- model/config_test.go | 32 -------------------------------- utils/file.go | 3 +++ utils/file_backend_test.go | 26 +++++++++++++++++++++++--- 4 files changed, 30 insertions(+), 52 deletions(-) diff --git a/model/config.go b/model/config.go index ac1d034d4..544f9b33f 100644 --- a/model/config.go +++ b/model/config.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "net/url" - "path/filepath" "strings" "time" ) @@ -25,6 +24,10 @@ const ( DATABASE_DRIVER_MYSQL = "mysql" DATABASE_DRIVER_POSTGRES = "postgres" + MINIO_ACCESS_KEY = "minioaccesskey" + MINIO_SECRET_KEY = "miniosecretkey" + MINIO_BUCKET = "mattermost-test" + PASSWORD_MAXIMUM_LENGTH = 64 PASSWORD_MINIMUM_LENGTH = 5 @@ -1523,7 +1526,6 @@ type MessageExportSettings struct { EnableExport *bool DailyRunTime *string ExportFromTimestamp *int64 - FileLocation *string BatchSize *int } @@ -1532,10 +1534,6 @@ func (s *MessageExportSettings) SetDefaults() { s.EnableExport = NewBool(false) } - if s.FileLocation == nil { - s.FileLocation = NewString("export") - } - if s.DailyRunTime == nil { s.DailyRunTime = NewString("01:00") } @@ -2064,19 +2062,8 @@ func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError { return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, "", http.StatusBadRequest) } else if _, err := time.Parse("15:04", *mes.DailyRunTime); err != nil { return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest) - } else if mes.FileLocation == nil { - return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.app_error", nil, "", http.StatusBadRequest) } else if mes.BatchSize == nil || *mes.BatchSize < 0 { return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest) - } else if *fs.DriverName != IMAGE_DRIVER_LOCAL { - if absFileDir, err := filepath.Abs(fs.Directory); err != nil { - return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest) - } else if absMessageExportDir, err := filepath.Abs(*mes.FileLocation); err != nil { - return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest) - } else if !strings.HasPrefix(absMessageExportDir, absFileDir) { - // configured export directory must be relative to data directory - return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, "", http.StatusBadRequest) - } } } return nil diff --git a/model/config_test.go b/model/config_test.go index 217751252..ceede6be4 100644 --- a/model/config_test.go +++ b/model/config_test.go @@ -6,8 +6,6 @@ package model import ( "testing" - "os" - "github.com/stretchr/testify/require" ) @@ -100,34 +98,12 @@ func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) { EnableExport: NewBool(true), ExportFromTimestamp: NewInt64(0), DailyRunTime: NewString("15:04"), - FileLocation: NewString("foo"), } // should fail fast because batch size isn't set require.Error(t, mes.isValid(*fs)) } -func TestMessageExportSettingsIsValidFileLocationInvalid(t *testing.T) { - fs := &FileSettings{} - mes := &MessageExportSettings{ - EnableExport: NewBool(true), - ExportFromTimestamp: NewInt64(0), - DailyRunTime: NewString("15:04"), - BatchSize: NewInt(100), - } - - // should fail fast because FileLocation isn't set - require.Error(t, mes.isValid(*fs)) - - // if using the local file driver, there are more rules for FileLocation - fs.DriverName = NewString(IMAGE_DRIVER_LOCAL) - fs.Directory, _ = os.Getwd() - mes.FileLocation = NewString("") - - // should fail fast because file location is not relative to basepath - require.Error(t, mes.isValid(*fs)) -} - func TestMessageExportSettingsIsValid(t *testing.T) { fs := &FileSettings{ DriverName: NewString("foo"), // bypass file location check @@ -136,7 +112,6 @@ func TestMessageExportSettingsIsValid(t *testing.T) { EnableExport: NewBool(true), ExportFromTimestamp: NewInt64(0), DailyRunTime: NewString("15:04"), - FileLocation: NewString("foo"), BatchSize: NewInt(100), } @@ -149,7 +124,6 @@ func TestMessageExportSetDefaults(t *testing.T) { mes.SetDefaults() require.False(t, *mes.EnableExport) - require.Equal(t, "export", *mes.FileLocation) require.Equal(t, "01:00", *mes.DailyRunTime) require.Equal(t, int64(0), *mes.ExportFromTimestamp) require.Equal(t, 10000, *mes.BatchSize) @@ -162,7 +136,6 @@ func TestMessageExportSetDefaultsExportEnabledExportFromTimestampNil(t *testing. mes.SetDefaults() require.True(t, *mes.EnableExport) - require.Equal(t, "export", *mes.FileLocation) require.Equal(t, "01:00", *mes.DailyRunTime) require.NotEqual(t, int64(0), *mes.ExportFromTimestamp) require.True(t, *mes.ExportFromTimestamp <= GetMillis()) @@ -177,7 +150,6 @@ func TestMessageExportSetDefaultsExportEnabledExportFromTimestampZero(t *testing mes.SetDefaults() require.True(t, *mes.EnableExport) - require.Equal(t, "export", *mes.FileLocation) require.Equal(t, "01:00", *mes.DailyRunTime) require.NotEqual(t, int64(0), *mes.ExportFromTimestamp) require.True(t, *mes.ExportFromTimestamp <= GetMillis()) @@ -192,7 +164,6 @@ func TestMessageExportSetDefaultsExportEnabledExportFromTimestampNonZero(t *test mes.SetDefaults() require.True(t, *mes.EnableExport) - require.Equal(t, "export", *mes.FileLocation) require.Equal(t, "01:00", *mes.DailyRunTime) require.Equal(t, int64(12345), *mes.ExportFromTimestamp) require.Equal(t, 10000, *mes.BatchSize) @@ -205,7 +176,6 @@ func TestMessageExportSetDefaultsExportDisabledExportFromTimestampNil(t *testing mes.SetDefaults() require.False(t, *mes.EnableExport) - require.Equal(t, "export", *mes.FileLocation) require.Equal(t, "01:00", *mes.DailyRunTime) require.Equal(t, int64(0), *mes.ExportFromTimestamp) require.Equal(t, 10000, *mes.BatchSize) @@ -219,7 +189,6 @@ func TestMessageExportSetDefaultsExportDisabledExportFromTimestampZero(t *testin mes.SetDefaults() require.False(t, *mes.EnableExport) - require.Equal(t, "export", *mes.FileLocation) require.Equal(t, "01:00", *mes.DailyRunTime) require.Equal(t, int64(0), *mes.ExportFromTimestamp) require.Equal(t, 10000, *mes.BatchSize) @@ -233,7 +202,6 @@ func TestMessageExportSetDefaultsExportDisabledExportFromTimestampNonZero(t *tes mes.SetDefaults() require.False(t, *mes.EnableExport) - require.Equal(t, "export", *mes.FileLocation) require.Equal(t, "01:00", *mes.DailyRunTime) require.Equal(t, int64(0), *mes.ExportFromTimestamp) require.Equal(t, 10000, *mes.BatchSize) diff --git a/utils/file.go b/utils/file.go index 13b25bdab..923c559cc 100644 --- a/utils/file.go +++ b/utils/file.go @@ -21,6 +21,9 @@ func CopyFile(src, dst string) (err error) { } defer in.Close() + if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return + } out, err := os.Create(dst) if err != nil { return diff --git a/utils/file_backend_test.go b/utils/file_backend_test.go index 098f86bbd..76cd1f4a8 100644 --- a/utils/file_backend_test.go +++ b/utils/file_backend_test.go @@ -51,9 +51,9 @@ func TestS3FileBackendTestSuite(t *testing.T) { suite.Run(t, &FileBackendTestSuite{ settings: model.FileSettings{ DriverName: model.NewString(model.IMAGE_DRIVER_S3), - AmazonS3AccessKeyId: "minioaccesskey", - AmazonS3SecretAccessKey: "miniosecretkey", - AmazonS3Bucket: "mattermost-test", + AmazonS3AccessKeyId: model.MINIO_ACCESS_KEY, + AmazonS3SecretAccessKey: model.MINIO_SECRET_KEY, + AmazonS3Bucket: model.MINIO_BUCKET, AmazonS3Endpoint: s3Endpoint, AmazonS3SSL: model.NewBool(false), }, @@ -106,6 +106,26 @@ func (s *FileBackendTestSuite) TestCopyFile() { s.Nil(err) } +func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/newdirectory/" + model.NewId() + + err := s.backend.WriteFile(b, path1) + s.Nil(err) + defer s.backend.RemoveFile(path1) + + err = s.backend.CopyFile(path1, path2) + s.Nil(err) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Nil(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + func (s *FileBackendTestSuite) TestMoveFile() { b := []byte("test") path1 := "tests/" + model.NewId() -- cgit v1.2.3-1-g7c22