From a08df883b4ddb514d53b518f41431ce7efb50d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Thu, 20 Sep 2018 19:07:03 +0200 Subject: Move file backend to its own service (#9435) * Move file backend to its own service * Moving utils/inbucket to mailservice package --- services/filesstore/filesstore.go | 48 +++++ services/filesstore/filesstore_test.go | 288 ++++++++++++++++++++++++++++++ services/filesstore/localstore.go | 131 ++++++++++++++ services/filesstore/mocks/FileBackend.go | 215 ++++++++++++++++++++++ services/filesstore/s3store.go | 294 +++++++++++++++++++++++++++++++ services/filesstore/s3store_test.go | 32 ++++ 6 files changed, 1008 insertions(+) create mode 100644 services/filesstore/filesstore.go create mode 100644 services/filesstore/filesstore_test.go create mode 100644 services/filesstore/localstore.go create mode 100644 services/filesstore/mocks/FileBackend.go create mode 100644 services/filesstore/s3store.go create mode 100644 services/filesstore/s3store_test.go (limited to 'services/filesstore') diff --git a/services/filesstore/filesstore.go b/services/filesstore/filesstore.go new file mode 100644 index 000000000..59b3121ff --- /dev/null +++ b/services/filesstore/filesstore.go @@ -0,0 +1,48 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "io" + "net/http" + + "github.com/mattermost/mattermost-server/model" +) + +type FileBackend interface { + TestConnection() *model.AppError + + Reader(path string) (io.ReadCloser, *model.AppError) + ReadFile(path string) ([]byte, *model.AppError) + FileExists(path string) (bool, *model.AppError) + CopyFile(oldPath, newPath string) *model.AppError + MoveFile(oldPath, newPath string) *model.AppError + WriteFile(fr io.Reader, path string) (int64, *model.AppError) + RemoveFile(path string) *model.AppError + + ListDirectory(path string) (*[]string, *model.AppError) + RemoveDirectory(path string) *model.AppError +} + +func NewFileBackend(settings *model.FileSettings, enableComplianceFeatures bool) (FileBackend, *model.AppError) { + switch *settings.DriverName { + case model.IMAGE_DRIVER_S3: + return &S3FileBackend{ + endpoint: settings.AmazonS3Endpoint, + accessKey: settings.AmazonS3AccessKeyId, + secretKey: settings.AmazonS3SecretAccessKey, + secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, + signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, + region: settings.AmazonS3Region, + bucket: settings.AmazonS3Bucket, + encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && enableComplianceFeatures, + trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, + }, nil + case model.IMAGE_DRIVER_LOCAL: + return &LocalFileBackend{ + directory: settings.Directory, + }, nil + } + return nil, model.NewAppError("NewFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError) +} diff --git a/services/filesstore/filesstore_test.go b/services/filesstore/filesstore_test.go new file mode 100644 index 000000000..270f6e9f9 --- /dev/null +++ b/services/filesstore/filesstore_test.go @@ -0,0 +1,288 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils" +) + +type FileBackendTestSuite struct { + suite.Suite + + settings model.FileSettings + backend FileBackend +} + +func TestLocalFileBackendTestSuite(t *testing.T) { + // Setup a global logger to catch tests logging outside of app context + // The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen. + mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{ + EnableConsole: true, + ConsoleJson: true, + ConsoleLevel: "error", + EnableFile: false, + })) + + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(dir) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL), + Directory: dir, + }, + }) +} + +func TestS3FileBackendTestSuite(t *testing.T) { + runBackendTest(t, false) +} + +func TestS3FileBackendTestSuiteWithEncryption(t *testing.T) { + runBackendTest(t, true) +} + +func runBackendTest(t *testing.T, encrypt bool) { + s3Host := os.Getenv("CI_HOST") + if s3Host == "" { + s3Host = "dockerhost" + } + + s3Port := os.Getenv("CI_MINIO_PORT") + if s3Port == "" { + s3Port = "9001" + } + + s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_S3), + AmazonS3AccessKeyId: model.MINIO_ACCESS_KEY, + AmazonS3SecretAccessKey: model.MINIO_SECRET_KEY, + AmazonS3Bucket: model.MINIO_BUCKET, + AmazonS3Endpoint: s3Endpoint, + AmazonS3SSL: model.NewBool(false), + AmazonS3SSE: model.NewBool(encrypt), + }, + }) +} + +func (s *FileBackendTestSuite) SetupTest() { + utils.TranslationsPreInit() + + backend, err := NewFileBackend(&s.settings, true) + require.Nil(s.T(), err) + s.backend = backend +} + +func (s *FileBackendTestSuite) TestConnection() { + s.Nil(s.backend.TestConnection()) +} + +func (s *FileBackendTestSuite) TestReadWriteFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path) + + read, err := s.backend.ReadFile(path) + s.Nil(err) + + readString := string(read) + s.EqualValues(readString, "test") +} + +func (s *FileBackendTestSuite) TestReadWriteFileImage() { + b := []byte("testimage") + path := "tests/" + model.NewId() + ".png" + + written, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path) + + read, err := s.backend.ReadFile(path) + s.Nil(err) + + readString := string(read) + s.EqualValues(readString, "testimage") +} + +func (s *FileBackendTestSuite) TestFileExists() { + b := []byte("testimage") + path := "tests/" + model.NewId() + ".png" + + _, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + defer s.backend.RemoveFile(path) + + res, err := s.backend.FileExists(path) + s.Nil(err) + s.True(res) + + res, err = s.backend.FileExists("tests/idontexist.png") + s.Nil(err) + s.False(res) +} + +func (s *FileBackendTestSuite) TestCopyFile() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + err = s.backend.CopyFile(path1, path2) + s.Nil(err) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Nil(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/newdirectory/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + err = s.backend.CopyFile(path1, path2) + s.Nil(err) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Nil(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestMoveFile() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + s.Nil(s.backend.MoveFile(path1, path2)) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Error(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestRemoveFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + s.Nil(s.backend.RemoveFile(path)) + + _, err = s.backend.ReadFile(path) + s.Error(err) + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/foo") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/asdf") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + s.Nil(s.backend.RemoveDirectory("tests2")) +} + +func (s *FileBackendTestSuite) TestListDirectory() { + b := []byte("test") + path1 := "19700101/" + model.NewId() + path2 := "19800101/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + written, err = s.backend.WriteFile(bytes.NewReader(b), path2) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path2) + + paths, err := s.backend.ListDirectory("") + s.Nil(err) + + found1 := false + found2 := false + for _, path := range *paths { + if path == "19700101" { + found1 = true + } else if path == "19800101" { + found2 = true + } + } + s.True(found1) + s.True(found2) +} + +func (s *FileBackendTestSuite) TestRemoveDirectory() { + b := []byte("test") + + written, err := s.backend.WriteFile(bytes.NewReader(b), "tests2/foo") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/aaa") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + s.Nil(s.backend.RemoveDirectory("tests2")) + + _, err = s.backend.ReadFile("tests2/foo") + s.Error(err) + _, err = s.backend.ReadFile("tests2/bar") + s.Error(err) + _, err = s.backend.ReadFile("tests2/asdf") + s.Error(err) +} diff --git a/services/filesstore/localstore.go b/services/filesstore/localstore.go new file mode 100644 index 000000000..8d79a982e --- /dev/null +++ b/services/filesstore/localstore.go @@ -0,0 +1,131 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils" +) + +const ( + TEST_FILE_PATH = "/testfile" +) + +type LocalFileBackend struct { + directory string +} + +func (b *LocalFileBackend) TestConnection() *model.AppError { + f := bytes.NewReader([]byte("testingwrite")) + if _, err := writeFileLocally(f, filepath.Join(b.directory, TEST_FILE_PATH)); err != nil { + return model.NewAppError("TestFileConnection", "api.file.test_connection.local.connection.app_error", nil, err.Error(), http.StatusInternalServerError) + } + os.Remove(filepath.Join(b.directory, TEST_FILE_PATH)) + mlog.Info("Able to write files to local storage.") + return nil +} + +func (b *LocalFileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + if f, err := os.Open(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) { + if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *LocalFileBackend) FileExists(path string) (bool, *model.AppError) { + _, err := os.Stat(filepath.Join(b.directory, path)) + + if os.IsNotExist(err) { + return false, nil + } else if err == nil { + return true, nil + } + + return false, model.NewAppError("ReadFile", "api.file.file_exists.exists_local.app_error", nil, err.Error(), http.StatusInternalServerError) +} + +func (b *LocalFileBackend) CopyFile(oldPath, newPath string) *model.AppError { + if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + return model.NewAppError("copyFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { + return writeFileLocally(fr, filepath.Join(b.directory, path)) +} + +func writeFileLocally(fr io.Reader, path string) (int64, *model.AppError) { + if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { + directory, _ := filepath.Abs(filepath.Dir(path)) + return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) + } + fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) + } + defer fw.Close() + written, err := io.Copy(fw, fr) + if err != nil { + return written, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return written, nil +} + +func (b *LocalFileBackend) RemoveFile(path string) *model.AppError { + if err := os.Remove(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + if fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + for _, fileInfo := range fileInfos { + if fileInfo.IsDir() { + paths = append(paths, filepath.Join(path, fileInfo.Name())) + } + } + } + return &paths, nil +} + +func (b *LocalFileBackend) RemoveDirectory(path string) *model.AppError { + if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} diff --git a/services/filesstore/mocks/FileBackend.go b/services/filesstore/mocks/FileBackend.go new file mode 100644 index 000000000..5d75ae5d1 --- /dev/null +++ b/services/filesstore/mocks/FileBackend.go @@ -0,0 +1,215 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +// Regenerate this file using `make files-store-mocks`. + +package mocks + +import io "io" +import mock "github.com/stretchr/testify/mock" +import model "github.com/mattermost/mattermost-server/model" + +// FileBackend is an autogenerated mock type for the FileBackend type +type FileBackend struct { + mock.Mock +} + +// CopyFile provides a mock function with given fields: oldPath, newPath +func (_m *FileBackend) CopyFile(oldPath string, newPath string) *model.AppError { + ret := _m.Called(oldPath, newPath) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string, string) *model.AppError); ok { + r0 = rf(oldPath, newPath) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// FileExists provides a mock function with given fields: path +func (_m *FileBackend) FileExists(path string) (bool, *model.AppError) { + ret := _m.Called(path) + + var r0 bool + if rf, ok := ret.Get(0).(func(string) bool); ok { + r0 = rf(path) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// ListDirectory provides a mock function with given fields: path +func (_m *FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + ret := _m.Called(path) + + var r0 *[]string + if rf, ok := ret.Get(0).(func(string) *[]string); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*[]string) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// MoveFile provides a mock function with given fields: oldPath, newPath +func (_m *FileBackend) MoveFile(oldPath string, newPath string) *model.AppError { + ret := _m.Called(oldPath, newPath) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string, string) *model.AppError); ok { + r0 = rf(oldPath, newPath) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// ReadFile provides a mock function with given fields: path +func (_m *FileBackend) ReadFile(path string) ([]byte, *model.AppError) { + ret := _m.Called(path) + + var r0 []byte + if rf, ok := ret.Get(0).(func(string) []byte); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// Reader provides a mock function with given fields: path +func (_m *FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + ret := _m.Called(path) + + var r0 io.ReadCloser + if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.ReadCloser) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// RemoveDirectory provides a mock function with given fields: path +func (_m *FileBackend) RemoveDirectory(path string) *model.AppError { + ret := _m.Called(path) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string) *model.AppError); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// RemoveFile provides a mock function with given fields: path +func (_m *FileBackend) RemoveFile(path string) *model.AppError { + ret := _m.Called(path) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string) *model.AppError); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// TestConnection provides a mock function with given fields: +func (_m *FileBackend) TestConnection() *model.AppError { + ret := _m.Called() + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func() *model.AppError); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// WriteFile provides a mock function with given fields: fr, path +func (_m *FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { + ret := _m.Called(fr, path) + + var r0 int64 + if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok { + r0 = rf(fr, path) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(io.Reader, string) *model.AppError); ok { + r1 = rf(fr, path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} diff --git a/services/filesstore/s3store.go b/services/filesstore/s3store.go new file mode 100644 index 000000000..0a0f057ea --- /dev/null +++ b/services/filesstore/s3store.go @@ -0,0 +1,294 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + s3 "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/credentials" + "github.com/minio/minio-go/pkg/encrypt" + + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" +) + +type S3FileBackend struct { + endpoint string + accessKey string + secretKey string + secure bool + signV2 bool + region string + bucket string + encrypt bool + trace bool +} + +// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. +// If signV2 input is false, function always returns signature v4. +// +// Additionally this function also takes a user defined region, if set +// disables automatic region lookup. +func (b *S3FileBackend) s3New() (*s3.Client, error) { + var creds *credentials.Credentials + + if b.accessKey == "" && b.secretKey == "" { + creds = credentials.NewIAM("") + } else if b.signV2 { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) + } else { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) + } + + s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) + if err != nil { + return nil, err + } + + if b.trace { + s3Clnt.TraceOn(os.Stdout) + } + + return s3Clnt, nil +} + +func (b *S3FileBackend) TestConnection() *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.connection.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + exists, err := s3Clnt.BucketExists(b.bucket) + if err != nil { + return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucket_exists.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if !exists { + mlog.Warn("Bucket specified does not exist. Attempting to create...") + err := s3Clnt.MakeBucket(b.bucket, b.region) + if err != nil { + mlog.Error("Unable to create bucket.") + return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucked_create.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + mlog.Info("Connection to S3 or minio is good. Bucket exists.") + return nil +} + +// Caller must close the first return value +func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) + if err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return minioObject, nil +} + +func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + defer minioObject.Close() + if f, err := ioutil.ReadAll(minioObject); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) { + s3Clnt, err := b.s3New() + + if err != nil { + return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + _, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{}) + + if err == nil { + return true, nil + } + + if err.(s3.ErrorResponse).Code == "NoSuchKey" { + return false, nil + } + + return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) +} + +func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + source := s3.NewSourceInfo(b.bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) + if err != nil { + return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + source := s3.NewSourceInfo(b.bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + var contentType string + if ext := filepath.Ext(path); model.IsFileExtImage(ext) { + contentType = model.GetImageMimeType(ext) + } else { + contentType = "binary/octet-stream" + } + + options := s3PutOptions(b.encrypt, contentType) + var buf bytes.Buffer + _, err = buf.ReadFrom(fr) + if err != nil { + return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + written, err := s3Clnt.PutObject(b.bucket, path, &buf, int64(buf.Len()), options) + if err != nil { + return written, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return written, nil +} + +func (b *S3FileBackend) RemoveFile(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { + out := make(chan string, 1) + + go func() { + defer close(out) + + for { + info, done := <-in + + if !done { + break + } + + out <- info.Key + } + }() + + return out +} + +func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + defer close(doneCh) + + for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { + if object.Err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) + } + paths = append(paths, strings.Trim(object.Key, "/")) + } + + return &paths, nil +} + +func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { + if err.Err != nil { + doneCh <- struct{}{} + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) + } + } + + close(doneCh) + return nil +} + +func s3PutOptions(encrypted bool, contentType string) s3.PutObjectOptions { + options := s3.PutObjectOptions{} + if encrypted { + options.ServerSideEncryption = encrypt.NewSSE() + } + options.ContentType = contentType + + return options +} + +func CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError { + if len(settings.AmazonS3Bucket) == 0 { + return model.NewAppError("S3File", "api.admin.test_s3.missing_s3_bucket", nil, "", http.StatusBadRequest) + } + + // if S3 endpoint is not set call the set defaults to set that + if len(settings.AmazonS3Endpoint) == 0 { + settings.SetDefaults() + } + + return nil +} diff --git a/services/filesstore/s3store_test.go b/services/filesstore/s3store_test.go new file mode 100644 index 000000000..a958a1f3f --- /dev/null +++ b/services/filesstore/s3store_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2018-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "testing" + + "github.com/mattermost/mattermost-server/model" +) + +func TestCheckMandatoryS3Fields(t *testing.T) { + cfg := model.FileSettings{} + + err := CheckMandatoryS3Fields(&cfg) + if err == nil || err.Message != "api.admin.test_s3.missing_s3_bucket" { + t.Fatal("should've failed with missing s3 bucket") + } + + cfg.AmazonS3Bucket = "test-mm" + err = CheckMandatoryS3Fields(&cfg) + if err != nil { + t.Fatal("should've not failed") + } + + cfg.AmazonS3Endpoint = "" + err = CheckMandatoryS3Fields(&cfg) + if err != nil || cfg.AmazonS3Endpoint != "s3.amazonaws.com" { + t.Fatal("should've not failed because it should set the endpoint to the default") + } + +} -- cgit v1.2.3-1-g7c22