From a08df883b4ddb514d53b518f41431ce7efb50d8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Thu, 20 Sep 2018 19:07:03 +0200 Subject: Move file backend to its own service (#9435) * Move file backend to its own service * Moving utils/inbucket to mailservice package --- Makefile | 4 + api4/system.go | 6 +- api4/team_test.go | 13 +- api4/user_test.go | 10 +- app/admin.go | 3 +- app/email.go | 3 +- app/file.go | 5 +- cmd/mattermost/commands/server.go | 3 +- scripts/license-check.sh | 2 +- services/filesstore/filesstore.go | 48 +++++ services/filesstore/filesstore_test.go | 288 +++++++++++++++++++++++++++++ services/filesstore/localstore.go | 131 ++++++++++++++ services/filesstore/mocks/FileBackend.go | 215 ++++++++++++++++++++++ services/filesstore/s3store.go | 294 ++++++++++++++++++++++++++++++ services/filesstore/s3store_test.go | 32 ++++ services/mailservice/inbucket.go | 200 +++++++++++++++++++++ services/mailservice/mail.go | 299 +++++++++++++++++++++++++++++++ services/mailservice/mail_test.go | 290 ++++++++++++++++++++++++++++++ utils/file_backend.go | 48 ----- utils/file_backend_local.go | 130 -------------- utils/file_backend_s3.go | 294 ------------------------------ utils/file_backend_s3_test.go | 32 ---- utils/file_backend_test.go | 287 ----------------------------- utils/inbucket.go | 200 --------------------- utils/mail.go | 297 ------------------------------ utils/mail_test.go | 288 ----------------------------- 26 files changed, 1826 insertions(+), 1596 deletions(-) create mode 100644 services/filesstore/filesstore.go create mode 100644 services/filesstore/filesstore_test.go create mode 100644 services/filesstore/localstore.go create mode 100644 services/filesstore/mocks/FileBackend.go create mode 100644 services/filesstore/s3store.go create mode 100644 services/filesstore/s3store_test.go create mode 100644 services/mailservice/inbucket.go create mode 100644 services/mailservice/mail.go create mode 100644 services/mailservice/mail_test.go delete mode 100644 utils/file_backend.go delete mode 100644 utils/file_backend_local.go delete mode 100644 utils/file_backend_s3.go delete mode 100644 utils/file_backend_s3_test.go delete mode 100644 utils/file_backend_test.go delete mode 100644 utils/inbucket.go delete mode 100644 utils/mail.go delete mode 100644 utils/mail_test.go diff --git a/Makefile b/Makefile index 6ac85a785..62e9f2bd5 100644 --- a/Makefile +++ b/Makefile @@ -287,6 +287,10 @@ store-mocks: ## Creates mock files. go get -u github.com/vektra/mockery/... $(GOPATH)/bin/mockery -dir store -all -output store/storetest/mocks -note 'Regenerate this file using `make store-mocks`.' +filesstore-mocks: ## Creates mock files. + go get -u github.com/vektra/mockery/... + $(GOPATH)/bin/mockery -dir services/filesstore -all -output services/filesstore/mocks -note 'Regenerate this file using `make filesstore-mocks`.' + ldap-mocks: ## Creates mock files for ldap. go get -u github.com/vektra/mockery/... $(GOPATH)/bin/mockery -dir enterprise/ldap -all -output enterprise/ldap/mocks -note 'Regenerate this file using `make ldap-mocks`.' diff --git a/api4/system.go b/api4/system.go index 65d3b424b..2f6e53ab8 100644 --- a/api4/system.go +++ b/api4/system.go @@ -12,7 +12,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/services/filesstore" ) func (api *API) InitSystem() { @@ -427,7 +427,7 @@ func testS3(c *Context, w http.ResponseWriter, r *http.Request) { return } - err := utils.CheckMandatoryS3Fields(&cfg.FileSettings) + err := filesstore.CheckMandatoryS3Fields(&cfg.FileSettings) if err != nil { c.Err = err return @@ -438,7 +438,7 @@ func testS3(c *Context, w http.ResponseWriter, r *http.Request) { } license := c.App.License() - backend, appErr := utils.NewFileBackend(&cfg.FileSettings, license != nil && *license.Features.Compliance) + backend, appErr := filesstore.NewFileBackend(&cfg.FileSettings, license != nil && *license.Features.Compliance) if appErr == nil { appErr = backend.TestConnection() } diff --git a/api4/team_test.go b/api4/team_test.go index 547b8d4a9..8f1027cc5 100644 --- a/api4/team_test.go +++ b/api4/team_test.go @@ -15,6 +15,7 @@ import ( "github.com/mattermost/mattermost-server/app" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/mailservice" "github.com/mattermost/mattermost-server/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1894,8 +1895,8 @@ func TestInviteUsersToTeam(t *testing.T) { emailList := []string{user1, user2} //Delete all the messages before check the sample email - utils.DeleteMailBox(user1) - utils.DeleteMailBox(user2) + mailservice.DeleteMailBox(user1) + mailservice.DeleteMailBox(user2) enableEmailInvitations := *th.App.Config().ServiceSettings.EnableEmailInvitations restrictCreationToDomains := th.App.Config().TeamSettings.RestrictCreationToDomains @@ -1925,10 +1926,10 @@ func TestInviteUsersToTeam(t *testing.T) { //Check if the email was send to the rigth email address for _, email := range emailList { - var resultsMailbox utils.JSONMessageHeaderInbucket - err := utils.RetryInbucket(5, func() error { + var resultsMailbox mailservice.JSONMessageHeaderInbucket + err := mailservice.RetryInbucket(5, func() error { var err error - resultsMailbox, err = utils.GetMailBox(email) + resultsMailbox, err = mailservice.GetMailBox(email) return err }) if err != nil { @@ -1939,7 +1940,7 @@ func TestInviteUsersToTeam(t *testing.T) { if !strings.ContainsAny(resultsMailbox[len(resultsMailbox)-1].To[0], email) { t.Fatal("Wrong To recipient") } else { - if resultsEmail, err := utils.GetMessageFromMailbox(email, resultsMailbox[len(resultsMailbox)-1].ID); err == nil { + if resultsEmail, err := mailservice.GetMessageFromMailbox(email, resultsMailbox[len(resultsMailbox)-1].ID); err == nil { if resultsEmail.Subject != expectedSubject { t.Log(resultsEmail.Subject) t.Log(expectedSubject) diff --git a/api4/user_test.go b/api4/user_test.go index e624d747d..fd60a40ee 100644 --- a/api4/user_test.go +++ b/api4/user_test.go @@ -1919,7 +1919,7 @@ func TestUpdateUserPassword(t *testing.T) { Client.Logout() user := th.BasicUser // Delete all the messages before check the reset password - utils.DeleteMailBox(user.Email) + mailservice.DeleteMailBox(user.Email) success, resp := Client.SendPasswordResetEmail(user.Email) CheckNoError(t, resp) if !success { @@ -1934,10 +1934,10 @@ func TestUpdateUserPassword(t *testing.T) { t.Fatal("should have succeeded") } // Check if the email was send to the right email address and the recovery key match - var resultsMailbox utils.JSONMessageHeaderInbucket - err := utils.RetryInbucket(5, func() error { + var resultsMailbox mailservice.JSONMessageHeaderInbucket + err := mailservice.RetryInbucket(5, func() error { var err error - resultsMailbox, err = utils.GetMailBox(user.Email) + resultsMailbox, err = mailservice.GetMailBox(user.Email) return err }) if err != nil { @@ -1949,7 +1949,7 @@ func TestUpdateUserPassword(t *testing.T) { if !strings.ContainsAny(resultsMailbox[0].To[0], user.Email) { t.Fatal("Wrong To recipient") } else { - if resultsEmail, err := utils.GetMessageFromMailbox(user.Email, resultsMailbox[0].ID); err == nil { + if resultsEmail, err := mailservice.GetMessageFromMailbox(user.Email, resultsMailbox[0].ID); err == nil { loc := strings.Index(resultsEmail.Body.Text, "token=") if loc == -1 { t.Log(resultsEmail.Body.Text) diff --git a/app/admin.go b/app/admin.go index 892e2d16b..940d85410 100644 --- a/app/admin.go +++ b/app/admin.go @@ -15,6 +15,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/mailservice" "github.com/mattermost/mattermost-server/utils" ) @@ -242,7 +243,7 @@ func (a *App) TestEmail(userId string, cfg *model.Config) *model.AppError { } else { T := utils.GetUserTranslations(user.Locale) license := a.License() - if err := utils.SendMailUsingConfig(user.Email, T("api.admin.test_email.subject"), T("api.admin.test_email.body"), cfg, license != nil && *license.Features.Compliance); err != nil { + if err := mailservice.SendMailUsingConfig(user.Email, T("api.admin.test_email.subject"), T("api.admin.test_email.body"), cfg, license != nil && *license.Features.Compliance); err != nil { return model.NewAppError("testEmail", "app.admin.test_email.failure", map[string]interface{}{"Error": err.Error()}, "", http.StatusInternalServerError) } } diff --git a/app/email.go b/app/email.go index 8d6535e2b..eefe83a81 100644 --- a/app/email.go +++ b/app/email.go @@ -16,6 +16,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/mailservice" "github.com/mattermost/mattermost-server/utils" ) @@ -402,5 +403,5 @@ func (a *App) SendDeactivateAccountEmail(email string, locale, siteURL string) * func (a *App) SendMail(to, subject, htmlBody string) *model.AppError { license := a.License() - return utils.SendMailUsingConfig(to, subject, htmlBody, a.Config(), license != nil && *license.Features.Compliance) + return mailservice.SendMailUsingConfig(to, subject, htmlBody, a.Config(), license != nil && *license.Features.Compliance) } diff --git a/app/file.go b/app/file.go index 278990b49..5dfc23bd8 100644 --- a/app/file.go +++ b/app/file.go @@ -29,6 +29,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/plugin" + "github.com/mattermost/mattermost-server/services/filesstore" "github.com/mattermost/mattermost-server/utils" ) @@ -58,9 +59,9 @@ const ( IMAGE_PREVIEW_PIXEL_WIDTH = 1920 ) -func (a *App) FileBackend() (utils.FileBackend, *model.AppError) { +func (a *App) FileBackend() (filesstore.FileBackend, *model.AppError) { license := a.License() - return utils.NewFileBackend(&a.Config().FileSettings, license != nil && *license.Features.Compliance) + return filesstore.NewFileBackend(&a.Config().FileSettings, license != nil && *license.Features.Compliance) } func (a *App) ReadFile(path string) ([]byte, *model.AppError) { diff --git a/cmd/mattermost/commands/server.go b/cmd/mattermost/commands/server.go index 95dc2f1c7..e996e6a2e 100644 --- a/cmd/mattermost/commands/server.go +++ b/cmd/mattermost/commands/server.go @@ -17,6 +17,7 @@ import ( "github.com/mattermost/mattermost-server/manualtesting" "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/mailservice" "github.com/mattermost/mattermost-server/utils" "github.com/mattermost/mattermost-server/web" "github.com/mattermost/mattermost-server/wsapi" @@ -67,7 +68,7 @@ func runServer(configFileLocation string, disableConfigWatch bool, usedPlatform } defer a.Shutdown() - utils.TestConnection(a.Config()) + mailservice.TestConnection(a.Config()) pwd, _ := os.Getwd() if usedPlatform { diff --git a/scripts/license-check.sh b/scripts/license-check.sh index a6187829d..c9c60e081 100755 --- a/scripts/license-check.sh +++ b/scripts/license-check.sh @@ -5,7 +5,7 @@ count=0 for fileType in GoFiles; do for file in `go list -f $'{{range .GoFiles}}{{$.Dir}}/{{.}}\n{{end}}' "$@"`; do case $file in - */utils/lru.go|*/store/storetest/mocks/*|*/app/plugin/jira/plugin_*|*/plugin/plugintest/*|*/app/plugin/zoom/plugin_*) + */utils/lru.go|*/store/storetest/mocks/*|*/services/*/mocks/*|*/app/plugin/jira/plugin_*|*/plugin/plugintest/*|*/app/plugin/zoom/plugin_*) # Third-party, doesn't require a header. ;; *) diff --git a/services/filesstore/filesstore.go b/services/filesstore/filesstore.go new file mode 100644 index 000000000..59b3121ff --- /dev/null +++ b/services/filesstore/filesstore.go @@ -0,0 +1,48 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "io" + "net/http" + + "github.com/mattermost/mattermost-server/model" +) + +type FileBackend interface { + TestConnection() *model.AppError + + Reader(path string) (io.ReadCloser, *model.AppError) + ReadFile(path string) ([]byte, *model.AppError) + FileExists(path string) (bool, *model.AppError) + CopyFile(oldPath, newPath string) *model.AppError + MoveFile(oldPath, newPath string) *model.AppError + WriteFile(fr io.Reader, path string) (int64, *model.AppError) + RemoveFile(path string) *model.AppError + + ListDirectory(path string) (*[]string, *model.AppError) + RemoveDirectory(path string) *model.AppError +} + +func NewFileBackend(settings *model.FileSettings, enableComplianceFeatures bool) (FileBackend, *model.AppError) { + switch *settings.DriverName { + case model.IMAGE_DRIVER_S3: + return &S3FileBackend{ + endpoint: settings.AmazonS3Endpoint, + accessKey: settings.AmazonS3AccessKeyId, + secretKey: settings.AmazonS3SecretAccessKey, + secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, + signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, + region: settings.AmazonS3Region, + bucket: settings.AmazonS3Bucket, + encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && enableComplianceFeatures, + trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, + }, nil + case model.IMAGE_DRIVER_LOCAL: + return &LocalFileBackend{ + directory: settings.Directory, + }, nil + } + return nil, model.NewAppError("NewFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError) +} diff --git a/services/filesstore/filesstore_test.go b/services/filesstore/filesstore_test.go new file mode 100644 index 000000000..270f6e9f9 --- /dev/null +++ b/services/filesstore/filesstore_test.go @@ -0,0 +1,288 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils" +) + +type FileBackendTestSuite struct { + suite.Suite + + settings model.FileSettings + backend FileBackend +} + +func TestLocalFileBackendTestSuite(t *testing.T) { + // Setup a global logger to catch tests logging outside of app context + // The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen. + mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{ + EnableConsole: true, + ConsoleJson: true, + ConsoleLevel: "error", + EnableFile: false, + })) + + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(dir) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL), + Directory: dir, + }, + }) +} + +func TestS3FileBackendTestSuite(t *testing.T) { + runBackendTest(t, false) +} + +func TestS3FileBackendTestSuiteWithEncryption(t *testing.T) { + runBackendTest(t, true) +} + +func runBackendTest(t *testing.T, encrypt bool) { + s3Host := os.Getenv("CI_HOST") + if s3Host == "" { + s3Host = "dockerhost" + } + + s3Port := os.Getenv("CI_MINIO_PORT") + if s3Port == "" { + s3Port = "9001" + } + + s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_S3), + AmazonS3AccessKeyId: model.MINIO_ACCESS_KEY, + AmazonS3SecretAccessKey: model.MINIO_SECRET_KEY, + AmazonS3Bucket: model.MINIO_BUCKET, + AmazonS3Endpoint: s3Endpoint, + AmazonS3SSL: model.NewBool(false), + AmazonS3SSE: model.NewBool(encrypt), + }, + }) +} + +func (s *FileBackendTestSuite) SetupTest() { + utils.TranslationsPreInit() + + backend, err := NewFileBackend(&s.settings, true) + require.Nil(s.T(), err) + s.backend = backend +} + +func (s *FileBackendTestSuite) TestConnection() { + s.Nil(s.backend.TestConnection()) +} + +func (s *FileBackendTestSuite) TestReadWriteFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path) + + read, err := s.backend.ReadFile(path) + s.Nil(err) + + readString := string(read) + s.EqualValues(readString, "test") +} + +func (s *FileBackendTestSuite) TestReadWriteFileImage() { + b := []byte("testimage") + path := "tests/" + model.NewId() + ".png" + + written, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path) + + read, err := s.backend.ReadFile(path) + s.Nil(err) + + readString := string(read) + s.EqualValues(readString, "testimage") +} + +func (s *FileBackendTestSuite) TestFileExists() { + b := []byte("testimage") + path := "tests/" + model.NewId() + ".png" + + _, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + defer s.backend.RemoveFile(path) + + res, err := s.backend.FileExists(path) + s.Nil(err) + s.True(res) + + res, err = s.backend.FileExists("tests/idontexist.png") + s.Nil(err) + s.False(res) +} + +func (s *FileBackendTestSuite) TestCopyFile() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + err = s.backend.CopyFile(path1, path2) + s.Nil(err) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Nil(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/newdirectory/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + err = s.backend.CopyFile(path1, path2) + s.Nil(err) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Nil(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestMoveFile() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + s.Nil(s.backend.MoveFile(path1, path2)) + defer s.backend.RemoveFile(path2) + + _, err = s.backend.ReadFile(path1) + s.Error(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestRemoveFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + s.Nil(s.backend.RemoveFile(path)) + + _, err = s.backend.ReadFile(path) + s.Error(err) + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/foo") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/asdf") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + s.Nil(s.backend.RemoveDirectory("tests2")) +} + +func (s *FileBackendTestSuite) TestListDirectory() { + b := []byte("test") + path1 := "19700101/" + model.NewId() + path2 := "19800101/" + model.NewId() + + written, err := s.backend.WriteFile(bytes.NewReader(b), path1) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path1) + + written, err = s.backend.WriteFile(bytes.NewReader(b), path2) + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + defer s.backend.RemoveFile(path2) + + paths, err := s.backend.ListDirectory("") + s.Nil(err) + + found1 := false + found2 := false + for _, path := range *paths { + if path == "19700101" { + found1 = true + } else if path == "19800101" { + found2 = true + } + } + s.True(found1) + s.True(found2) +} + +func (s *FileBackendTestSuite) TestRemoveDirectory() { + b := []byte("test") + + written, err := s.backend.WriteFile(bytes.NewReader(b), "tests2/foo") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/aaa") + s.Nil(err) + s.EqualValues(len(b), written, "expected given number of bytes to have been written") + + s.Nil(s.backend.RemoveDirectory("tests2")) + + _, err = s.backend.ReadFile("tests2/foo") + s.Error(err) + _, err = s.backend.ReadFile("tests2/bar") + s.Error(err) + _, err = s.backend.ReadFile("tests2/asdf") + s.Error(err) +} diff --git a/services/filesstore/localstore.go b/services/filesstore/localstore.go new file mode 100644 index 000000000..8d79a982e --- /dev/null +++ b/services/filesstore/localstore.go @@ -0,0 +1,131 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils" +) + +const ( + TEST_FILE_PATH = "/testfile" +) + +type LocalFileBackend struct { + directory string +} + +func (b *LocalFileBackend) TestConnection() *model.AppError { + f := bytes.NewReader([]byte("testingwrite")) + if _, err := writeFileLocally(f, filepath.Join(b.directory, TEST_FILE_PATH)); err != nil { + return model.NewAppError("TestFileConnection", "api.file.test_connection.local.connection.app_error", nil, err.Error(), http.StatusInternalServerError) + } + os.Remove(filepath.Join(b.directory, TEST_FILE_PATH)) + mlog.Info("Able to write files to local storage.") + return nil +} + +func (b *LocalFileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + if f, err := os.Open(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) { + if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *LocalFileBackend) FileExists(path string) (bool, *model.AppError) { + _, err := os.Stat(filepath.Join(b.directory, path)) + + if os.IsNotExist(err) { + return false, nil + } else if err == nil { + return true, nil + } + + return false, model.NewAppError("ReadFile", "api.file.file_exists.exists_local.app_error", nil, err.Error(), http.StatusInternalServerError) +} + +func (b *LocalFileBackend) CopyFile(oldPath, newPath string) *model.AppError { + if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + return model.NewAppError("copyFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { + return writeFileLocally(fr, filepath.Join(b.directory, path)) +} + +func writeFileLocally(fr io.Reader, path string) (int64, *model.AppError) { + if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { + directory, _ := filepath.Abs(filepath.Dir(path)) + return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) + } + fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) + } + defer fw.Close() + written, err := io.Copy(fw, fr) + if err != nil { + return written, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return written, nil +} + +func (b *LocalFileBackend) RemoveFile(path string) *model.AppError { + if err := os.Remove(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + if fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + for _, fileInfo := range fileInfos { + if fileInfo.IsDir() { + paths = append(paths, filepath.Join(path, fileInfo.Name())) + } + } + } + return &paths, nil +} + +func (b *LocalFileBackend) RemoveDirectory(path string) *model.AppError { + if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} diff --git a/services/filesstore/mocks/FileBackend.go b/services/filesstore/mocks/FileBackend.go new file mode 100644 index 000000000..5d75ae5d1 --- /dev/null +++ b/services/filesstore/mocks/FileBackend.go @@ -0,0 +1,215 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +// Regenerate this file using `make files-store-mocks`. + +package mocks + +import io "io" +import mock "github.com/stretchr/testify/mock" +import model "github.com/mattermost/mattermost-server/model" + +// FileBackend is an autogenerated mock type for the FileBackend type +type FileBackend struct { + mock.Mock +} + +// CopyFile provides a mock function with given fields: oldPath, newPath +func (_m *FileBackend) CopyFile(oldPath string, newPath string) *model.AppError { + ret := _m.Called(oldPath, newPath) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string, string) *model.AppError); ok { + r0 = rf(oldPath, newPath) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// FileExists provides a mock function with given fields: path +func (_m *FileBackend) FileExists(path string) (bool, *model.AppError) { + ret := _m.Called(path) + + var r0 bool + if rf, ok := ret.Get(0).(func(string) bool); ok { + r0 = rf(path) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// ListDirectory provides a mock function with given fields: path +func (_m *FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + ret := _m.Called(path) + + var r0 *[]string + if rf, ok := ret.Get(0).(func(string) *[]string); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*[]string) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// MoveFile provides a mock function with given fields: oldPath, newPath +func (_m *FileBackend) MoveFile(oldPath string, newPath string) *model.AppError { + ret := _m.Called(oldPath, newPath) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string, string) *model.AppError); ok { + r0 = rf(oldPath, newPath) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// ReadFile provides a mock function with given fields: path +func (_m *FileBackend) ReadFile(path string) ([]byte, *model.AppError) { + ret := _m.Called(path) + + var r0 []byte + if rf, ok := ret.Get(0).(func(string) []byte); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// Reader provides a mock function with given fields: path +func (_m *FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + ret := _m.Called(path) + + var r0 io.ReadCloser + if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.ReadCloser) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// RemoveDirectory provides a mock function with given fields: path +func (_m *FileBackend) RemoveDirectory(path string) *model.AppError { + ret := _m.Called(path) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string) *model.AppError); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// RemoveFile provides a mock function with given fields: path +func (_m *FileBackend) RemoveFile(path string) *model.AppError { + ret := _m.Called(path) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string) *model.AppError); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// TestConnection provides a mock function with given fields: +func (_m *FileBackend) TestConnection() *model.AppError { + ret := _m.Called() + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func() *model.AppError); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// WriteFile provides a mock function with given fields: fr, path +func (_m *FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { + ret := _m.Called(fr, path) + + var r0 int64 + if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok { + r0 = rf(fr, path) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(io.Reader, string) *model.AppError); ok { + r1 = rf(fr, path) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} diff --git a/services/filesstore/s3store.go b/services/filesstore/s3store.go new file mode 100644 index 000000000..0a0f057ea --- /dev/null +++ b/services/filesstore/s3store.go @@ -0,0 +1,294 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + s3 "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/credentials" + "github.com/minio/minio-go/pkg/encrypt" + + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" +) + +type S3FileBackend struct { + endpoint string + accessKey string + secretKey string + secure bool + signV2 bool + region string + bucket string + encrypt bool + trace bool +} + +// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. +// If signV2 input is false, function always returns signature v4. +// +// Additionally this function also takes a user defined region, if set +// disables automatic region lookup. +func (b *S3FileBackend) s3New() (*s3.Client, error) { + var creds *credentials.Credentials + + if b.accessKey == "" && b.secretKey == "" { + creds = credentials.NewIAM("") + } else if b.signV2 { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) + } else { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) + } + + s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) + if err != nil { + return nil, err + } + + if b.trace { + s3Clnt.TraceOn(os.Stdout) + } + + return s3Clnt, nil +} + +func (b *S3FileBackend) TestConnection() *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.connection.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + exists, err := s3Clnt.BucketExists(b.bucket) + if err != nil { + return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucket_exists.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if !exists { + mlog.Warn("Bucket specified does not exist. Attempting to create...") + err := s3Clnt.MakeBucket(b.bucket, b.region) + if err != nil { + mlog.Error("Unable to create bucket.") + return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucked_create.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + mlog.Info("Connection to S3 or minio is good. Bucket exists.") + return nil +} + +// Caller must close the first return value +func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) + if err != nil { + return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return minioObject, nil +} + +func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + defer minioObject.Close() + if f, err := ioutil.ReadAll(minioObject); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) { + s3Clnt, err := b.s3New() + + if err != nil { + return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + _, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{}) + + if err == nil { + return true, nil + } + + if err.(s3.ErrorResponse).Code == "NoSuchKey" { + return false, nil + } + + return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) +} + +func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + source := s3.NewSourceInfo(b.bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) + if err != nil { + return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + source := s3.NewSourceInfo(b.bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + var contentType string + if ext := filepath.Ext(path); model.IsFileExtImage(ext) { + contentType = model.GetImageMimeType(ext) + } else { + contentType = "binary/octet-stream" + } + + options := s3PutOptions(b.encrypt, contentType) + var buf bytes.Buffer + _, err = buf.ReadFrom(fr) + if err != nil { + return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + written, err := s3Clnt.PutObject(b.bucket, path, &buf, int64(buf.Len()), options) + if err != nil { + return written, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return written, nil +} + +func (b *S3FileBackend) RemoveFile(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { + out := make(chan string, 1) + + go func() { + defer close(out) + + for { + info, done := <-in + + if !done { + break + } + + out <- info.Key + } + }() + + return out +} + +func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + defer close(doneCh) + + for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { + if object.Err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) + } + paths = append(paths, strings.Trim(object.Key, "/")) + } + + return &paths, nil +} + +func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { + if err.Err != nil { + doneCh <- struct{}{} + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) + } + } + + close(doneCh) + return nil +} + +func s3PutOptions(encrypted bool, contentType string) s3.PutObjectOptions { + options := s3.PutObjectOptions{} + if encrypted { + options.ServerSideEncryption = encrypt.NewSSE() + } + options.ContentType = contentType + + return options +} + +func CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError { + if len(settings.AmazonS3Bucket) == 0 { + return model.NewAppError("S3File", "api.admin.test_s3.missing_s3_bucket", nil, "", http.StatusBadRequest) + } + + // if S3 endpoint is not set call the set defaults to set that + if len(settings.AmazonS3Endpoint) == 0 { + settings.SetDefaults() + } + + return nil +} diff --git a/services/filesstore/s3store_test.go b/services/filesstore/s3store_test.go new file mode 100644 index 000000000..a958a1f3f --- /dev/null +++ b/services/filesstore/s3store_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2018-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package filesstore + +import ( + "testing" + + "github.com/mattermost/mattermost-server/model" +) + +func TestCheckMandatoryS3Fields(t *testing.T) { + cfg := model.FileSettings{} + + err := CheckMandatoryS3Fields(&cfg) + if err == nil || err.Message != "api.admin.test_s3.missing_s3_bucket" { + t.Fatal("should've failed with missing s3 bucket") + } + + cfg.AmazonS3Bucket = "test-mm" + err = CheckMandatoryS3Fields(&cfg) + if err != nil { + t.Fatal("should've not failed") + } + + cfg.AmazonS3Endpoint = "" + err = CheckMandatoryS3Fields(&cfg) + if err != nil || cfg.AmazonS3Endpoint != "s3.amazonaws.com" { + t.Fatal("should've not failed because it should set the endpoint to the default") + } + +} diff --git a/services/mailservice/inbucket.go b/services/mailservice/inbucket.go new file mode 100644 index 000000000..28922d660 --- /dev/null +++ b/services/mailservice/inbucket.go @@ -0,0 +1,200 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package mailservice + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +const ( + INBUCKET_API = "/api/v1/mailbox/" +) + +// OutputJSONHeader holds the received Header to test sending emails (inbucket) +type JSONMessageHeaderInbucket []struct { + Mailbox string + ID string `json:"Id"` + From, Subject, Date string + To []string + Size int +} + +// OutputJSONMessage holds the received Message fto test sending emails (inbucket) +type JSONMessageInbucket struct { + Mailbox string + ID string `json:"Id"` + From, Subject, Date string + Size int + Header map[string][]string + Body struct { + Text string + HTML string `json:"Html"` + } + Attachments []struct { + Filename string + ContentType string `json:"content-type"` + DownloadLink string `json:"download-link"` + Bytes []byte `json:"-"` + } +} + +func ParseEmail(email string) string { + pos := strings.Index(email, "@") + parsedEmail := email[0:pos] + return parsedEmail +} + +func GetMailBox(email string) (results JSONMessageHeaderInbucket, err error) { + + parsedEmail := ParseEmail(email) + + url := fmt.Sprintf("%s%s%s", getInbucketHost(), INBUCKET_API, parsedEmail) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + client := &http.Client{} + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.Body == nil { + return nil, fmt.Errorf("No Mailbox") + } + + var record JSONMessageHeaderInbucket + err = json.NewDecoder(resp.Body).Decode(&record) + switch { + case err == io.EOF: + return nil, fmt.Errorf("Error: %s", err) + case err != nil: + return nil, fmt.Errorf("Error: %s", err) + } + if len(record) == 0 { + return nil, fmt.Errorf("No mailbox") + } + + return record, nil +} + +func GetMessageFromMailbox(email, id string) (results JSONMessageInbucket, err error) { + + parsedEmail := ParseEmail(email) + + var record JSONMessageInbucket + + url := fmt.Sprintf("%s%s%s/%s", getInbucketHost(), INBUCKET_API, parsedEmail, id) + emailResponse, err := get(url) + if err != nil { + return record, err + } + defer emailResponse.Body.Close() + + err = json.NewDecoder(emailResponse.Body).Decode(&record) + + // download attachments + if record.Attachments != nil && len(record.Attachments) > 0 { + for i := range record.Attachments { + if bytes, err := downloadAttachment(record.Attachments[i].DownloadLink); err != nil { + return record, err + } else { + record.Attachments[i].Bytes = make([]byte, len(bytes)) + copy(record.Attachments[i].Bytes, bytes) + } + } + } + + return record, err +} + +func downloadAttachment(url string) ([]byte, error) { + attachmentResponse, err := get(url) + if err != nil { + return nil, err + } + defer attachmentResponse.Body.Close() + + buf := new(bytes.Buffer) + io.Copy(buf, attachmentResponse.Body) + return buf.Bytes(), nil +} + +func get(url string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + return resp, nil +} + +func DeleteMailBox(email string) (err error) { + + parsedEmail := ParseEmail(email) + + url := fmt.Sprintf("%s%s%s", getInbucketHost(), INBUCKET_API, parsedEmail) + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + client := &http.Client{} + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + +func RetryInbucket(attempts int, callback func() error) (err error) { + for i := 0; ; i++ { + err = callback() + if err == nil { + return nil + } + + if i >= (attempts - 1) { + break + } + + time.Sleep(5 * time.Second) + + fmt.Println("retrying...") + } + return fmt.Errorf("After %d attempts, last error: %s", attempts, err) +} + +func getInbucketHost() (host string) { + + inbucket_host := os.Getenv("CI_HOST") + if inbucket_host == "" { + inbucket_host = "dockerhost" + } + + inbucket_port := os.Getenv("CI_INBUCKET_PORT") + if inbucket_port == "" { + inbucket_port = "9000" + } + return fmt.Sprintf("http://%s:%s", inbucket_host, inbucket_port) +} diff --git a/services/mailservice/mail.go b/services/mailservice/mail.go new file mode 100644 index 000000000..e9b689f20 --- /dev/null +++ b/services/mailservice/mail.go @@ -0,0 +1,299 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package mailservice + +import ( + "crypto/tls" + "errors" + "fmt" + "io" + "mime" + "net" + "net/mail" + "net/smtp" + "time" + + "gopkg.in/gomail.v2" + + "net/http" + + "github.com/jaytaylor/html2text" + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/filesstore" + "github.com/mattermost/mattermost-server/utils" +) + +func encodeRFC2047Word(s string) string { + return mime.BEncoding.Encode("utf-8", s) +} + +type SmtpConnectionInfo struct { + SmtpUsername string + SmtpPassword string + SmtpServerName string + SmtpServerHost string + SmtpPort string + SkipCertVerification bool + ConnectionSecurity string + Auth bool +} + +type authChooser struct { + smtp.Auth + connectionInfo *SmtpConnectionInfo +} + +func (a *authChooser) Start(server *smtp.ServerInfo) (string, []byte, error) { + smtpAddress := a.connectionInfo.SmtpServerName + ":" + a.connectionInfo.SmtpPort + a.Auth = LoginAuth(a.connectionInfo.SmtpUsername, a.connectionInfo.SmtpPassword, smtpAddress) + for _, method := range server.Auth { + if method == "PLAIN" { + a.Auth = smtp.PlainAuth("", a.connectionInfo.SmtpUsername, a.connectionInfo.SmtpPassword, a.connectionInfo.SmtpServerName+":"+a.connectionInfo.SmtpPort) + break + } + } + return a.Auth.Start(server) +} + +type loginAuth struct { + username, password, host string +} + +func LoginAuth(username, password, host string) smtp.Auth { + return &loginAuth{username, password, host} +} + +func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) { + if !server.TLS { + return "", nil, errors.New("unencrypted connection") + } + + if server.Name != a.host { + return "", nil, errors.New("wrong host name") + } + + return "LOGIN", []byte{}, nil +} + +func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) { + if more { + switch string(fromServer) { + case "Username:": + return []byte(a.username), nil + case "Password:": + return []byte(a.password), nil + default: + return nil, errors.New("Unknown fromServer") + } + } + return nil, nil +} + +func ConnectToSMTPServerAdvanced(connectionInfo *SmtpConnectionInfo) (net.Conn, *model.AppError) { + var conn net.Conn + var err error + + smtpAddress := connectionInfo.SmtpServerHost + ":" + connectionInfo.SmtpPort + if connectionInfo.ConnectionSecurity == model.CONN_SECURITY_TLS { + tlsconfig := &tls.Config{ + InsecureSkipVerify: connectionInfo.SkipCertVerification, + ServerName: connectionInfo.SmtpServerName, + } + + conn, err = tls.Dial("tcp", smtpAddress, tlsconfig) + if err != nil { + return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.open_tls.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } else { + conn, err = net.Dial("tcp", smtpAddress) + if err != nil { + return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.open.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + return conn, nil +} + +func ConnectToSMTPServer(config *model.Config) (net.Conn, *model.AppError) { + return ConnectToSMTPServerAdvanced( + &SmtpConnectionInfo{ + ConnectionSecurity: config.EmailSettings.ConnectionSecurity, + SkipCertVerification: *config.EmailSettings.SkipServerCertificateVerification, + SmtpServerName: config.EmailSettings.SMTPServer, + SmtpServerHost: config.EmailSettings.SMTPServer, + SmtpPort: config.EmailSettings.SMTPPort, + }, + ) +} + +func NewSMTPClientAdvanced(conn net.Conn, hostname string, connectionInfo *SmtpConnectionInfo) (*smtp.Client, *model.AppError) { + c, err := smtp.NewClient(conn, connectionInfo.SmtpServerName+":"+connectionInfo.SmtpPort) + if err != nil { + mlog.Error(fmt.Sprintf("Failed to open a connection to SMTP server %v", err)) + return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.open_tls.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if hostname != "" { + err := c.Hello(hostname) + if err != nil { + mlog.Error(fmt.Sprintf("Failed to to set the HELO to SMTP server %v", err)) + return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.helo.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + if connectionInfo.ConnectionSecurity == model.CONN_SECURITY_STARTTLS { + tlsconfig := &tls.Config{ + InsecureSkipVerify: connectionInfo.SkipCertVerification, + ServerName: connectionInfo.SmtpServerName, + } + c.StartTLS(tlsconfig) + } + + if connectionInfo.Auth { + if err = c.Auth(&authChooser{connectionInfo: connectionInfo}); err != nil { + return nil, model.NewAppError("SendMail", "utils.mail.new_client.auth.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + return c, nil +} + +func NewSMTPClient(conn net.Conn, config *model.Config) (*smtp.Client, *model.AppError) { + return NewSMTPClientAdvanced( + conn, + utils.GetHostnameFromSiteURL(*config.ServiceSettings.SiteURL), + &SmtpConnectionInfo{ + ConnectionSecurity: config.EmailSettings.ConnectionSecurity, + SkipCertVerification: *config.EmailSettings.SkipServerCertificateVerification, + SmtpServerName: config.EmailSettings.SMTPServer, + SmtpServerHost: config.EmailSettings.SMTPServer, + SmtpPort: config.EmailSettings.SMTPPort, + Auth: *config.EmailSettings.EnableSMTPAuth, + SmtpUsername: config.EmailSettings.SMTPUsername, + SmtpPassword: config.EmailSettings.SMTPPassword, + }, + ) +} + +func TestConnection(config *model.Config) { + if !config.EmailSettings.SendEmailNotifications { + return + } + + conn, err1 := ConnectToSMTPServer(config) + if err1 != nil { + mlog.Error(fmt.Sprintf("SMTP server settings do not appear to be configured properly err=%v details=%v", utils.T(err1.Message), err1.DetailedError)) + return + } + defer conn.Close() + + c, err2 := NewSMTPClient(conn, config) + if err2 != nil { + mlog.Error(fmt.Sprintf("SMTP server settings do not appear to be configured properly err=%v details=%v", utils.T(err2.Message), err2.DetailedError)) + return + } + defer c.Quit() + defer c.Close() +} + +func SendMailUsingConfig(to, subject, htmlBody string, config *model.Config, enableComplianceFeatures bool) *model.AppError { + fromMail := mail.Address{Name: config.EmailSettings.FeedbackName, Address: config.EmailSettings.FeedbackEmail} + + return SendMailUsingConfigAdvanced(to, to, fromMail, subject, htmlBody, nil, nil, config, enableComplianceFeatures) +} + +// allows for sending an email with attachments and differing MIME/SMTP recipients +func SendMailUsingConfigAdvanced(mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, config *model.Config, enableComplianceFeatures bool) *model.AppError { + if !config.EmailSettings.SendEmailNotifications || len(config.EmailSettings.SMTPServer) == 0 { + return nil + } + + conn, err := ConnectToSMTPServer(config) + if err != nil { + return err + } + defer conn.Close() + + c, err := NewSMTPClient(conn, config) + if err != nil { + return err + } + defer c.Quit() + defer c.Close() + + fileBackend, err := filesstore.NewFileBackend(&config.FileSettings, enableComplianceFeatures) + if err != nil { + return err + } + + return SendMail(c, mimeTo, smtpTo, from, subject, htmlBody, attachments, mimeHeaders, fileBackend, time.Now()) +} + +func SendMail(c *smtp.Client, mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, fileBackend filesstore.FileBackend, date time.Time) *model.AppError { + mlog.Debug(fmt.Sprintf("sending mail to %v with subject of '%v'", smtpTo, subject)) + + htmlMessage := "\r\n" + htmlBody + "" + + txtBody, err := html2text.FromString(htmlBody) + if err != nil { + mlog.Warn(fmt.Sprint(err)) + txtBody = "" + } + + headers := map[string][]string{ + "From": {from.String()}, + "To": {mimeTo}, + "Subject": {encodeRFC2047Word(subject)}, + "Content-Transfer-Encoding": {"8bit"}, + "Auto-Submitted": {"auto-generated"}, + "Precedence": {"bulk"}, + } + for k, v := range mimeHeaders { + headers[k] = []string{encodeRFC2047Word(v)} + } + + m := gomail.NewMessage(gomail.SetCharset("UTF-8")) + m.SetHeaders(headers) + m.SetDateHeader("Date", date) + m.SetBody("text/plain", txtBody) + m.AddAlternative("text/html", htmlMessage) + + for _, fileInfo := range attachments { + bytes, err := fileBackend.ReadFile(fileInfo.Path) + if err != nil { + return err + } + + m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error { + if _, err := writer.Write(bytes); err != nil { + return model.NewAppError("SendMail", "utils.mail.sendMail.attachments.write_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil + })) + } + + if err := c.Mail(from.Address); err != nil { + return model.NewAppError("SendMail", "utils.mail.send_mail.from_address.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := c.Rcpt(smtpTo); err != nil { + return model.NewAppError("SendMail", "utils.mail.send_mail.to_address.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + w, err := c.Data() + if err != nil { + return model.NewAppError("SendMail", "utils.mail.send_mail.msg_data.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + _, err = m.WriteTo(w) + if err != nil { + return model.NewAppError("SendMail", "utils.mail.send_mail.msg.app_error", nil, err.Error(), http.StatusInternalServerError) + } + err = w.Close() + if err != nil { + return model.NewAppError("SendMail", "utils.mail.send_mail.close.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} diff --git a/services/mailservice/mail_test.go b/services/mailservice/mail_test.go new file mode 100644 index 000000000..9ad48c703 --- /dev/null +++ b/services/mailservice/mail_test.go @@ -0,0 +1,290 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package mailservice + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "net/mail" + "net/smtp" + + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/filesstore" + "github.com/mattermost/mattermost-server/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMailConnectionFromConfig(t *testing.T) { + cfg, _, _, err := utils.LoadConfig("config.json") + require.Nil(t, err) + + if conn, err := ConnectToSMTPServer(cfg); err != nil { + t.Log(err) + t.Fatal("Should connect to the STMP Server") + } else { + if _, err1 := NewSMTPClient(conn, cfg); err1 != nil { + t.Log(err) + t.Fatal("Should get new smtp client") + } + } + + cfg.EmailSettings.SMTPServer = "wrongServer" + cfg.EmailSettings.SMTPPort = "553" + + if _, err := ConnectToSMTPServer(cfg); err == nil { + t.Log(err) + t.Fatal("Should not to the STMP Server") + } +} + +func TestMailConnectionAdvanced(t *testing.T) { + cfg, _, _, err := utils.LoadConfig("config.json") + require.Nil(t, err) + + if conn, err := ConnectToSMTPServerAdvanced( + &SmtpConnectionInfo{ + ConnectionSecurity: cfg.EmailSettings.ConnectionSecurity, + SkipCertVerification: *cfg.EmailSettings.SkipServerCertificateVerification, + SmtpServerName: cfg.EmailSettings.SMTPServer, + SmtpServerHost: cfg.EmailSettings.SMTPServer, + SmtpPort: cfg.EmailSettings.SMTPPort, + }, + ); err != nil { + t.Log(err) + t.Fatal("Should connect to the STMP Server") + } else { + if _, err1 := NewSMTPClientAdvanced( + conn, + utils.GetHostnameFromSiteURL(*cfg.ServiceSettings.SiteURL), + &SmtpConnectionInfo{ + ConnectionSecurity: cfg.EmailSettings.ConnectionSecurity, + SkipCertVerification: *cfg.EmailSettings.SkipServerCertificateVerification, + SmtpServerName: cfg.EmailSettings.SMTPServer, + SmtpServerHost: cfg.EmailSettings.SMTPServer, + SmtpPort: cfg.EmailSettings.SMTPPort, + Auth: *cfg.EmailSettings.EnableSMTPAuth, + SmtpUsername: cfg.EmailSettings.SMTPUsername, + SmtpPassword: cfg.EmailSettings.SMTPPassword, + }, + ); err1 != nil { + t.Log(err) + t.Fatal("Should get new smtp client") + } + } + + if _, err := ConnectToSMTPServerAdvanced( + &SmtpConnectionInfo{ + ConnectionSecurity: cfg.EmailSettings.ConnectionSecurity, + SkipCertVerification: *cfg.EmailSettings.SkipServerCertificateVerification, + SmtpServerName: "wrongServer", + SmtpServerHost: "wrongServer", + SmtpPort: "553", + }, + ); err == nil { + t.Log(err) + t.Fatal("Should not to the STMP Server") + } + +} + +func TestSendMailUsingConfig(t *testing.T) { + cfg, _, _, err := utils.LoadConfig("config.json") + require.Nil(t, err) + utils.T = utils.GetUserTranslations("en") + + var emailTo = "test@example.com" + var emailSubject = "Testing this email" + var emailBody = "This is a test from autobot" + + //Delete all the messages before check the sample email + DeleteMailBox(emailTo) + + if err := SendMailUsingConfig(emailTo, emailSubject, emailBody, cfg, true); err != nil { + t.Log(err) + t.Fatal("Should connect to the STMP Server") + } else { + //Check if the email was send to the right email address + var resultsMailbox JSONMessageHeaderInbucket + err := RetryInbucket(5, func() error { + var err error + resultsMailbox, err = GetMailBox(emailTo) + return err + }) + if err != nil { + t.Log(err) + t.Log("No email was received, maybe due load on the server. Disabling this verification") + } + if err == nil && len(resultsMailbox) > 0 { + if !strings.ContainsAny(resultsMailbox[0].To[0], emailTo) { + t.Fatal("Wrong To recipient") + } else { + if resultsEmail, err := GetMessageFromMailbox(emailTo, resultsMailbox[0].ID); err == nil { + if !strings.Contains(resultsEmail.Body.Text, emailBody) { + t.Log(resultsEmail.Body.Text) + t.Fatal("Received message") + } + } + } + } + } +} + +func TestSendMailUsingConfigAdvanced(t *testing.T) { + cfg, _, _, err := utils.LoadConfig("config.json") + require.Nil(t, err) + utils.T = utils.GetUserTranslations("en") + + var mimeTo = "test@example.com" + var smtpTo = "test2@example.com" + var from = mail.Address{Name: "Nobody", Address: "nobody@mattermost.com"} + var emailSubject = "Testing this email" + var emailBody = "This is a test from autobot" + + //Delete all the messages before check the sample email + DeleteMailBox(smtpTo) + + fileBackend, err := filesstore.NewFileBackend(&cfg.FileSettings, true) + assert.Nil(t, err) + + // create two files with the same name that will both be attached to the email + fileName := "file.txt" + filePath1 := fmt.Sprintf("test1/%s", fileName) + filePath2 := fmt.Sprintf("test2/%s", fileName) + fileContents1 := []byte("hello world") + fileContents2 := []byte("foo bar") + _, err = fileBackend.WriteFile(bytes.NewReader(fileContents1), filePath1) + assert.Nil(t, err) + _, err = fileBackend.WriteFile(bytes.NewReader(fileContents2), filePath2) + assert.Nil(t, err) + defer fileBackend.RemoveFile(filePath1) + defer fileBackend.RemoveFile(filePath2) + + attachments := make([]*model.FileInfo, 2) + attachments[0] = &model.FileInfo{ + Name: fileName, + Path: filePath1, + } + attachments[1] = &model.FileInfo{ + Name: fileName, + Path: filePath2, + } + + headers := make(map[string]string) + headers["TestHeader"] = "TestValue" + + if err := SendMailUsingConfigAdvanced(mimeTo, smtpTo, from, emailSubject, emailBody, attachments, headers, cfg, true); err != nil { + t.Log(err) + t.Fatal("Should connect to the STMP Server") + } else { + //Check if the email was send to the right email address + var resultsMailbox JSONMessageHeaderInbucket + err := RetryInbucket(5, func() error { + var err error + resultsMailbox, err = GetMailBox(smtpTo) + return err + }) + if err != nil { + t.Log(err) + t.Fatal("No emails found for address " + smtpTo) + } + if err == nil && len(resultsMailbox) > 0 { + if !strings.ContainsAny(resultsMailbox[0].To[0], smtpTo) { + t.Fatal("Wrong To recipient") + } else { + if resultsEmail, err := GetMessageFromMailbox(smtpTo, resultsMailbox[0].ID); err == nil { + if !strings.Contains(resultsEmail.Body.Text, emailBody) { + t.Log(resultsEmail.Body.Text) + t.Fatal("Received message") + } + + // verify that the To header of the email message is set to the MIME recipient, even though we got it out of the SMTP recipient's email inbox + assert.Equal(t, mimeTo, resultsEmail.Header["To"][0]) + + // verify that the MIME from address is correct - unfortunately, we can't verify the SMTP from address + assert.Equal(t, from.String(), resultsEmail.Header["From"][0]) + + // check that the custom mime headers came through - header case seems to get mutated + assert.Equal(t, "TestValue", resultsEmail.Header["Testheader"][0]) + + // ensure that the attachments were successfully sent + assert.Len(t, resultsEmail.Attachments, 2) + assert.Equal(t, fileName, resultsEmail.Attachments[0].Filename) + assert.Equal(t, fileName, resultsEmail.Attachments[1].Filename) + attachment1 := string(resultsEmail.Attachments[0].Bytes) + attachment2 := string(resultsEmail.Attachments[1].Bytes) + if attachment1 == string(fileContents1) { + assert.Equal(t, attachment2, string(fileContents2)) + } else if attachment1 == string(fileContents2) { + assert.Equal(t, attachment2, string(fileContents1)) + } else { + assert.Fail(t, "Unrecognized attachment contents") + } + } + } + } + } +} + +func TestAuthMethods(t *testing.T) { + auth := &authChooser{ + connectionInfo: &SmtpConnectionInfo{ + SmtpUsername: "test", + SmtpPassword: "fakepass", + SmtpServerName: "fakeserver", + SmtpServerHost: "fakeserver", + SmtpPort: "25", + }, + } + tests := []struct { + desc string + server *smtp.ServerInfo + err string + }{ + { + desc: "auth PLAIN success", + server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: true}, + }, + { + desc: "auth PLAIN unencrypted connection fail", + server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: false}, + err: "unencrypted connection", + }, + { + desc: "auth PLAIN wrong host name", + server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"PLAIN"}, TLS: true}, + err: "wrong host name", + }, + { + desc: "auth LOGIN success", + server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: true}, + }, + { + desc: "auth LOGIN unencrypted connection fail", + server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"LOGIN"}, TLS: true}, + err: "wrong host name", + }, + { + desc: "auth LOGIN wrong host name", + server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: false}, + err: "unencrypted connection", + }, + } + + for i, test := range tests { + t.Run(test.desc, func(t *testing.T) { + _, _, err := auth.Start(test.server) + got := "" + if err != nil { + got = err.Error() + } + if got != test.err { + t.Errorf("%d. got error = %q; want %q", i, got, test.err) + } + }) + } +} diff --git a/utils/file_backend.go b/utils/file_backend.go deleted file mode 100644 index 368e1ba28..000000000 --- a/utils/file_backend.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "io" - "net/http" - - "github.com/mattermost/mattermost-server/model" -) - -type FileBackend interface { - TestConnection() *model.AppError - - Reader(path string) (io.ReadCloser, *model.AppError) - ReadFile(path string) ([]byte, *model.AppError) - FileExists(path string) (bool, *model.AppError) - CopyFile(oldPath, newPath string) *model.AppError - MoveFile(oldPath, newPath string) *model.AppError - WriteFile(fr io.Reader, path string) (int64, *model.AppError) - RemoveFile(path string) *model.AppError - - ListDirectory(path string) (*[]string, *model.AppError) - RemoveDirectory(path string) *model.AppError -} - -func NewFileBackend(settings *model.FileSettings, enableComplianceFeatures bool) (FileBackend, *model.AppError) { - switch *settings.DriverName { - case model.IMAGE_DRIVER_S3: - return &S3FileBackend{ - endpoint: settings.AmazonS3Endpoint, - accessKey: settings.AmazonS3AccessKeyId, - secretKey: settings.AmazonS3SecretAccessKey, - secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, - signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, - region: settings.AmazonS3Region, - bucket: settings.AmazonS3Bucket, - encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && enableComplianceFeatures, - trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, - }, nil - case model.IMAGE_DRIVER_LOCAL: - return &LocalFileBackend{ - directory: settings.Directory, - }, nil - } - return nil, model.NewAppError("NewFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError) -} diff --git a/utils/file_backend_local.go b/utils/file_backend_local.go deleted file mode 100644 index 681ab9234..000000000 --- a/utils/file_backend_local.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - - "github.com/mattermost/mattermost-server/mlog" - "github.com/mattermost/mattermost-server/model" -) - -const ( - TEST_FILE_PATH = "/testfile" -) - -type LocalFileBackend struct { - directory string -} - -func (b *LocalFileBackend) TestConnection() *model.AppError { - f := bytes.NewReader([]byte("testingwrite")) - if _, err := writeFileLocally(f, filepath.Join(b.directory, TEST_FILE_PATH)); err != nil { - return model.NewAppError("TestFileConnection", "api.file.test_connection.local.connection.app_error", nil, err.Error(), http.StatusInternalServerError) - } - os.Remove(filepath.Join(b.directory, TEST_FILE_PATH)) - mlog.Info("Able to write files to local storage.") - return nil -} - -func (b *LocalFileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { - if f, err := os.Open(filepath.Join(b.directory, path)); err != nil { - return nil, model.NewAppError("Reader", "api.file.reader.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } -} - -func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) { - if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } -} - -func (b *LocalFileBackend) FileExists(path string) (bool, *model.AppError) { - _, err := os.Stat(filepath.Join(b.directory, path)) - - if os.IsNotExist(err) { - return false, nil - } else if err == nil { - return true, nil - } - - return false, model.NewAppError("ReadFile", "api.file.file_exists.exists_local.app_error", nil, err.Error(), http.StatusInternalServerError) -} - -func (b *LocalFileBackend) CopyFile(oldPath, newPath string) *model.AppError { - if err := CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { - return model.NewAppError("copyFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil -} - -func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError { - if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return nil -} - -func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { - return writeFileLocally(fr, filepath.Join(b.directory, path)) -} - -func writeFileLocally(fr io.Reader, path string) (int64, *model.AppError) { - if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { - directory, _ := filepath.Abs(filepath.Dir(path)) - return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) - } - fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) - } - defer fw.Close() - written, err := io.Copy(fw, fr) - if err != nil { - return written, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return written, nil -} - -func (b *LocalFileBackend) RemoveFile(path string) *model.AppError { - if err := os.Remove(filepath.Join(b.directory, path)); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil -} - -func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppError) { - var paths []string - if fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path)); err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - for _, fileInfo := range fileInfos { - if fileInfo.IsDir() { - paths = append(paths, filepath.Join(path, fileInfo.Name())) - } - } - } - return &paths, nil -} - -func (b *LocalFileBackend) RemoveDirectory(path string) *model.AppError { - if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil -} diff --git a/utils/file_backend_s3.go b/utils/file_backend_s3.go deleted file mode 100644 index f5f96f878..000000000 --- a/utils/file_backend_s3.go +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - - s3 "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/encrypt" - - "github.com/mattermost/mattermost-server/mlog" - "github.com/mattermost/mattermost-server/model" -) - -type S3FileBackend struct { - endpoint string - accessKey string - secretKey string - secure bool - signV2 bool - region string - bucket string - encrypt bool - trace bool -} - -// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. -// If signV2 input is false, function always returns signature v4. -// -// Additionally this function also takes a user defined region, if set -// disables automatic region lookup. -func (b *S3FileBackend) s3New() (*s3.Client, error) { - var creds *credentials.Credentials - - if b.accessKey == "" && b.secretKey == "" { - creds = credentials.NewIAM("") - } else if b.signV2 { - creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) - } else { - creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) - } - - s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) - if err != nil { - return nil, err - } - - if b.trace { - s3Clnt.TraceOn(os.Stdout) - } - - return s3Clnt, nil -} - -func (b *S3FileBackend) TestConnection() *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.connection.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - exists, err := s3Clnt.BucketExists(b.bucket) - if err != nil { - return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucket_exists.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if !exists { - mlog.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(b.bucket, b.region) - if err != nil { - mlog.Error("Unable to create bucket.") - return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucked_create.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } - mlog.Info("Connection to S3 or minio is good. Bucket exists.") - return nil -} - -// Caller must close the first return value -func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { - s3Clnt, err := b.s3New() - if err != nil { - return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) - if err != nil { - return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return minioObject, nil -} - -func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { - s3Clnt, err := b.s3New() - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - defer minioObject.Close() - if f, err := ioutil.ReadAll(minioObject); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } -} - -func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) { - s3Clnt, err := b.s3New() - - if err != nil { - return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - _, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{}) - - if err == nil { - return true, nil - } - - if err.(s3.ErrorResponse).Code == "NoSuchKey" { - return false, nil - } - - return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) -} - -func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - source := s3.NewSourceInfo(b.bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) - if err != nil { - return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil -} - -func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - source := s3.NewSourceInfo(b.bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil -} - -func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { - s3Clnt, err := b.s3New() - if err != nil { - return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - var contentType string - if ext := filepath.Ext(path); model.IsFileExtImage(ext) { - contentType = model.GetImageMimeType(ext) - } else { - contentType = "binary/octet-stream" - } - - options := s3PutOptions(b.encrypt, contentType) - var buf bytes.Buffer - _, err = buf.ReadFrom(fr) - if err != nil { - return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - written, err := s3Clnt.PutObject(b.bucket, path, &buf, int64(buf.Len()), options) - if err != nil { - return written, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return written, nil -} - -func (b *S3FileBackend) RemoveFile(path string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return nil -} - -func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { - out := make(chan string, 1) - - go func() { - defer close(out) - - for { - info, done := <-in - - if !done { - break - } - - out <- info.Key - } - }() - - return out -} - -func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { - var paths []string - - s3Clnt, err := b.s3New() - if err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - defer close(doneCh) - - for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { - if object.Err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) - } - paths = append(paths, strings.Trim(object.Key, "/")) - } - - return &paths, nil -} - -func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { - if err.Err != nil { - doneCh <- struct{}{} - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) - } - } - - close(doneCh) - return nil -} - -func s3PutOptions(encrypted bool, contentType string) s3.PutObjectOptions { - options := s3.PutObjectOptions{} - if encrypted { - options.ServerSideEncryption = encrypt.NewSSE() - } - options.ContentType = contentType - - return options -} - -func CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError { - if len(settings.AmazonS3Bucket) == 0 { - return model.NewAppError("S3File", "api.admin.test_s3.missing_s3_bucket", nil, "", http.StatusBadRequest) - } - - // if S3 endpoint is not set call the set defaults to set that - if len(settings.AmazonS3Endpoint) == 0 { - settings.SetDefaults() - } - - return nil -} diff --git a/utils/file_backend_s3_test.go b/utils/file_backend_s3_test.go deleted file mode 100644 index a8834f226..000000000 --- a/utils/file_backend_s3_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2018-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "testing" - - "github.com/mattermost/mattermost-server/model" -) - -func TestCheckMandatoryS3Fields(t *testing.T) { - cfg := model.FileSettings{} - - err := CheckMandatoryS3Fields(&cfg) - if err == nil || err.Message != "api.admin.test_s3.missing_s3_bucket" { - t.Fatal("should've failed with missing s3 bucket") - } - - cfg.AmazonS3Bucket = "test-mm" - err = CheckMandatoryS3Fields(&cfg) - if err != nil { - t.Fatal("should've not failed") - } - - cfg.AmazonS3Endpoint = "" - err = CheckMandatoryS3Fields(&cfg) - if err != nil || cfg.AmazonS3Endpoint != "s3.amazonaws.com" { - t.Fatal("should've not failed because it should set the endpoint to the default") - } - -} diff --git a/utils/file_backend_test.go b/utils/file_backend_test.go deleted file mode 100644 index f7ce7ca61..000000000 --- a/utils/file_backend_test.go +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "testing" - - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/mattermost/mattermost-server/mlog" - "github.com/mattermost/mattermost-server/model" -) - -type FileBackendTestSuite struct { - suite.Suite - - settings model.FileSettings - backend FileBackend -} - -func TestLocalFileBackendTestSuite(t *testing.T) { - // Setup a global logger to catch tests logging outside of app context - // The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen. - mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{ - EnableConsole: true, - ConsoleJson: true, - ConsoleLevel: "error", - EnableFile: false, - })) - - dir, err := ioutil.TempDir("", "") - require.NoError(t, err) - defer os.RemoveAll(dir) - - suite.Run(t, &FileBackendTestSuite{ - settings: model.FileSettings{ - DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL), - Directory: dir, - }, - }) -} - -func TestS3FileBackendTestSuite(t *testing.T) { - runBackendTest(t, false) -} - -func TestS3FileBackendTestSuiteWithEncryption(t *testing.T) { - runBackendTest(t, true) -} - -func runBackendTest(t *testing.T, encrypt bool) { - s3Host := os.Getenv("CI_HOST") - if s3Host == "" { - s3Host = "dockerhost" - } - - s3Port := os.Getenv("CI_MINIO_PORT") - if s3Port == "" { - s3Port = "9001" - } - - s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) - - suite.Run(t, &FileBackendTestSuite{ - settings: model.FileSettings{ - DriverName: model.NewString(model.IMAGE_DRIVER_S3), - AmazonS3AccessKeyId: model.MINIO_ACCESS_KEY, - AmazonS3SecretAccessKey: model.MINIO_SECRET_KEY, - AmazonS3Bucket: model.MINIO_BUCKET, - AmazonS3Endpoint: s3Endpoint, - AmazonS3SSL: model.NewBool(false), - AmazonS3SSE: model.NewBool(encrypt), - }, - }) -} - -func (s *FileBackendTestSuite) SetupTest() { - TranslationsPreInit() - - backend, err := NewFileBackend(&s.settings, true) - require.Nil(s.T(), err) - s.backend = backend -} - -func (s *FileBackendTestSuite) TestConnection() { - s.Nil(s.backend.TestConnection()) -} - -func (s *FileBackendTestSuite) TestReadWriteFile() { - b := []byte("test") - path := "tests/" + model.NewId() - - written, err := s.backend.WriteFile(bytes.NewReader(b), path) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - defer s.backend.RemoveFile(path) - - read, err := s.backend.ReadFile(path) - s.Nil(err) - - readString := string(read) - s.EqualValues(readString, "test") -} - -func (s *FileBackendTestSuite) TestReadWriteFileImage() { - b := []byte("testimage") - path := "tests/" + model.NewId() + ".png" - - written, err := s.backend.WriteFile(bytes.NewReader(b), path) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - defer s.backend.RemoveFile(path) - - read, err := s.backend.ReadFile(path) - s.Nil(err) - - readString := string(read) - s.EqualValues(readString, "testimage") -} - -func (s *FileBackendTestSuite) TestFileExists() { - b := []byte("testimage") - path := "tests/" + model.NewId() + ".png" - - _, err := s.backend.WriteFile(bytes.NewReader(b), path) - s.Nil(err) - defer s.backend.RemoveFile(path) - - res, err := s.backend.FileExists(path) - s.Nil(err) - s.True(res) - - res, err = s.backend.FileExists("tests/idontexist.png") - s.Nil(err) - s.False(res) -} - -func (s *FileBackendTestSuite) TestCopyFile() { - b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/" + model.NewId() - - written, err := s.backend.WriteFile(bytes.NewReader(b), path1) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - defer s.backend.RemoveFile(path1) - - err = s.backend.CopyFile(path1, path2) - s.Nil(err) - defer s.backend.RemoveFile(path2) - - _, err = s.backend.ReadFile(path1) - s.Nil(err) - - _, err = s.backend.ReadFile(path2) - s.Nil(err) -} - -func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() { - b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/newdirectory/" + model.NewId() - - written, err := s.backend.WriteFile(bytes.NewReader(b), path1) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - defer s.backend.RemoveFile(path1) - - err = s.backend.CopyFile(path1, path2) - s.Nil(err) - defer s.backend.RemoveFile(path2) - - _, err = s.backend.ReadFile(path1) - s.Nil(err) - - _, err = s.backend.ReadFile(path2) - s.Nil(err) -} - -func (s *FileBackendTestSuite) TestMoveFile() { - b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/" + model.NewId() - - written, err := s.backend.WriteFile(bytes.NewReader(b), path1) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - defer s.backend.RemoveFile(path1) - - s.Nil(s.backend.MoveFile(path1, path2)) - defer s.backend.RemoveFile(path2) - - _, err = s.backend.ReadFile(path1) - s.Error(err) - - _, err = s.backend.ReadFile(path2) - s.Nil(err) -} - -func (s *FileBackendTestSuite) TestRemoveFile() { - b := []byte("test") - path := "tests/" + model.NewId() - - written, err := s.backend.WriteFile(bytes.NewReader(b), path) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - s.Nil(s.backend.RemoveFile(path)) - - _, err = s.backend.ReadFile(path) - s.Error(err) - - written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/foo") - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - - written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar") - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - - written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/asdf") - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - - s.Nil(s.backend.RemoveDirectory("tests2")) -} - -func (s *FileBackendTestSuite) TestListDirectory() { - b := []byte("test") - path1 := "19700101/" + model.NewId() - path2 := "19800101/" + model.NewId() - - written, err := s.backend.WriteFile(bytes.NewReader(b), path1) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - defer s.backend.RemoveFile(path1) - - written, err = s.backend.WriteFile(bytes.NewReader(b), path2) - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - defer s.backend.RemoveFile(path2) - - paths, err := s.backend.ListDirectory("") - s.Nil(err) - - found1 := false - found2 := false - for _, path := range *paths { - if path == "19700101" { - found1 = true - } else if path == "19800101" { - found2 = true - } - } - s.True(found1) - s.True(found2) -} - -func (s *FileBackendTestSuite) TestRemoveDirectory() { - b := []byte("test") - - written, err := s.backend.WriteFile(bytes.NewReader(b), "tests2/foo") - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - - written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar") - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - - written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/aaa") - s.Nil(err) - s.EqualValues(len(b), written, "expected given number of bytes to have been written") - - s.Nil(s.backend.RemoveDirectory("tests2")) - - _, err = s.backend.ReadFile("tests2/foo") - s.Error(err) - _, err = s.backend.ReadFile("tests2/bar") - s.Error(err) - _, err = s.backend.ReadFile("tests2/asdf") - s.Error(err) -} diff --git a/utils/inbucket.go b/utils/inbucket.go deleted file mode 100644 index 5c40d5757..000000000 --- a/utils/inbucket.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "strings" - "time" -) - -const ( - INBUCKET_API = "/api/v1/mailbox/" -) - -// OutputJSONHeader holds the received Header to test sending emails (inbucket) -type JSONMessageHeaderInbucket []struct { - Mailbox string - ID string `json:"Id"` - From, Subject, Date string - To []string - Size int -} - -// OutputJSONMessage holds the received Message fto test sending emails (inbucket) -type JSONMessageInbucket struct { - Mailbox string - ID string `json:"Id"` - From, Subject, Date string - Size int - Header map[string][]string - Body struct { - Text string - HTML string `json:"Html"` - } - Attachments []struct { - Filename string - ContentType string `json:"content-type"` - DownloadLink string `json:"download-link"` - Bytes []byte `json:"-"` - } -} - -func ParseEmail(email string) string { - pos := strings.Index(email, "@") - parsedEmail := email[0:pos] - return parsedEmail -} - -func GetMailBox(email string) (results JSONMessageHeaderInbucket, err error) { - - parsedEmail := ParseEmail(email) - - url := fmt.Sprintf("%s%s%s", getInbucketHost(), INBUCKET_API, parsedEmail) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - - client := &http.Client{} - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.Body == nil { - return nil, fmt.Errorf("No Mailbox") - } - - var record JSONMessageHeaderInbucket - err = json.NewDecoder(resp.Body).Decode(&record) - switch { - case err == io.EOF: - return nil, fmt.Errorf("Error: %s", err) - case err != nil: - return nil, fmt.Errorf("Error: %s", err) - } - if len(record) == 0 { - return nil, fmt.Errorf("No mailbox") - } - - return record, nil -} - -func GetMessageFromMailbox(email, id string) (results JSONMessageInbucket, err error) { - - parsedEmail := ParseEmail(email) - - var record JSONMessageInbucket - - url := fmt.Sprintf("%s%s%s/%s", getInbucketHost(), INBUCKET_API, parsedEmail, id) - emailResponse, err := get(url) - if err != nil { - return record, err - } - defer emailResponse.Body.Close() - - err = json.NewDecoder(emailResponse.Body).Decode(&record) - - // download attachments - if record.Attachments != nil && len(record.Attachments) > 0 { - for i := range record.Attachments { - if bytes, err := downloadAttachment(record.Attachments[i].DownloadLink); err != nil { - return record, err - } else { - record.Attachments[i].Bytes = make([]byte, len(bytes)) - copy(record.Attachments[i].Bytes, bytes) - } - } - } - - return record, err -} - -func downloadAttachment(url string) ([]byte, error) { - attachmentResponse, err := get(url) - if err != nil { - return nil, err - } - defer attachmentResponse.Body.Close() - - buf := new(bytes.Buffer) - io.Copy(buf, attachmentResponse.Body) - return buf.Bytes(), nil -} - -func get(url string) (*http.Response, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, err - } - - return resp, nil -} - -func DeleteMailBox(email string) (err error) { - - parsedEmail := ParseEmail(email) - - url := fmt.Sprintf("%s%s%s", getInbucketHost(), INBUCKET_API, parsedEmail) - req, err := http.NewRequest("DELETE", url, nil) - if err != nil { - return err - } - - client := &http.Client{} - - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - return nil -} - -func RetryInbucket(attempts int, callback func() error) (err error) { - for i := 0; ; i++ { - err = callback() - if err == nil { - return nil - } - - if i >= (attempts - 1) { - break - } - - time.Sleep(5 * time.Second) - - fmt.Println("retrying...") - } - return fmt.Errorf("After %d attempts, last error: %s", attempts, err) -} - -func getInbucketHost() (host string) { - - inbucket_host := os.Getenv("CI_HOST") - if inbucket_host == "" { - inbucket_host = "dockerhost" - } - - inbucket_port := os.Getenv("CI_INBUCKET_PORT") - if inbucket_port == "" { - inbucket_port = "9000" - } - return fmt.Sprintf("http://%s:%s", inbucket_host, inbucket_port) -} diff --git a/utils/mail.go b/utils/mail.go deleted file mode 100644 index 750cb64fe..000000000 --- a/utils/mail.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "crypto/tls" - "errors" - "fmt" - "io" - "mime" - "net" - "net/mail" - "net/smtp" - "time" - - "gopkg.in/gomail.v2" - - "net/http" - - "github.com/jaytaylor/html2text" - "github.com/mattermost/mattermost-server/mlog" - "github.com/mattermost/mattermost-server/model" -) - -func encodeRFC2047Word(s string) string { - return mime.BEncoding.Encode("utf-8", s) -} - -type SmtpConnectionInfo struct { - SmtpUsername string - SmtpPassword string - SmtpServerName string - SmtpServerHost string - SmtpPort string - SkipCertVerification bool - ConnectionSecurity string - Auth bool -} - -type authChooser struct { - smtp.Auth - connectionInfo *SmtpConnectionInfo -} - -func (a *authChooser) Start(server *smtp.ServerInfo) (string, []byte, error) { - smtpAddress := a.connectionInfo.SmtpServerName + ":" + a.connectionInfo.SmtpPort - a.Auth = LoginAuth(a.connectionInfo.SmtpUsername, a.connectionInfo.SmtpPassword, smtpAddress) - for _, method := range server.Auth { - if method == "PLAIN" { - a.Auth = smtp.PlainAuth("", a.connectionInfo.SmtpUsername, a.connectionInfo.SmtpPassword, a.connectionInfo.SmtpServerName+":"+a.connectionInfo.SmtpPort) - break - } - } - return a.Auth.Start(server) -} - -type loginAuth struct { - username, password, host string -} - -func LoginAuth(username, password, host string) smtp.Auth { - return &loginAuth{username, password, host} -} - -func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) { - if !server.TLS { - return "", nil, errors.New("unencrypted connection") - } - - if server.Name != a.host { - return "", nil, errors.New("wrong host name") - } - - return "LOGIN", []byte{}, nil -} - -func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) { - if more { - switch string(fromServer) { - case "Username:": - return []byte(a.username), nil - case "Password:": - return []byte(a.password), nil - default: - return nil, errors.New("Unknown fromServer") - } - } - return nil, nil -} - -func ConnectToSMTPServerAdvanced(connectionInfo *SmtpConnectionInfo) (net.Conn, *model.AppError) { - var conn net.Conn - var err error - - smtpAddress := connectionInfo.SmtpServerHost + ":" + connectionInfo.SmtpPort - if connectionInfo.ConnectionSecurity == model.CONN_SECURITY_TLS { - tlsconfig := &tls.Config{ - InsecureSkipVerify: connectionInfo.SkipCertVerification, - ServerName: connectionInfo.SmtpServerName, - } - - conn, err = tls.Dial("tcp", smtpAddress, tlsconfig) - if err != nil { - return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.open_tls.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - conn, err = net.Dial("tcp", smtpAddress) - if err != nil { - return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.open.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } - - return conn, nil -} - -func ConnectToSMTPServer(config *model.Config) (net.Conn, *model.AppError) { - return ConnectToSMTPServerAdvanced( - &SmtpConnectionInfo{ - ConnectionSecurity: config.EmailSettings.ConnectionSecurity, - SkipCertVerification: *config.EmailSettings.SkipServerCertificateVerification, - SmtpServerName: config.EmailSettings.SMTPServer, - SmtpServerHost: config.EmailSettings.SMTPServer, - SmtpPort: config.EmailSettings.SMTPPort, - }, - ) -} - -func NewSMTPClientAdvanced(conn net.Conn, hostname string, connectionInfo *SmtpConnectionInfo) (*smtp.Client, *model.AppError) { - c, err := smtp.NewClient(conn, connectionInfo.SmtpServerName+":"+connectionInfo.SmtpPort) - if err != nil { - mlog.Error(fmt.Sprintf("Failed to open a connection to SMTP server %v", err)) - return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.open_tls.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if hostname != "" { - err := c.Hello(hostname) - if err != nil { - mlog.Error(fmt.Sprintf("Failed to to set the HELO to SMTP server %v", err)) - return nil, model.NewAppError("SendMail", "utils.mail.connect_smtp.helo.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } - - if connectionInfo.ConnectionSecurity == model.CONN_SECURITY_STARTTLS { - tlsconfig := &tls.Config{ - InsecureSkipVerify: connectionInfo.SkipCertVerification, - ServerName: connectionInfo.SmtpServerName, - } - c.StartTLS(tlsconfig) - } - - if connectionInfo.Auth { - if err = c.Auth(&authChooser{connectionInfo: connectionInfo}); err != nil { - return nil, model.NewAppError("SendMail", "utils.mail.new_client.auth.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } - return c, nil -} - -func NewSMTPClient(conn net.Conn, config *model.Config) (*smtp.Client, *model.AppError) { - return NewSMTPClientAdvanced( - conn, - GetHostnameFromSiteURL(*config.ServiceSettings.SiteURL), - &SmtpConnectionInfo{ - ConnectionSecurity: config.EmailSettings.ConnectionSecurity, - SkipCertVerification: *config.EmailSettings.SkipServerCertificateVerification, - SmtpServerName: config.EmailSettings.SMTPServer, - SmtpServerHost: config.EmailSettings.SMTPServer, - SmtpPort: config.EmailSettings.SMTPPort, - Auth: *config.EmailSettings.EnableSMTPAuth, - SmtpUsername: config.EmailSettings.SMTPUsername, - SmtpPassword: config.EmailSettings.SMTPPassword, - }, - ) -} - -func TestConnection(config *model.Config) { - if !config.EmailSettings.SendEmailNotifications { - return - } - - conn, err1 := ConnectToSMTPServer(config) - if err1 != nil { - mlog.Error(fmt.Sprintf("SMTP server settings do not appear to be configured properly err=%v details=%v", T(err1.Message), err1.DetailedError)) - return - } - defer conn.Close() - - c, err2 := NewSMTPClient(conn, config) - if err2 != nil { - mlog.Error(fmt.Sprintf("SMTP server settings do not appear to be configured properly err=%v details=%v", T(err2.Message), err2.DetailedError)) - return - } - defer c.Quit() - defer c.Close() -} - -func SendMailUsingConfig(to, subject, htmlBody string, config *model.Config, enableComplianceFeatures bool) *model.AppError { - fromMail := mail.Address{Name: config.EmailSettings.FeedbackName, Address: config.EmailSettings.FeedbackEmail} - - return SendMailUsingConfigAdvanced(to, to, fromMail, subject, htmlBody, nil, nil, config, enableComplianceFeatures) -} - -// allows for sending an email with attachments and differing MIME/SMTP recipients -func SendMailUsingConfigAdvanced(mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, config *model.Config, enableComplianceFeatures bool) *model.AppError { - if !config.EmailSettings.SendEmailNotifications || len(config.EmailSettings.SMTPServer) == 0 { - return nil - } - - conn, err := ConnectToSMTPServer(config) - if err != nil { - return err - } - defer conn.Close() - - c, err := NewSMTPClient(conn, config) - if err != nil { - return err - } - defer c.Quit() - defer c.Close() - - fileBackend, err := NewFileBackend(&config.FileSettings, enableComplianceFeatures) - if err != nil { - return err - } - - return SendMail(c, mimeTo, smtpTo, from, subject, htmlBody, attachments, mimeHeaders, fileBackend, time.Now()) -} - -func SendMail(c *smtp.Client, mimeTo, smtpTo string, from mail.Address, subject, htmlBody string, attachments []*model.FileInfo, mimeHeaders map[string]string, fileBackend FileBackend, date time.Time) *model.AppError { - mlog.Debug(fmt.Sprintf("sending mail to %v with subject of '%v'", smtpTo, subject)) - - htmlMessage := "\r\n" + htmlBody + "" - - txtBody, err := html2text.FromString(htmlBody) - if err != nil { - mlog.Warn(fmt.Sprint(err)) - txtBody = "" - } - - headers := map[string][]string{ - "From": {from.String()}, - "To": {mimeTo}, - "Subject": {encodeRFC2047Word(subject)}, - "Content-Transfer-Encoding": {"8bit"}, - "Auto-Submitted": {"auto-generated"}, - "Precedence": {"bulk"}, - } - for k, v := range mimeHeaders { - headers[k] = []string{encodeRFC2047Word(v)} - } - - m := gomail.NewMessage(gomail.SetCharset("UTF-8")) - m.SetHeaders(headers) - m.SetDateHeader("Date", date) - m.SetBody("text/plain", txtBody) - m.AddAlternative("text/html", htmlMessage) - - for _, fileInfo := range attachments { - bytes, err := fileBackend.ReadFile(fileInfo.Path) - if err != nil { - return err - } - - m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error { - if _, err := writer.Write(bytes); err != nil { - return model.NewAppError("SendMail", "utils.mail.sendMail.attachments.write_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil - })) - } - - if err := c.Mail(from.Address); err != nil { - return model.NewAppError("SendMail", "utils.mail.send_mail.from_address.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if err := c.Rcpt(smtpTo); err != nil { - return model.NewAppError("SendMail", "utils.mail.send_mail.to_address.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - w, err := c.Data() - if err != nil { - return model.NewAppError("SendMail", "utils.mail.send_mail.msg_data.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - _, err = m.WriteTo(w) - if err != nil { - return model.NewAppError("SendMail", "utils.mail.send_mail.msg.app_error", nil, err.Error(), http.StatusInternalServerError) - } - err = w.Close() - if err != nil { - return model.NewAppError("SendMail", "utils.mail.send_mail.close.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return nil -} diff --git a/utils/mail_test.go b/utils/mail_test.go deleted file mode 100644 index 4cb2d7594..000000000 --- a/utils/mail_test.go +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "bytes" - "fmt" - "strings" - "testing" - - "net/mail" - "net/smtp" - - "github.com/mattermost/mattermost-server/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMailConnectionFromConfig(t *testing.T) { - cfg, _, _, err := LoadConfig("config.json") - require.Nil(t, err) - - if conn, err := ConnectToSMTPServer(cfg); err != nil { - t.Log(err) - t.Fatal("Should connect to the STMP Server") - } else { - if _, err1 := NewSMTPClient(conn, cfg); err1 != nil { - t.Log(err) - t.Fatal("Should get new smtp client") - } - } - - cfg.EmailSettings.SMTPServer = "wrongServer" - cfg.EmailSettings.SMTPPort = "553" - - if _, err := ConnectToSMTPServer(cfg); err == nil { - t.Log(err) - t.Fatal("Should not to the STMP Server") - } -} - -func TestMailConnectionAdvanced(t *testing.T) { - cfg, _, _, err := LoadConfig("config.json") - require.Nil(t, err) - - if conn, err := ConnectToSMTPServerAdvanced( - &SmtpConnectionInfo{ - ConnectionSecurity: cfg.EmailSettings.ConnectionSecurity, - SkipCertVerification: *cfg.EmailSettings.SkipServerCertificateVerification, - SmtpServerName: cfg.EmailSettings.SMTPServer, - SmtpServerHost: cfg.EmailSettings.SMTPServer, - SmtpPort: cfg.EmailSettings.SMTPPort, - }, - ); err != nil { - t.Log(err) - t.Fatal("Should connect to the STMP Server") - } else { - if _, err1 := NewSMTPClientAdvanced( - conn, - GetHostnameFromSiteURL(*cfg.ServiceSettings.SiteURL), - &SmtpConnectionInfo{ - ConnectionSecurity: cfg.EmailSettings.ConnectionSecurity, - SkipCertVerification: *cfg.EmailSettings.SkipServerCertificateVerification, - SmtpServerName: cfg.EmailSettings.SMTPServer, - SmtpServerHost: cfg.EmailSettings.SMTPServer, - SmtpPort: cfg.EmailSettings.SMTPPort, - Auth: *cfg.EmailSettings.EnableSMTPAuth, - SmtpUsername: cfg.EmailSettings.SMTPUsername, - SmtpPassword: cfg.EmailSettings.SMTPPassword, - }, - ); err1 != nil { - t.Log(err) - t.Fatal("Should get new smtp client") - } - } - - if _, err := ConnectToSMTPServerAdvanced( - &SmtpConnectionInfo{ - ConnectionSecurity: cfg.EmailSettings.ConnectionSecurity, - SkipCertVerification: *cfg.EmailSettings.SkipServerCertificateVerification, - SmtpServerName: "wrongServer", - SmtpServerHost: "wrongServer", - SmtpPort: "553", - }, - ); err == nil { - t.Log(err) - t.Fatal("Should not to the STMP Server") - } - -} - -func TestSendMailUsingConfig(t *testing.T) { - cfg, _, _, err := LoadConfig("config.json") - require.Nil(t, err) - T = GetUserTranslations("en") - - var emailTo = "test@example.com" - var emailSubject = "Testing this email" - var emailBody = "This is a test from autobot" - - //Delete all the messages before check the sample email - DeleteMailBox(emailTo) - - if err := SendMailUsingConfig(emailTo, emailSubject, emailBody, cfg, true); err != nil { - t.Log(err) - t.Fatal("Should connect to the STMP Server") - } else { - //Check if the email was send to the right email address - var resultsMailbox JSONMessageHeaderInbucket - err := RetryInbucket(5, func() error { - var err error - resultsMailbox, err = GetMailBox(emailTo) - return err - }) - if err != nil { - t.Log(err) - t.Log("No email was received, maybe due load on the server. Disabling this verification") - } - if err == nil && len(resultsMailbox) > 0 { - if !strings.ContainsAny(resultsMailbox[0].To[0], emailTo) { - t.Fatal("Wrong To recipient") - } else { - if resultsEmail, err := GetMessageFromMailbox(emailTo, resultsMailbox[0].ID); err == nil { - if !strings.Contains(resultsEmail.Body.Text, emailBody) { - t.Log(resultsEmail.Body.Text) - t.Fatal("Received message") - } - } - } - } - } -} - -func TestSendMailUsingConfigAdvanced(t *testing.T) { - cfg, _, _, err := LoadConfig("config.json") - require.Nil(t, err) - T = GetUserTranslations("en") - - var mimeTo = "test@example.com" - var smtpTo = "test2@example.com" - var from = mail.Address{Name: "Nobody", Address: "nobody@mattermost.com"} - var emailSubject = "Testing this email" - var emailBody = "This is a test from autobot" - - //Delete all the messages before check the sample email - DeleteMailBox(smtpTo) - - fileBackend, err := NewFileBackend(&cfg.FileSettings, true) - assert.Nil(t, err) - - // create two files with the same name that will both be attached to the email - fileName := "file.txt" - filePath1 := fmt.Sprintf("test1/%s", fileName) - filePath2 := fmt.Sprintf("test2/%s", fileName) - fileContents1 := []byte("hello world") - fileContents2 := []byte("foo bar") - _, err = fileBackend.WriteFile(bytes.NewReader(fileContents1), filePath1) - assert.Nil(t, err) - _, err = fileBackend.WriteFile(bytes.NewReader(fileContents2), filePath2) - assert.Nil(t, err) - defer fileBackend.RemoveFile(filePath1) - defer fileBackend.RemoveFile(filePath2) - - attachments := make([]*model.FileInfo, 2) - attachments[0] = &model.FileInfo{ - Name: fileName, - Path: filePath1, - } - attachments[1] = &model.FileInfo{ - Name: fileName, - Path: filePath2, - } - - headers := make(map[string]string) - headers["TestHeader"] = "TestValue" - - if err := SendMailUsingConfigAdvanced(mimeTo, smtpTo, from, emailSubject, emailBody, attachments, headers, cfg, true); err != nil { - t.Log(err) - t.Fatal("Should connect to the STMP Server") - } else { - //Check if the email was send to the right email address - var resultsMailbox JSONMessageHeaderInbucket - err := RetryInbucket(5, func() error { - var err error - resultsMailbox, err = GetMailBox(smtpTo) - return err - }) - if err != nil { - t.Log(err) - t.Fatal("No emails found for address " + smtpTo) - } - if err == nil && len(resultsMailbox) > 0 { - if !strings.ContainsAny(resultsMailbox[0].To[0], smtpTo) { - t.Fatal("Wrong To recipient") - } else { - if resultsEmail, err := GetMessageFromMailbox(smtpTo, resultsMailbox[0].ID); err == nil { - if !strings.Contains(resultsEmail.Body.Text, emailBody) { - t.Log(resultsEmail.Body.Text) - t.Fatal("Received message") - } - - // verify that the To header of the email message is set to the MIME recipient, even though we got it out of the SMTP recipient's email inbox - assert.Equal(t, mimeTo, resultsEmail.Header["To"][0]) - - // verify that the MIME from address is correct - unfortunately, we can't verify the SMTP from address - assert.Equal(t, from.String(), resultsEmail.Header["From"][0]) - - // check that the custom mime headers came through - header case seems to get mutated - assert.Equal(t, "TestValue", resultsEmail.Header["Testheader"][0]) - - // ensure that the attachments were successfully sent - assert.Len(t, resultsEmail.Attachments, 2) - assert.Equal(t, fileName, resultsEmail.Attachments[0].Filename) - assert.Equal(t, fileName, resultsEmail.Attachments[1].Filename) - attachment1 := string(resultsEmail.Attachments[0].Bytes) - attachment2 := string(resultsEmail.Attachments[1].Bytes) - if attachment1 == string(fileContents1) { - assert.Equal(t, attachment2, string(fileContents2)) - } else if attachment1 == string(fileContents2) { - assert.Equal(t, attachment2, string(fileContents1)) - } else { - assert.Fail(t, "Unrecognized attachment contents") - } - } - } - } - } -} - -func TestAuthMethods(t *testing.T) { - auth := &authChooser{ - connectionInfo: &SmtpConnectionInfo{ - SmtpUsername: "test", - SmtpPassword: "fakepass", - SmtpServerName: "fakeserver", - SmtpServerHost: "fakeserver", - SmtpPort: "25", - }, - } - tests := []struct { - desc string - server *smtp.ServerInfo - err string - }{ - { - desc: "auth PLAIN success", - server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: true}, - }, - { - desc: "auth PLAIN unencrypted connection fail", - server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: false}, - err: "unencrypted connection", - }, - { - desc: "auth PLAIN wrong host name", - server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"PLAIN"}, TLS: true}, - err: "wrong host name", - }, - { - desc: "auth LOGIN success", - server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: true}, - }, - { - desc: "auth LOGIN unencrypted connection fail", - server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"LOGIN"}, TLS: true}, - err: "wrong host name", - }, - { - desc: "auth LOGIN wrong host name", - server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: false}, - err: "unencrypted connection", - }, - } - - for i, test := range tests { - t.Run(test.desc, func(t *testing.T) { - _, _, err := auth.Start(test.server) - got := "" - if err != nil { - got = err.Error() - } - if got != test.err { - t.Errorf("%d. got error = %q; want %q", i, got, test.err) - } - }) - } -} -- cgit v1.2.3-1-g7c22