From eb1a00ef5f93b19c2d49b26de057ee2c51c09e45 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 16 Nov 2017 15:04:27 -0600 Subject: Reorganize file util functionality (#7848) * reorganize file util functionality * fix api test compilation * fix rebase issue --- api/emoji.go | 5 +- api/emoji_test.go | 2 +- api/file.go | 10 +- api/file_test.go | 2 +- api4/emoji.go | 3 +- api4/file.go | 8 +- app/app.go | 30 +++- app/emoji.go | 22 ++- app/file.go | 62 ++++++-- app/file_test.go | 9 +- app/import.go | 4 +- app/user.go | 6 +- cmd/platform/server.go | 12 +- einterfaces/brand.go | 10 -- einterfaces/elasticsearch.go | 10 -- einterfaces/emoji.go | 10 -- utils/file.go | 359 ------------------------------------------- utils/file_backend.go | 44 ++++++ utils/file_backend_local.go | 98 ++++++++++++ utils/file_backend_s3.go | 226 +++++++++++++++++++++++++++ utils/file_backend_test.go | 164 ++++++++++++++++++++ utils/file_test.go | 172 --------------------- 22 files changed, 649 insertions(+), 619 deletions(-) create mode 100644 utils/file_backend.go create mode 100644 utils/file_backend_local.go create mode 100644 utils/file_backend_s3.go create mode 100644 utils/file_backend_test.go diff --git a/api/emoji.go b/api/emoji.go index 4edbbd082..cbe7b07bf 100644 --- a/api/emoji.go +++ b/api/emoji.go @@ -16,7 +16,6 @@ import ( "github.com/disintegration/imaging" "github.com/gorilla/mux" "github.com/mattermost/mattermost-server/app" - "github.com/mattermost/mattermost-server/einterfaces" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" ) @@ -51,7 +50,7 @@ func createEmoji(c *Context, w http.ResponseWriter, r *http.Request) { return } - if emojiInterface := einterfaces.GetEmojiInterface(); emojiInterface != nil && + if emojiInterface := c.App.Emoji; emojiInterface != nil && !emojiInterface.CanUserCreateEmoji(c.Session.Roles, c.Session.TeamMembers) { c.Err = model.NewAppError("createEmoji", "api.emoji.create.permissions.app_error", nil, "user_id="+c.Session.UserId, http.StatusUnauthorized) return @@ -106,7 +105,7 @@ func createEmoji(c *Context, w http.ResponseWriter, r *http.Request) { if imageData := m.File["image"]; len(imageData) == 0 { c.SetInvalidParam("createEmoji", "image") return - } else if err := app.UploadEmojiImage(emoji.Id, imageData[0]); err != nil { + } else if err := c.App.UploadEmojiImage(emoji.Id, imageData[0]); err != nil { c.Err = err return } diff --git a/api/emoji_test.go b/api/emoji_test.go index f4e376bd2..69922a54a 100644 --- a/api/emoji_test.go +++ b/api/emoji_test.go @@ -272,7 +272,7 @@ func TestDeleteEmoji(t *testing.T) { func createTestEmoji(t *testing.T, a *app.App, emoji *model.Emoji, imageData []byte) *model.Emoji { emoji = store.Must(a.Srv.Store.Emoji().Save(emoji)).(*model.Emoji) - if err := utils.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil { + if err := a.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil { store.Must(a.Srv.Store.Emoji().Delete(emoji.Id, time.Now().Unix())) t.Fatalf("failed to write image: %v", err.Error()) } diff --git a/api/file.go b/api/file.go index 20b13fcb7..75739486c 100644 --- a/api/file.go +++ b/api/file.go @@ -95,7 +95,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.Path); err != nil { + if data, err := c.App.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil { @@ -116,7 +116,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.ThumbnailPath); err != nil { + if data, err := c.App.ReadFile(info.ThumbnailPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, w, r); err != nil { @@ -137,7 +137,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.PreviewPath); err != nil { + if data, err := c.App.ReadFile(info.PreviewPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, w, r); err != nil { @@ -186,7 +186,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.Path); err != nil { + if data, err := c.App.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil { @@ -277,7 +277,7 @@ func getPublicFileOld(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.Path); err != nil { + if data, err := c.App.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil { diff --git a/api/file_test.go b/api/file_test.go index 8b04c732c..8e5fc6f67 100644 --- a/api/file_test.go +++ b/api/file_test.go @@ -849,7 +849,7 @@ func TestGetInfoForFilename(t *testing.T) { date := time.Now().Format("20060102") - if info := app.GetInfoForFilename(post1, team1.Id, post1.Filenames[0]); info == nil { + if info := th.App.GetInfoForFilename(post1, team1.Id, post1.Filenames[0]); info == nil { t.Fatal("info shouldn't be nil") } else if info.Id == "" { t.Fatal("info.Id shouldn't be empty") diff --git a/api4/emoji.go b/api4/emoji.go index 7a05e1ae1..e96e0755f 100644 --- a/api4/emoji.go +++ b/api4/emoji.go @@ -10,7 +10,6 @@ import ( l4g "github.com/alecthomas/log4go" "github.com/mattermost/mattermost-server/app" - "github.com/mattermost/mattermost-server/einterfaces" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" ) @@ -31,7 +30,7 @@ func createEmoji(c *Context, w http.ResponseWriter, r *http.Request) { return } - if emojiInterface := einterfaces.GetEmojiInterface(); emojiInterface != nil && + if emojiInterface := c.App.Emoji; emojiInterface != nil && !emojiInterface.CanUserCreateEmoji(c.Session.Roles, c.Session.TeamMembers) { c.Err = model.NewAppError("getEmoji", "api.emoji.disabled.app_error", nil, "user_id="+c.Session.UserId, http.StatusUnauthorized) return diff --git a/api4/file.go b/api4/file.go index 883d4f3c8..7e13d0290 100644 --- a/api4/file.go +++ b/api4/file.go @@ -123,7 +123,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - data, err := utils.ReadFile(info.Path) + data, err := c.App.ReadFile(info.Path) if err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound @@ -164,7 +164,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.ThumbnailPath); err != nil { + if data, err := c.App.ReadFile(info.ThumbnailPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, forceDownload, w, r); err != nil { @@ -233,7 +233,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.PreviewPath); err != nil { + if data, err := c.App.ReadFile(info.PreviewPath); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, forceDownload, w, r); err != nil { @@ -294,7 +294,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - if data, err := utils.ReadFile(info.Path); err != nil { + if data, err := c.App.ReadFile(info.Path); err != nil { c.Err = err c.Err.StatusCode = http.StatusNotFound } else if err := writeFileResponse(info.Name, info.MimeType, data, true, w, r); err != nil { diff --git a/app/app.go b/app/app.go index 4be897f59..49ac620e8 100644 --- a/app/app.go +++ b/app/app.go @@ -43,6 +43,7 @@ type App struct { Compliance einterfaces.ComplianceInterface DataRetention einterfaces.DataRetentionInterface Elasticsearch einterfaces.ElasticsearchInterface + Emoji einterfaces.EmojiInterface Ldap einterfaces.LdapInterface Metrics einterfaces.MetricsInterface Mfa einterfaces.MfaInterface @@ -133,6 +134,12 @@ func RegisterAccountMigrationInterface(f func(*App) einterfaces.AccountMigration accountMigrationInterface = f } +var brandInterface func(*App) einterfaces.BrandInterface + +func RegisterBrandInterface(f func(*App) einterfaces.BrandInterface) { + brandInterface = f +} + var clusterInterface func(*App) einterfaces.ClusterInterface func RegisterClusterInterface(f func(*App) einterfaces.ClusterInterface) { @@ -151,6 +158,18 @@ func RegisterDataRetentionInterface(f func(*App) einterfaces.DataRetentionInterf dataRetentionInterface = f } +var elasticsearchInterface func(*App) einterfaces.ElasticsearchInterface + +func RegisterElasticsearchInterface(f func(*App) einterfaces.ElasticsearchInterface) { + elasticsearchInterface = f +} + +var emojiInterface func(*App) einterfaces.EmojiInterface + +func RegisterEmojiInterface(f func(*App) einterfaces.EmojiInterface) { + emojiInterface = f +} + var jobsDataRetentionJobInterface func(*App) ejobs.DataRetentionJobInterface func RegisterJobsDataRetentionJobInterface(f func(*App) ejobs.DataRetentionJobInterface) { @@ -203,14 +222,21 @@ func (a *App) initEnterprise() { if accountMigrationInterface != nil { a.AccountMigration = accountMigrationInterface(a) } - a.Brand = einterfaces.GetBrandInterface() + if brandInterface != nil { + a.Brand = brandInterface(a) + } if clusterInterface != nil { a.Cluster = clusterInterface(a) } if complianceInterface != nil { a.Compliance = complianceInterface(a) } - a.Elasticsearch = einterfaces.GetElasticsearchInterface() + if elasticsearchInterface != nil { + a.Elasticsearch = elasticsearchInterface(a) + } + if emojiInterface != nil { + a.Emoji = emojiInterface(a) + } if ldapInterface != nil { a.Ldap = ldapInterface(a) utils.AddConfigListener(func(_, cfg *model.Config) { diff --git a/app/emoji.go b/app/emoji.go index ba2bb4494..f62a8686b 100644 --- a/app/emoji.go +++ b/app/emoji.go @@ -51,7 +51,7 @@ func (a *App) CreateEmoji(sessionUserId string, emoji *model.Emoji, multiPartIma if imageData := multiPartImageData.File["image"]; len(imageData) == 0 { err := model.NewAppError("Context", "api.context.invalid_body_param.app_error", map[string]interface{}{"Name": "createEmoji"}, "", http.StatusBadRequest) return nil, err - } else if err := UploadEmojiImage(emoji.Id, imageData[0]); err != nil { + } else if err := a.UploadEmojiImage(emoji.Id, imageData[0]); err != nil { return nil, err } @@ -74,7 +74,7 @@ func (a *App) GetEmojiList(page, perPage int) ([]*model.Emoji, *model.AppError) } } -func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppError { +func (a *App) UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppError { file, err := imageData.Open() if err != nil { return model.NewAppError("uploadEmojiImage", "api.emoji.upload.open.app_error", nil, "", http.StatusBadRequest) @@ -100,7 +100,7 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro if err := gif.EncodeAll(newbuf, resized_gif); err != nil { return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_encode_error", nil, "", http.StatusBadRequest) } - if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := a.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } @@ -112,18 +112,14 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro if err := png.Encode(newbuf, resized_image); err != nil { return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.encode_error", nil, "", http.StatusBadRequest) } - if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { + if err := a.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil { return err } } } - } else { - if err := utils.WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil { - return err - } } - return nil + return a.WriteFile(buf.Bytes(), getEmojiImagePath(id)) } func (a *App) DeleteEmoji(emoji *model.Emoji) *model.AppError { @@ -131,7 +127,7 @@ func (a *App) DeleteEmoji(emoji *model.Emoji) *model.AppError { return err } - deleteEmojiImage(emoji.Id) + a.deleteEmojiImage(emoji.Id) a.deleteReactionsForEmoji(emoji.Name) return nil } @@ -158,7 +154,7 @@ func (a *App) GetEmojiImage(emojiId string) (imageByte []byte, imageType string, } else { var img []byte - if data, err := utils.ReadFile(getEmojiImagePath(emojiId)); err != nil { + if data, err := a.ReadFile(getEmojiImagePath(emojiId)); err != nil { return nil, "", model.NewAppError("getEmojiImage", "api.emoji.get_image.read.app_error", nil, err.Error(), http.StatusNotFound) } else { img = data @@ -217,8 +213,8 @@ func imageToPaletted(img image.Image) *image.Paletted { return pm } -func deleteEmojiImage(id string) { - if err := utils.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil { +func (a *App) deleteEmojiImage(id string) { + if err := a.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil { l4g.Error("Failed to rename image when deleting emoji %v", id) } } diff --git a/app/file.go b/app/file.go index c1389ef37..d66c64adb 100644 --- a/app/file.go +++ b/app/file.go @@ -57,7 +57,43 @@ const ( IMAGE_PREVIEW_PIXEL_WIDTH = 1024 ) -func GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { +func (a *App) FileBackend() (utils.FileBackend, *model.AppError) { + return utils.NewFileBackend(&a.Config().FileSettings) +} + +func (a *App) ReadFile(path string) ([]byte, *model.AppError) { + backend, err := a.FileBackend() + if err != nil { + return nil, err + } + return backend.ReadFile(path) +} + +func (a *App) MoveFile(oldPath, newPath string) *model.AppError { + backend, err := a.FileBackend() + if err != nil { + return err + } + return backend.MoveFile(oldPath, newPath) +} + +func (a *App) WriteFile(f []byte, path string) *model.AppError { + backend, err := a.FileBackend() + if err != nil { + return err + } + return backend.WriteFile(f, path) +} + +func (a *App) RemoveFile(path string) *model.AppError { + backend, err := a.FileBackend() + if err != nil { + return err + } + return backend.RemoveFile(path) +} + +func (a *App) GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { // Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension} split := strings.SplitN(filename, "/", 5) if len(split) < 5 { @@ -79,7 +115,7 @@ func GetInfoForFilename(post *model.Post, teamId string, filename string) *model // Open the file and populate the fields of the FileInfo var info *model.FileInfo - if data, err := utils.ReadFile(path); err != nil { + if data, err := a.ReadFile(path); err != nil { l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err) return nil } else { @@ -121,7 +157,7 @@ func (a *App) FindTeamIdForFilename(post *model.Post, filename string) string { } else { for _, team := range teams { path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name) - if _, err := utils.ReadFile(path); err == nil { + if _, err := a.ReadFile(path); err == nil { // Found the team that this file was posted from return team.Id } @@ -168,7 +204,7 @@ func (a *App) MigrateFilenamesToFileInfos(post *model.Post) []*model.FileInfo { l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.team_id.error"), post.Id, filenames) } else { for _, filename := range filenames { - info := GetInfoForFilename(post, teamId, filename) + info := a.GetInfoForFilename(post, teamId, filename) if info == nil { continue } @@ -286,7 +322,7 @@ func (a *App) UploadFiles(teamId string, channelId string, userId string, fileHe } } - HandleImages(previewPathList, thumbnailPathList, imageDataList) + a.HandleImages(previewPathList, thumbnailPathList, imageDataList) return resStruct, nil } @@ -321,7 +357,7 @@ func (a *App) DoUploadFile(now time.Time, rawTeamId string, rawChannelId string, info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" } - if err := utils.WriteFile(data, info.Path); err != nil { + if err := a.WriteFile(data, info.Path); err != nil { return nil, err } @@ -332,7 +368,7 @@ func (a *App) DoUploadFile(now time.Time, rawTeamId string, rawChannelId string, return info, nil } -func HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) { +func (a *App) HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) { wg := new(sync.WaitGroup) for i := range fileData { @@ -341,12 +377,12 @@ func HandleImages(previewPathList []string, thumbnailPathList []string, fileData wg.Add(2) go func(img *image.Image, path string, width int, height int) { defer wg.Done() - generateThumbnailImage(*img, path, width, height) + a.generateThumbnailImage(*img, path, width, height) }(img, thumbnailPathList[i], width, height) go func(img *image.Image, path string, width int) { defer wg.Done() - generatePreviewImage(*img, path, width) + a.generatePreviewImage(*img, path, width) }(img, previewPathList[i], width) } } @@ -417,7 +453,7 @@ func getImageOrientation(input io.Reader) (int, error) { } } -func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) { +func (a *App) generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) { thumbWidth := float64(IMAGE_THUMBNAIL_PIXEL_WIDTH) thumbHeight := float64(IMAGE_THUMBNAIL_PIXEL_HEIGHT) imgWidth := float64(width) @@ -438,13 +474,13 @@ func generateThumbnailImage(img image.Image, thumbnailPath string, width int, he return } - if err := utils.WriteFile(buf.Bytes(), thumbnailPath); err != nil { + if err := a.WriteFile(buf.Bytes(), thumbnailPath); err != nil { l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err) return } } -func generatePreviewImage(img image.Image, previewPath string, width int) { +func (a *App) generatePreviewImage(img image.Image, previewPath string, width int) { var preview image.Image if width > IMAGE_PREVIEW_PIXEL_WIDTH { @@ -460,7 +496,7 @@ func generatePreviewImage(img image.Image, previewPath string, width int) { return } - if err := utils.WriteFile(buf.Bytes(), previewPath); err != nil { + if err := a.WriteFile(buf.Bytes(), previewPath); err != nil { l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err) return } diff --git a/app/file_test.go b/app/file_test.go index d86272063..204113782 100644 --- a/app/file_test.go +++ b/app/file_test.go @@ -9,7 +9,6 @@ import ( "time" "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" ) func TestGeneratePublicLinkHash(t *testing.T) { @@ -51,7 +50,7 @@ func TestDoUploadFile(t *testing.T) { } else { defer func() { <-th.App.Srv.Store.FileInfo().PermanentDelete(info1.Id) - utils.RemoveFile(info1.Path) + th.App.RemoveFile(info1.Path) }() } @@ -65,7 +64,7 @@ func TestDoUploadFile(t *testing.T) { } else { defer func() { <-th.App.Srv.Store.FileInfo().PermanentDelete(info2.Id) - utils.RemoveFile(info2.Path) + th.App.RemoveFile(info2.Path) }() } @@ -79,7 +78,7 @@ func TestDoUploadFile(t *testing.T) { } else { defer func() { <-th.App.Srv.Store.FileInfo().PermanentDelete(info3.Id) - utils.RemoveFile(info3.Path) + th.App.RemoveFile(info3.Path) }() } @@ -93,7 +92,7 @@ func TestDoUploadFile(t *testing.T) { } else { defer func() { <-th.App.Srv.Store.FileInfo().PermanentDelete(info3.Id) - utils.RemoveFile(info3.Path) + th.App.RemoveFile(info3.Path) }() } diff --git a/app/import.go b/app/import.go index 21ce0ba53..08decb676 100644 --- a/app/import.go +++ b/app/import.go @@ -1497,8 +1497,8 @@ func (a *App) OldImportFile(timestamp time.Time, file io.Reader, teamId string, img, width, height := prepareImage(data) if img != nil { - generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height) - generatePreviewImage(*img, fileInfo.PreviewPath, width) + a.generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height) + a.generatePreviewImage(*img, fileInfo.PreviewPath, width) } return fileInfo, nil diff --git a/app/user.go b/app/user.go index 48237f300..358a87711 100644 --- a/app/user.go +++ b/app/user.go @@ -751,7 +751,7 @@ func (a *App) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) } else { path := "users/" + user.Id + "/profile.png" - if data, err := utils.ReadFile(path); err != nil { + if data, err := a.ReadFile(path); err != nil { readFailed = true if img, err = CreateProfileImage(user.Username, user.Id, a.Config().FileSettings.InitialFont); err != nil { @@ -759,7 +759,7 @@ func (a *App) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) } if user.LastPictureUpdate == 0 { - if err := utils.WriteFile(img, path); err != nil { + if err := a.WriteFile(img, path); err != nil { return nil, false, err } } @@ -812,7 +812,7 @@ func (a *App) SetProfileImage(userId string, imageData *multipart.FileHeader) *m path := "users/" + userId + "/profile.png" - if err := utils.WriteFile(buf.Bytes(), path); err != nil { + if err := a.WriteFile(buf.Bytes(), path); err != nil { return model.NewAppError("SetProfileImage", "api.user.upload_profile_user.upload_profile.app_error", nil, "", http.StatusInternalServerError) } diff --git a/cmd/platform/server.go b/cmd/platform/server.go index 36118d007..51f5fa67e 100644 --- a/cmd/platform/server.go +++ b/cmd/platform/server.go @@ -60,13 +60,17 @@ func runServer(configFileLocation string) { l4g.Info(utils.T("mattermost.working_dir"), pwd) l4g.Info(utils.T("mattermost.config_file"), utils.FindConfigFile(configFileLocation)) - if err := utils.TestFileConnection(); err != nil { - l4g.Error("Problem with file storage settings: " + err.Error()) - } - a := app.New(app.ConfigFile(configFileLocation)) defer a.Shutdown() + backend, err := a.FileBackend() + if err == nil { + err = backend.TestConnection() + } + if err != nil { + l4g.Error("Problem with file storage settings: " + err.Error()) + } + if model.BuildEnterpriseReady == "true" { a.LoadLicense() } diff --git a/einterfaces/brand.go b/einterfaces/brand.go index ae187fdcd..fc584a91c 100644 --- a/einterfaces/brand.go +++ b/einterfaces/brand.go @@ -12,13 +12,3 @@ type BrandInterface interface { SaveBrandImage(*multipart.FileHeader) *model.AppError GetBrandImage() ([]byte, *model.AppError) } - -var theBrandInterface BrandInterface - -func RegisterBrandInterface(newInterface BrandInterface) { - theBrandInterface = newInterface -} - -func GetBrandInterface() BrandInterface { - return theBrandInterface -} diff --git a/einterfaces/elasticsearch.go b/einterfaces/elasticsearch.go index b81605b4f..5582fd4e8 100644 --- a/einterfaces/elasticsearch.go +++ b/einterfaces/elasticsearch.go @@ -18,13 +18,3 @@ type ElasticsearchInterface interface { PurgeIndexes() *model.AppError DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError } - -var theElasticsearchInterface ElasticsearchInterface - -func RegisterElasticsearchInterface(newInterface ElasticsearchInterface) { - theElasticsearchInterface = newInterface -} - -func GetElasticsearchInterface() ElasticsearchInterface { - return theElasticsearchInterface -} diff --git a/einterfaces/emoji.go b/einterfaces/emoji.go index dc685e359..b8d61e748 100644 --- a/einterfaces/emoji.go +++ b/einterfaces/emoji.go @@ -10,13 +10,3 @@ import ( type EmojiInterface interface { CanUserCreateEmoji(string, []*model.TeamMember) bool } - -var theEmojiInterface EmojiInterface - -func RegisterEmojiInterface(newInterface EmojiInterface) { - theEmojiInterface = newInterface -} - -func GetEmojiInterface() EmojiInterface { - return theEmojiInterface -} diff --git a/utils/file.go b/utils/file.go index 6472770a0..13b25bdab 100644 --- a/utils/file.go +++ b/utils/file.go @@ -4,372 +4,13 @@ package utils import ( - "bytes" "fmt" "io" "io/ioutil" - "net/http" "os" "path/filepath" - "strings" - - l4g "github.com/alecthomas/log4go" - s3 "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/credentials" - - "github.com/mattermost/mattermost-server/model" -) - -const ( - TEST_FILE_PATH = "/testfile" ) -// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. -// If signV2 input is false, function always returns signature v4. -// -// Additionally this function also takes a user defined region, if set -// disables automatic region lookup. -func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) { - var creds *credentials.Credentials - if signV2 { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2) - } else { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4) - } - - s3Clnt, err := s3.NewWithCredentials(endpoint, creds, secure, region) - if err != nil { - return nil, err - } - - if *Cfg.FileSettings.AmazonS3Trace { - s3Clnt.TraceOn(os.Stdout) - } - - return s3Clnt, nil -} - -func TestFileConnection() *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - bucket := Cfg.FileSettings.AmazonS3Bucket - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError) - } - - exists, err := s3Clnt.BucketExists(bucket) - if err != nil { - return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError) - } - - if !exists { - l4g.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(bucket, region) - if err != nil { - l4g.Error("Unable to create bucket.") - return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) - } - } - l4g.Info("Connection to S3 or minio is good. Bucket exists.") - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - f := []byte("testingwrite") - if err := writeFileLocally(f, Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil { - return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) - } - os.Remove(Cfg.FileSettings.Directory + TEST_FILE_PATH) - l4g.Info("Able to write files to local storage.") - } else { - return model.NewAppError("TestFileConnection", "No file driver selected.", nil, "", http.StatusInternalServerError) - } - - return nil -} - -func ReadFile(path string) ([]byte, *model.AppError) { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - bucket := Cfg.FileSettings.AmazonS3Bucket - minioObject, err := s3Clnt.GetObject(bucket, path) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - defer minioObject.Close() - if f, err := ioutil.ReadAll(minioObject); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if f, err := ioutil.ReadFile(Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } - } else { - return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented) - } -} - -func MoveFile(oldPath, newPath string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - encrypt := false - if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { - encrypt = true - } - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - bucket := Cfg.FileSettings.AmazonS3Bucket - - source := s3.NewSourceInfo(bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt)) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(Cfg.FileSettings.Directory+newPath), 0774); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if err := os.Rename(Cfg.FileSettings.Directory+oldPath, Cfg.FileSettings.Directory+newPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("moveFile", "api.file.move_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func WriteFile(f []byte, path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - encrypt := false - if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { - encrypt = true - } - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - bucket := Cfg.FileSettings.AmazonS3Bucket - ext := filepath.Ext(path) - metaData := S3Metadata(encrypt, "binary/octet-stream") - if model.IsFileExtImage(ext) { - metaData = S3Metadata(encrypt, model.GetImageMimeType(ext)) - } - - _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil) - if err != nil { - return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := writeFileLocally(f, Cfg.FileSettings.Directory+path); err != nil { - return err - } - } else { - return model.NewAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func writeFileLocally(f []byte, path string) *model.AppError { - if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { - directory, _ := filepath.Abs(filepath.Dir(path)) - return model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) - } - - if err := ioutil.WriteFile(path, f, 0644); err != nil { - return model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return nil -} - -func RemoveFile(path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - bucket := Cfg.FileSettings.AmazonS3Bucket - if err := s3Clnt.RemoveObject(bucket, path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.Remove(Cfg.FileSettings.Directory + path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("RemoveFile", "utils.file.remove_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { - out := make(chan string, 1) - - go func() { - defer close(out) - - for { - info, done := <-in - - if !done { - break - } - - out <- info.Key - } - }() - - return out -} - -// Returns a list of all the directories within the path directory provided. -func ListDirectory(path string) (*[]string, *model.AppError) { - var paths []string - - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - defer close(doneCh) - - bucket := Cfg.FileSettings.AmazonS3Bucket - for object := range s3Clnt.ListObjects(bucket, path, false, doneCh) { - if object.Err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) - } - paths = append(paths, strings.Trim(object.Key, "/")) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if fileInfos, err := ioutil.ReadDir(Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - for _, fileInfo := range fileInfos { - if fileInfo.IsDir() { - paths = append(paths, filepath.Join(path, fileInfo.Name())) - } - } - } - } else { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.configured.app_error", nil, "", http.StatusInternalServerError) - } - - return &paths, nil -} - -func RemoveDirectory(path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - bucket := Cfg.FileSettings.AmazonS3Bucket - for err := range s3Clnt.RemoveObjects(bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(bucket, path, true, doneCh))) { - if err.Err != nil { - doneCh <- struct{}{} - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) - } - } - - close(doneCh) - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.RemoveAll(Cfg.FileSettings.Directory + path); err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func S3Metadata(encrypt bool, contentType string) map[string][]string { - metaData := make(map[string][]string) - if contentType != "" { - metaData["Content-Type"] = []string{"contentType"} - } - if encrypt { - metaData["x-amz-server-side-encryption"] = []string{"AES256"} - } - return metaData -} - -func CopyMetadata(encrypt bool) map[string]string { - metaData := make(map[string]string) - metaData["x-amz-server-side-encryption"] = "AES256" - return metaData -} - // CopyFile will copy a file from src path to dst path. // Overwrites any existing files at dst. // Permissions are copied from file at src to the new file at dst. diff --git a/utils/file_backend.go b/utils/file_backend.go new file mode 100644 index 000000000..3469a63fb --- /dev/null +++ b/utils/file_backend.go @@ -0,0 +1,44 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "net/http" + + "github.com/mattermost/mattermost-server/model" +) + +type FileBackend interface { + TestConnection() *model.AppError + + ReadFile(path string) ([]byte, *model.AppError) + MoveFile(oldPath, newPath string) *model.AppError + WriteFile(f []byte, path string) *model.AppError + RemoveFile(path string) *model.AppError + + ListDirectory(path string) (*[]string, *model.AppError) + RemoveDirectory(path string) *model.AppError +} + +func NewFileBackend(settings *model.FileSettings) (FileBackend, *model.AppError) { + switch *settings.DriverName { + case model.IMAGE_DRIVER_S3: + return &S3FileBackend{ + endpoint: settings.AmazonS3Endpoint, + accessKey: settings.AmazonS3AccessKeyId, + secretKey: settings.AmazonS3SecretAccessKey, + secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, + signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, + region: settings.AmazonS3Region, + bucket: settings.AmazonS3Bucket, + encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance, + trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, + }, nil + case model.IMAGE_DRIVER_LOCAL: + return &LocalFileBackend{ + directory: settings.Directory, + }, nil + } + return nil, model.NewAppError("NewFileBackend", "No file driver selected.", nil, "", http.StatusInternalServerError) +} diff --git a/utils/file_backend_local.go b/utils/file_backend_local.go new file mode 100644 index 000000000..b5e67f8f0 --- /dev/null +++ b/utils/file_backend_local.go @@ -0,0 +1,98 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "io/ioutil" + "net/http" + "os" + "path/filepath" + + l4g "github.com/alecthomas/log4go" + + "github.com/mattermost/mattermost-server/model" +) + +const ( + TEST_FILE_PATH = "/testfile" +) + +type LocalFileBackend struct { + directory string +} + +func (b *LocalFileBackend) TestConnection() *model.AppError { + f := []byte("testingwrite") + if err := writeFileLocally(f, filepath.Join(b.directory, TEST_FILE_PATH)); err != nil { + return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) + } + os.Remove(filepath.Join(b.directory, TEST_FILE_PATH)) + l4g.Info("Able to write files to local storage.") + return nil +} + +func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) { + if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *LocalFileBackend) WriteFile(f []byte, path string) *model.AppError { + return writeFileLocally(f, filepath.Join(b.directory, path)) +} + +func writeFileLocally(f []byte, path string) *model.AppError { + if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { + directory, _ := filepath.Abs(filepath.Dir(path)) + return model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) + } + + if err := ioutil.WriteFile(path, f, 0644); err != nil { + return model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *LocalFileBackend) RemoveFile(path string) *model.AppError { + if err := os.Remove(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + if fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path)); err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + for _, fileInfo := range fileInfos { + if fileInfo.IsDir() { + paths = append(paths, filepath.Join(path, fileInfo.Name())) + } + } + } + return &paths, nil +} + +func (b *LocalFileBackend) RemoveDirectory(path string) *model.AppError { + if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} diff --git a/utils/file_backend_s3.go b/utils/file_backend_s3.go new file mode 100644 index 000000000..ed88dc70c --- /dev/null +++ b/utils/file_backend_s3.go @@ -0,0 +1,226 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + l4g "github.com/alecthomas/log4go" + s3 "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/credentials" + + "github.com/mattermost/mattermost-server/model" +) + +type S3FileBackend struct { + endpoint string + accessKey string + secretKey string + secure bool + signV2 bool + region string + bucket string + encrypt bool + trace bool +} + +// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. +// If signV2 input is false, function always returns signature v4. +// +// Additionally this function also takes a user defined region, if set +// disables automatic region lookup. +func (b *S3FileBackend) s3New() (*s3.Client, error) { + var creds *credentials.Credentials + if b.signV2 { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) + } else { + creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) + } + + s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) + if err != nil { + return nil, err + } + + if b.trace { + s3Clnt.TraceOn(os.Stdout) + } + + return s3Clnt, nil +} + +func (b *S3FileBackend) TestConnection() *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError) + } + + exists, err := s3Clnt.BucketExists(b.bucket) + if err != nil { + return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError) + } + + if !exists { + l4g.Warn("Bucket specified does not exist. Attempting to create...") + err := s3Clnt.MakeBucket(b.bucket, b.region) + if err != nil { + l4g.Error("Unable to create bucket.") + return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) + } + } + l4g.Info("Connection to S3 or minio is good. Bucket exists.") + return nil +} + +func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + minioObject, err := s3Clnt.GetObject(b.bucket, path) + if err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + defer minioObject.Close() + if f, err := ioutil.ReadAll(minioObject); err != nil { + return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } else { + return f, nil + } +} + +func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + source := s3.NewSourceInfo(b.bucket, oldPath, nil) + destination, err := s3.NewDestinationInfo(b.bucket, newPath, nil, s3CopyMetadata(b.encrypt)) + if err != nil { + return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.CopyObject(destination, source); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { + return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *S3FileBackend) WriteFile(f []byte, path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + ext := filepath.Ext(path) + metaData := s3Metadata(b.encrypt, "binary/octet-stream") + if model.IsFileExtImage(ext) { + metaData = s3Metadata(b.encrypt, model.GetImageMimeType(ext)) + } + + if _, err = s3Clnt.PutObjectWithMetadata(b.bucket, path, bytes.NewReader(f), metaData, nil); err != nil { + return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (b *S3FileBackend) RemoveFile(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { + return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { + out := make(chan string, 1) + + go func() { + defer close(out) + + for { + info, done := <-in + + if !done { + break + } + + out <- info.Key + } + }() + + return out +} + +func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { + var paths []string + + s3Clnt, err := b.s3New() + if err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + defer close(doneCh) + + for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { + if object.Err != nil { + return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) + } + paths = append(paths, strings.Trim(object.Key, "/")) + } + + return &paths, nil +} + +func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { + s3Clnt, err := b.s3New() + if err != nil { + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + doneCh := make(chan struct{}) + + for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { + if err.Err != nil { + doneCh <- struct{}{} + return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) + } + } + + close(doneCh) + return nil +} + +func s3Metadata(encrypt bool, contentType string) map[string][]string { + metaData := make(map[string][]string) + if contentType != "" { + metaData["Content-Type"] = []string{"contentType"} + } + if encrypt { + metaData["x-amz-server-side-encryption"] = []string{"AES256"} + } + return metaData +} + +func s3CopyMetadata(encrypt bool) map[string]string { + metaData := make(map[string]string) + metaData["x-amz-server-side-encryption"] = "AES256" + return metaData +} diff --git a/utils/file_backend_test.go b/utils/file_backend_test.go new file mode 100644 index 000000000..0989f783c --- /dev/null +++ b/utils/file_backend_test.go @@ -0,0 +1,164 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package utils + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/mattermost/mattermost-server/model" +) + +type FileBackendTestSuite struct { + suite.Suite + + settings model.FileSettings + backend FileBackend +} + +func TestLocalFileBackendTestSuite(t *testing.T) { + dir, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(dir) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL), + Directory: dir, + }, + }) +} + +func TestS3FileBackendTestSuite(t *testing.T) { + s3Host := os.Getenv("CI_HOST") + if s3Host == "" { + s3Host = "dockerhost" + } + + s3Port := os.Getenv("CI_MINIO_PORT") + if s3Port == "" { + s3Port = "9001" + } + + s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) + + suite.Run(t, &FileBackendTestSuite{ + settings: model.FileSettings{ + DriverName: model.NewString(model.IMAGE_DRIVER_S3), + AmazonS3AccessKeyId: "minioaccesskey", + AmazonS3SecretAccessKey: "miniosecretkey", + AmazonS3Bucket: "mattermost-test", + AmazonS3Endpoint: s3Endpoint, + AmazonS3SSL: model.NewBool(false), + }, + }) +} + +func (s *FileBackendTestSuite) SetupTest() { + TranslationsPreInit() + + backend, err := NewFileBackend(&s.settings) + require.Nil(s.T(), err) + s.backend = backend +} + +func (s *FileBackendTestSuite) TestConnection() { + s.Nil(s.backend.TestConnection()) +} + +func (s *FileBackendTestSuite) TestReadWriteFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path)) + defer s.backend.RemoveFile(path) + + read, err := s.backend.ReadFile(path) + s.Nil(err) + + readString := string(read) + s.EqualValues(readString, "test") +} + +func (s *FileBackendTestSuite) TestMoveFile() { + b := []byte("test") + path1 := "tests/" + model.NewId() + path2 := "tests/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path1)) + defer s.backend.RemoveFile(path1) + + s.Nil(s.backend.MoveFile(path1, path2)) + defer s.backend.RemoveFile(path2) + + _, err := s.backend.ReadFile(path1) + s.Error(err) + + _, err = s.backend.ReadFile(path2) + s.Nil(err) +} + +func (s *FileBackendTestSuite) TestRemoveFile() { + b := []byte("test") + path := "tests/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path)) + s.Nil(s.backend.RemoveFile(path)) + + _, err := s.backend.ReadFile(path) + s.Error(err) + + s.Nil(s.backend.WriteFile(b, "tests2/foo")) + s.Nil(s.backend.WriteFile(b, "tests2/bar")) + s.Nil(s.backend.WriteFile(b, "tests2/asdf")) + s.Nil(s.backend.RemoveDirectory("tests2")) +} + +func (s *FileBackendTestSuite) TestListDirectory() { + b := []byte("test") + path1 := "19700101/" + model.NewId() + path2 := "19800101/" + model.NewId() + + s.Nil(s.backend.WriteFile(b, path1)) + defer s.backend.RemoveFile(path1) + s.Nil(s.backend.WriteFile(b, path2)) + defer s.backend.RemoveFile(path2) + + paths, err := s.backend.ListDirectory("") + s.Nil(err) + + found1 := false + found2 := false + for _, path := range *paths { + if path == "19700101" { + found1 = true + } else if path == "19800101" { + found2 = true + } + } + s.True(found1) + s.True(found2) +} + +func (s *FileBackendTestSuite) TestRemoveDirectory() { + b := []byte("test") + + s.Nil(s.backend.WriteFile(b, "tests2/foo")) + s.Nil(s.backend.WriteFile(b, "tests2/bar")) + s.Nil(s.backend.WriteFile(b, "tests2/aaa")) + + s.Nil(s.backend.RemoveDirectory("tests2")) + + _, err := s.backend.ReadFile("tests2/foo") + s.Error(err) + _, err = s.backend.ReadFile("tests2/bar") + s.Error(err) + _, err = s.backend.ReadFile("tests2/asdf") + s.Error(err) +} diff --git a/utils/file_test.go b/utils/file_test.go index 91e78f24e..6c7e3c462 100644 --- a/utils/file_test.go +++ b/utils/file_test.go @@ -4,7 +4,6 @@ package utils import ( - "fmt" "io/ioutil" "os" "path/filepath" @@ -12,179 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/mattermost/mattermost-server/model" ) -type FileTestSuite struct { - suite.Suite - - testDriver string - - // Config to be reset after tests. - driverName string - amazonS3AccessKeyId string - amazonS3SecretAccessKey string - amazonS3Bucket string - amazonS3Endpoint string - amazonS3SSL bool -} - -func TestFileLocalTestSuite(t *testing.T) { - testsuite := FileTestSuite{ - testDriver: model.IMAGE_DRIVER_LOCAL, - } - suite.Run(t, &testsuite) -} - -func TestFileMinioTestSuite(t *testing.T) { - testsuite := FileTestSuite{ - testDriver: model.IMAGE_DRIVER_S3, - } - suite.Run(t, &testsuite) -} - -func (s *FileTestSuite) SetupTest() { - TranslationsPreInit() - LoadGlobalConfig("config.json") - InitTranslations(Cfg.LocalizationSettings) - - // Save state to restore after the test has run. - s.driverName = *Cfg.FileSettings.DriverName - s.amazonS3AccessKeyId = Cfg.FileSettings.AmazonS3AccessKeyId - s.amazonS3SecretAccessKey = Cfg.FileSettings.AmazonS3SecretAccessKey - s.amazonS3Bucket = Cfg.FileSettings.AmazonS3Bucket - s.amazonS3Endpoint = Cfg.FileSettings.AmazonS3Endpoint - s.amazonS3SSL = *Cfg.FileSettings.AmazonS3SSL - - // Set up the state for the tests. - s3Host := os.Getenv("CI_HOST") - if s3Host == "" { - s3Host = "dockerhost" - } - - s3Port := os.Getenv("CI_MINIO_PORT") - if s3Port == "" { - s3Port = "9001" - } - - s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) - if s.testDriver == model.IMAGE_DRIVER_LOCAL { - *Cfg.FileSettings.DriverName = model.IMAGE_DRIVER_LOCAL - } else if s.testDriver == model.IMAGE_DRIVER_S3 { - *Cfg.FileSettings.DriverName = model.IMAGE_DRIVER_S3 - Cfg.FileSettings.AmazonS3AccessKeyId = "minioaccesskey" - Cfg.FileSettings.AmazonS3SecretAccessKey = "miniosecretkey" - Cfg.FileSettings.AmazonS3Bucket = "mattermost-test" - Cfg.FileSettings.AmazonS3Endpoint = s3Endpoint - *Cfg.FileSettings.AmazonS3SSL = false - } else { - s.T().Fatal("Invalid image driver set for test suite.") - } -} - -func (s *FileTestSuite) TearDownTest() { - // Restore the test state. - *Cfg.FileSettings.DriverName = s.driverName - Cfg.FileSettings.AmazonS3AccessKeyId = s.amazonS3AccessKeyId - Cfg.FileSettings.AmazonS3SecretAccessKey = s.amazonS3SecretAccessKey - Cfg.FileSettings.AmazonS3Bucket = s.amazonS3Bucket - Cfg.FileSettings.AmazonS3Endpoint = s.amazonS3Endpoint - *Cfg.FileSettings.AmazonS3SSL = s.amazonS3SSL -} - -func (s *FileTestSuite) TestReadWriteFile() { - b := []byte("test") - path := "tests/" + model.NewId() - - s.Nil(WriteFile(b, path)) - defer RemoveFile(path) - - read, err := ReadFile(path) - s.Nil(err) - - readString := string(read) - s.EqualValues(readString, "test") -} - -func (s *FileTestSuite) TestMoveFile() { - b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/" + model.NewId() - - s.Nil(WriteFile(b, path1)) - defer RemoveFile(path1) - - s.Nil(MoveFile(path1, path2)) - defer RemoveFile(path2) - - _, err := ReadFile(path1) - s.Error(err) - - _, err = ReadFile(path2) - s.Nil(err) -} - -func (s *FileTestSuite) TestRemoveFile() { - b := []byte("test") - path := "tests/" + model.NewId() - - s.Nil(WriteFile(b, path)) - s.Nil(RemoveFile(path)) - - _, err := ReadFile(path) - s.Error(err) - - s.Nil(WriteFile(b, "tests2/foo")) - s.Nil(WriteFile(b, "tests2/bar")) - s.Nil(WriteFile(b, "tests2/asdf")) - s.Nil(RemoveDirectory("tests2")) -} - -func (s *FileTestSuite) TestListDirectory() { - b := []byte("test") - path1 := "19700101/" + model.NewId() - path2 := "19800101/" + model.NewId() - - s.Nil(WriteFile(b, path1)) - defer RemoveFile(path1) - s.Nil(WriteFile(b, path2)) - defer RemoveFile(path2) - - paths, err := ListDirectory("") - s.Nil(err) - - found1 := false - found2 := false - for _, path := range *paths { - if path == "19700101" { - found1 = true - } else if path == "19800101" { - found2 = true - } - } - s.True(found1) - s.True(found2) -} - -func (s *FileTestSuite) TestRemoveDirectory() { - b := []byte("test") - - s.Nil(WriteFile(b, "tests2/foo")) - s.Nil(WriteFile(b, "tests2/bar")) - s.Nil(WriteFile(b, "tests2/aaa")) - - s.Nil(RemoveDirectory("tests2")) - - _, err := ReadFile("tests2/foo") - s.Error(err) - _, err = ReadFile("tests2/bar") - s.Error(err) - _, err = ReadFile("tests2/asdf") - s.Error(err) -} - func TestCopyDir(t *testing.T) { srcDir, err := ioutil.TempDir("", "src") require.NoError(t, err) -- cgit v1.2.3-1-g7c22