summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--api/emoji_test.go2
-rw-r--r--api/file.go10
-rw-r--r--api/file_test.go53
-rw-r--r--api4/file.go8
-rw-r--r--api4/file_test.go8
-rw-r--r--app/emoji.go10
-rw-r--r--app/file.go257
-rw-r--r--app/file_test.go55
-rw-r--r--app/import.go6
-rw-r--r--app/slackimport.go3
-rw-r--r--app/user.go6
-rw-r--r--cmd/platform/server.go2
-rw-r--r--i18n/en.json36
-rw-r--r--model/file_info.go3
-rw-r--r--store/sql_file_info_store.go40
-rw-r--r--store/sql_file_info_store_test.go38
-rw-r--r--store/store.go1
-rw-r--r--utils/file.go313
-rw-r--r--utils/file_test.go122
-rw-r--r--utils/time.go4
20 files changed, 678 insertions, 299 deletions
diff --git a/api/emoji_test.go b/api/emoji_test.go
index 600f7975e..4de5f41dd 100644
--- a/api/emoji_test.go
+++ b/api/emoji_test.go
@@ -266,7 +266,7 @@ func TestDeleteEmoji(t *testing.T) {
func createTestEmoji(t *testing.T, emoji *model.Emoji, imageData []byte) *model.Emoji {
emoji = store.Must(app.Srv.Store.Emoji().Save(emoji)).(*model.Emoji)
- if err := app.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil {
+ if err := utils.WriteFile(imageData, "emoji/"+emoji.Id+"/image"); err != nil {
store.Must(app.Srv.Store.Emoji().Delete(emoji.Id, time.Now().Unix()))
t.Fatalf("failed to write image: %v", err.Error())
}
diff --git a/api/file.go b/api/file.go
index 342430ed1..1eab30e76 100644
--- a/api/file.go
+++ b/api/file.go
@@ -96,7 +96,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.Path); err != nil {
+ if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
@@ -118,7 +118,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.ThumbnailPath); err != nil {
+ if data, err := utils.ReadFile(info.ThumbnailPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, w, r); err != nil {
@@ -140,7 +140,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.PreviewPath); err != nil {
+ if data, err := utils.ReadFile(info.PreviewPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, w, r); err != nil {
@@ -190,7 +190,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.Path); err != nil {
+ if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
@@ -285,7 +285,7 @@ func getPublicFileOld(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.Path); err != nil {
+ if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
diff --git a/api/file_test.go b/api/file_test.go
index 0e0a5772c..282cff2ec 100644
--- a/api/file_test.go
+++ b/api/file_test.go
@@ -81,20 +81,22 @@ func TestUploadFile(t *testing.T) {
t.Fatal("file preview path should be set in database")
}
+ date := time.Now().Format("20060102")
+
// This also makes sure that the relative path provided above is sanitized out
- expectedPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test.png", team.Id, channel.Id, user.Id, info.Id)
+ expectedPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test.png", date, team.Id, channel.Id, user.Id, info.Id)
if info.Path != expectedPath {
t.Logf("file is saved in %v", info.Path)
t.Fatalf("file should've been saved in %v", expectedPath)
}
- expectedThumbnailPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", team.Id, channel.Id, user.Id, info.Id)
+ expectedThumbnailPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", date, team.Id, channel.Id, user.Id, info.Id)
if info.ThumbnailPath != expectedThumbnailPath {
t.Logf("file thumbnail is saved in %v", info.ThumbnailPath)
t.Fatalf("file thumbnail should've been saved in %v", expectedThumbnailPath)
}
- expectedPreviewPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_preview.jpg", team.Id, channel.Id, user.Id, info.Id)
+ expectedPreviewPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_preview.jpg", date, team.Id, channel.Id, user.Id, info.Id)
if info.PreviewPath != expectedPreviewPath {
t.Logf("file preview is saved in %v", info.PreviewPath)
t.Fatalf("file preview should've been saved in %v", expectedPreviewPath)
@@ -466,7 +468,6 @@ func TestGetPublicFileOld(t *testing.T) {
utils.Cfg.FileSettings.EnablePublicLink = true
*utils.Cfg.FileSettings.PublicLinkSalt = model.NewId()
- Client := th.BasicClient
channel := th.BasicChannel
var fileId string
@@ -474,7 +475,16 @@ func TestGetPublicFileOld(t *testing.T) {
if err != nil {
t.Fatal(err)
} else {
- fileId = Client.MustGeneric(Client.UploadPostAttachment(data, channel.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
+ //fileId = Client.MustGeneric(Client.UploadPostAttachment(data, channel.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
+ fileId = model.NewId()
+ fileInfo := model.FileInfo{
+ Id: fileId,
+ CreateAt: model.GetMillis(),
+ CreatorId: th.BasicUser.Id,
+ Path: fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", th.BasicTeam.Id, channel.Id, th.BasicUser.Id, fileId, "test.png"),
+ }
+ store.Must(app.Srv.Store.FileInfo().Save(&fileInfo))
+ uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel.Id, th.BasicUser.Id, fileId), "test.png")
}
// Hacky way to assign file to a post (usually would be done by CreatePost call)
@@ -619,7 +629,9 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) {
t.Fatal(err)
} else {
fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
+ uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel1.Id, user1.Id, fileId1), "test.png")
fileId2 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
+ uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", th.BasicTeam.Id, channel1.Id, user1.Id, fileId2), "test.png")
}
// Bypass the Client whenever possible since we're trying to simulate a pre-3.5 post
@@ -686,6 +698,25 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) {
}
}
+func uploadFileOld(t *testing.T, data []byte, dest string, filename string) {
+ os.MkdirAll(dest, os.ModePerm)
+ eFile, err := os.Create(dest + "/" + filename)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer eFile.Close()
+
+ _, err = io.Copy(eFile, bytes.NewReader(data)) // first var shows number of bytes
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = eFile.Sync()
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
func TestFindTeamIdForFilename(t *testing.T) {
th := Setup().InitBasic()
@@ -717,9 +748,11 @@ func TestFindTeamIdForFilename(t *testing.T) {
t.Fatal(err)
} else {
fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
+ uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team1.Id, channel1.Id, user1.Id, fileId1), "test.png")
Client.SetTeamId(team2.Id)
fileId2 = Client.MustGeneric(Client.UploadPostAttachment(data, channel2.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
+ uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team2.Id, channel2.Id, user1.Id, fileId2), "test.png")
Client.SetTeamId(team1.Id)
}
@@ -732,6 +765,7 @@ func TestFindTeamIdForFilename(t *testing.T) {
})).(*model.Post)
if teamId := app.FindTeamIdForFilename(post1, post1.Filenames[0]); teamId != team1.Id {
+ t.Log(teamId)
t.Fatal("file should've been found under team1")
}
@@ -773,6 +807,7 @@ func TestGetInfoForFilename(t *testing.T) {
t.Fatal(err)
} else {
fileId1 = Client.MustGeneric(Client.UploadPostAttachment(data, channel1.Id, "test.png")).(*model.FileUploadResponse).FileInfos[0].Id
+ uploadFileOld(t, data, fmt.Sprintf("data/teams/%s/channels/%s/users/%s/%s", team1.Id, channel1.Id, user1.Id, fileId1), "test.png")
path = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).Path
thumbnailPath = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).ThumbnailPath
previewPath = store.Must(app.Srv.Store.FileInfo().Get(fileId1)).(*model.FileInfo).PreviewPath
@@ -786,6 +821,8 @@ func TestGetInfoForFilename(t *testing.T) {
Filenames: []string{fmt.Sprintf("/%s/%s/%s/%s", channel1.Id, user1.Id, fileId1, "test.png")},
})).(*model.Post)
+ date := time.Now().Format("20060102")
+
if info := app.GetInfoForFilename(post1, team1.Id, post1.Filenames[0]); info == nil {
t.Fatal("info shouldn't be nil")
} else if info.Id == "" {
@@ -794,11 +831,11 @@ func TestGetInfoForFilename(t *testing.T) {
t.Fatal("incorrect user id")
} else if info.PostId != post1.Id {
t.Fatal("incorrect user id")
- } else if info.Path != path {
+ } else if fmt.Sprintf("%s/%s", date, info.Path) != path {
t.Fatal("incorrect path")
- } else if info.ThumbnailPath != thumbnailPath {
+ } else if fmt.Sprintf("%s/%s", date, info.ThumbnailPath) != thumbnailPath {
t.Fatal("incorrect thumbnail path")
- } else if info.PreviewPath != previewPath {
+ } else if fmt.Sprintf("%s/%s", date, info.PreviewPath) != previewPath {
t.Fatal("incorrect preview path")
} else if info.Name != "test.png" {
t.Fatal("incorrect name")
diff --git a/api4/file.go b/api4/file.go
index 0607c1942..7dc13dafc 100644
--- a/api4/file.go
+++ b/api4/file.go
@@ -123,7 +123,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- data, err := app.ReadFile(info.Path)
+ data, err := utils.ReadFile(info.Path)
if err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
@@ -165,7 +165,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.ThumbnailPath); err != nil {
+ if data, err := utils.ReadFile(info.ThumbnailPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, THUMBNAIL_IMAGE_TYPE, data, forceDownload, w, r); err != nil {
@@ -237,7 +237,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.PreviewPath); err != nil {
+ if data, err := utils.ReadFile(info.PreviewPath); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, PREVIEW_IMAGE_TYPE, data, forceDownload, w, r); err != nil {
@@ -299,7 +299,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
- if data, err := app.ReadFile(info.Path); err != nil {
+ if data, err := utils.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, true, w, r); err != nil {
diff --git a/api4/file_test.go b/api4/file_test.go
index c2bc926e1..a2673dc8e 100644
--- a/api4/file_test.go
+++ b/api4/file_test.go
@@ -71,20 +71,22 @@ func TestUploadFile(t *testing.T) {
t.Fatal("file preview path should be set in database")
}
+ date := time.Now().Format("20060102")
+
// This also makes sure that the relative path provided above is sanitized out
- expectedPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test.png", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
+ expectedPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test.png", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.Path != expectedPath {
t.Logf("file is saved in %v", info.Path)
t.Fatalf("file should've been saved in %v", expectedPath)
}
- expectedThumbnailPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
+ expectedThumbnailPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.ThumbnailPath != expectedThumbnailPath {
t.Logf("file thumbnail is saved in %v", info.ThumbnailPath)
t.Fatalf("file thumbnail should've been saved in %v", expectedThumbnailPath)
}
- expectedPreviewPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_preview.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
+ expectedPreviewPath := fmt.Sprintf("%v/teams/%v/channels/%v/users/%v/%v/test_preview.jpg", date, FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.PreviewPath != expectedPreviewPath {
t.Logf("file preview is saved in %v", info.PreviewPath)
t.Fatalf("file preview should've been saved in %v", expectedPreviewPath)
diff --git a/app/emoji.go b/app/emoji.go
index f1362b798..e01ca97ae 100644
--- a/app/emoji.go
+++ b/app/emoji.go
@@ -101,7 +101,7 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := gif.EncodeAll(newbuf, resized_gif); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.gif_encode_error", nil, "", http.StatusBadRequest)
}
- if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
+ if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
@@ -113,13 +113,13 @@ func UploadEmojiImage(id string, imageData *multipart.FileHeader) *model.AppErro
if err := png.Encode(newbuf, resized_image); err != nil {
return model.NewAppError("uploadEmojiImage", "api.emoji.upload.large_image.encode_error", nil, "", http.StatusBadRequest)
}
- if err := WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
+ if err := utils.WriteFile(newbuf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
}
} else {
- if err := WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil {
+ if err := utils.WriteFile(buf.Bytes(), getEmojiImagePath(id)); err != nil {
return err
}
}
@@ -159,7 +159,7 @@ func GetEmojiImage(emojiId string) (imageByte []byte, imageType string, err *mod
} else {
var img []byte
- if data, err := ReadFile(getEmojiImagePath(emojiId)); err != nil {
+ if data, err := utils.ReadFile(getEmojiImagePath(emojiId)); err != nil {
return nil, "", model.NewAppError("getEmojiImage", "api.emoji.get_image.read.app_error", nil, err.Error(), http.StatusNotFound)
} else {
img = data
@@ -219,7 +219,7 @@ func imageToPaletted(img image.Image) *image.Paletted {
}
func deleteEmojiImage(id string) {
- if err := MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
+ if err := utils.MoveFile(getEmojiImagePath(id), "emoji/"+id+"/image_deleted"); err != nil {
l4g.Error("Failed to rename image when deleting emoji %v", id)
}
}
diff --git a/app/file.go b/app/file.go
index dc7caff41..10fb1425c 100644
--- a/app/file.go
+++ b/app/file.go
@@ -14,23 +14,21 @@ import (
_ "image/gif"
"image/jpeg"
"io"
- "io/ioutil"
"mime/multipart"
"net/http"
"net/url"
- "os"
"path/filepath"
"strings"
"sync"
+ "time"
l4g "github.com/alecthomas/log4go"
"github.com/disintegration/imaging"
- "github.com/mattermost/platform/model"
- "github.com/mattermost/platform/utils"
- s3 "github.com/minio/minio-go"
- "github.com/minio/minio-go/pkg/credentials"
"github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
+
+ "github.com/mattermost/platform/model"
+ "github.com/mattermost/platform/utils"
)
const (
@@ -57,222 +55,8 @@ const (
IMAGE_THUMBNAIL_PIXEL_WIDTH = 120
IMAGE_THUMBNAIL_PIXEL_HEIGHT = 100
IMAGE_PREVIEW_PIXEL_WIDTH = 1024
-
- TEST_FILE_PATH = "/testfile"
)
-// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
-// If signV2 input is false, function always returns signature v4.
-//
-// Additionally this function also takes a user defined region, if set
-// disables automatic region lookup.
-func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) {
- var creds *credentials.Credentials
- if signV2 {
- creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2)
- } else {
- creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4)
- }
- return s3.NewWithCredentials(endpoint, creds, secure, region)
-}
-
-func TestFileConnection() *model.AppError {
- if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
- endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
- accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
- secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
- secure := *utils.Cfg.FileSettings.AmazonS3SSL
- signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
- region := utils.Cfg.FileSettings.AmazonS3Region
- bucket := utils.Cfg.FileSettings.AmazonS3Bucket
-
- s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
- if err != nil {
- return model.NewLocAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error())
- }
-
- exists, err := s3Clnt.BucketExists(bucket)
- if err != nil {
- return model.NewLocAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error())
- }
-
- if !exists {
- l4g.Warn("Bucket specified does not exist. Attempting to create...")
- err := s3Clnt.MakeBucket(bucket, region)
- if err != nil {
- l4g.Error("Unable to create bucket.")
- return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError)
- }
- }
- l4g.Info("Connection to S3 or minio is good. Bucket exists.")
- } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
- f := []byte("testingwrite")
- if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil {
- return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError)
- }
- os.Remove(utils.Cfg.FileSettings.Directory + TEST_FILE_PATH)
- l4g.Info("Able to write files to local storage.")
- } else {
- return model.NewLocAppError("TestFileConnection", "No file driver selected.", nil, "")
- }
-
- return nil
-}
-
-func ReadFile(path string) ([]byte, *model.AppError) {
- if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
- endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
- accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
- secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
- secure := *utils.Cfg.FileSettings.AmazonS3SSL
- signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
- region := utils.Cfg.FileSettings.AmazonS3Region
- s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
- if err != nil {
- return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
- }
- bucket := utils.Cfg.FileSettings.AmazonS3Bucket
- minioObject, err := s3Clnt.GetObject(bucket, path)
- defer minioObject.Close()
- if err != nil {
- return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
- }
- if f, err := ioutil.ReadAll(minioObject); err != nil {
- return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
- } else {
- return f, nil
- }
- } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
- if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil {
- return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error())
- } else {
- return f, nil
- }
- } else {
- return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented)
- }
-}
-
-func MoveFile(oldPath, newPath string) *model.AppError {
- if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
- endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
- accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
- secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
- secure := *utils.Cfg.FileSettings.AmazonS3SSL
- signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
- region := utils.Cfg.FileSettings.AmazonS3Region
- encrypt := false
- if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance {
- encrypt = true
- }
- s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
- if err != nil {
- return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
- }
- bucket := utils.Cfg.FileSettings.AmazonS3Bucket
-
- source := s3.NewSourceInfo(bucket, oldPath, nil)
- destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt))
- if err != nil {
- return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
- }
- if err = s3Clnt.CopyObject(destination, source); err != nil {
- return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
- }
- if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil {
- return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
- }
- } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
- if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil {
- return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
- }
-
- if err := os.Rename(utils.Cfg.FileSettings.Directory+oldPath, utils.Cfg.FileSettings.Directory+newPath); err != nil {
- return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
- }
- } else {
- return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "")
- }
-
- return nil
-}
-
-func WriteFile(f []byte, path string) *model.AppError {
- if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
- endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
- accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
- secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
- secure := *utils.Cfg.FileSettings.AmazonS3SSL
- signV2 := *utils.Cfg.FileSettings.AmazonS3SignV2
- region := utils.Cfg.FileSettings.AmazonS3Region
- encrypt := false
- if *utils.Cfg.FileSettings.AmazonS3SSE && utils.IsLicensed() && *utils.License().Features.Compliance {
- encrypt = true
- }
-
- s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
- if err != nil {
- return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
- }
-
- bucket := utils.Cfg.FileSettings.AmazonS3Bucket
- ext := filepath.Ext(path)
- metaData := S3Metadata(encrypt, "binary/octet-stream")
- if model.IsFileExtImage(ext) {
- metaData = S3Metadata(encrypt, model.GetImageMimeType(ext))
- }
-
- _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil)
- if err != nil {
- return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
- }
- } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
- if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil {
- return err
- }
- } else {
- return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "")
- }
-
- return nil
-}
-
-func writeFileLocally(f []byte, path string) *model.AppError {
- if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil {
- directory, _ := filepath.Abs(filepath.Dir(path))
- return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error())
- }
-
- if err := ioutil.WriteFile(path, f, 0644); err != nil {
- return model.NewLocAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error())
- }
-
- return nil
-}
-
-func openFileWriteStream(path string) (io.Writer, *model.AppError) {
- if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
- return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.s3.app_error", nil, "")
- } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
- if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+path), 0774); err != nil {
- return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.creating_dir.app_error", nil, err.Error())
- }
-
- if fileHandle, err := os.Create(utils.Cfg.FileSettings.Directory + path); err != nil {
- return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.local_server.app_error", nil, err.Error())
- } else {
- fileHandle.Chmod(0644)
- return fileHandle, nil
- }
- }
-
- return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.configured.app_error", nil, "")
-}
-
-func closeFileWriteStream(file io.Writer) {
- file.(*os.File).Close()
-}
-
func GetInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo {
// Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension}
split := strings.SplitN(filename, "/", 5)
@@ -295,7 +79,7 @@ func GetInfoForFilename(post *model.Post, teamId string, filename string) *model
// Open the file and populate the fields of the FileInfo
var info *model.FileInfo
- if data, err := ReadFile(path); err != nil {
+ if data, err := utils.ReadFile(path); err != nil {
l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err)
return nil
} else {
@@ -337,7 +121,7 @@ func FindTeamIdForFilename(post *model.Post, filename string) string {
} else {
for _, team := range teams {
path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name)
- if _, err := ReadFile(path); err == nil {
+ if _, err := utils.ReadFile(path); err == nil {
// Found the team that this file was posted from
return team.Id
}
@@ -484,7 +268,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []*
io.Copy(buf, file)
data := buf.Bytes()
- info, err := DoUploadFile(teamId, channelId, userId, fileHeader.Filename, data)
+ info, err := DoUploadFile(time.Now(), teamId, channelId, userId, fileHeader.Filename, data)
if err != nil {
return nil, err
}
@@ -507,7 +291,7 @@ func UploadFiles(teamId string, channelId string, userId string, fileHeaders []*
return resStruct, nil
}
-func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
+func DoUploadFile(now time.Time, teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
filename := filepath.Base(rawFilename)
info, err := model.GetInfoForBytes(filename, data)
@@ -519,7 +303,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st
info.Id = model.NewId()
info.CreatorId = userId
- pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
+ pathPrefix := now.Format("20060102") + "/teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
info.Path = pathPrefix + filename
if info.IsImage() {
@@ -535,7 +319,7 @@ func DoUploadFile(teamId string, channelId string, userId string, rawFilename st
info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
}
- if err := WriteFile(data, info.Path); err != nil {
+ if err := utils.WriteFile(data, info.Path); err != nil {
return nil, err
}
@@ -652,7 +436,7 @@ func generateThumbnailImage(img image.Image, thumbnailPath string, width int, he
return
}
- if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil {
+ if err := utils.WriteFile(buf.Bytes(), thumbnailPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
return
}
@@ -674,7 +458,7 @@ func generatePreviewImage(img image.Image, previewPath string, width int) {
return
}
- if err := WriteFile(buf.Bytes(), previewPath); err != nil {
+ if err := utils.WriteFile(buf.Bytes(), previewPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
return
}
@@ -687,20 +471,3 @@ func GetFileInfo(fileId string) (*model.FileInfo, *model.AppError) {
return result.Data.(*model.FileInfo), nil
}
}
-
-func S3Metadata(encrypt bool, contentType string) map[string][]string {
- metaData := make(map[string][]string)
- if contentType != "" {
- metaData["Content-Type"] = []string{"contentType"}
- }
- if encrypt {
- metaData["x-amz-server-side-encryption"] = []string{"AES256"}
- }
- return metaData
-}
-
-func CopyMetadata(encrypt bool) map[string]string {
- metaData := make(map[string]string)
- metaData["x-amz-server-side-encryption"] = "AES256"
- return metaData
-}
diff --git a/app/file_test.go b/app/file_test.go
index 683b574b8..962661039 100644
--- a/app/file_test.go
+++ b/app/file_test.go
@@ -4,9 +4,12 @@
package app
import (
+ "fmt"
"testing"
+ "time"
"github.com/mattermost/platform/model"
+ "github.com/mattermost/platform/utils"
)
func TestGeneratePublicLinkHash(t *testing.T) {
@@ -31,3 +34,55 @@ func TestGeneratePublicLinkHash(t *testing.T) {
t.Fatal("hashes for the same file with different salts should not be equal")
}
}
+
+func TestDoUploadFile(t *testing.T) {
+ Setup()
+
+ teamId := model.NewId()
+ channelId := model.NewId()
+ userId := model.NewId()
+ filename := "test"
+ data := []byte("abcd")
+
+ info1, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data)
+ if err != nil {
+ t.Fatal(err)
+ } else {
+ defer func() {
+ <-Srv.Store.FileInfo().PermanentDelete(info1.Id)
+ utils.RemoveFile(info1.Path)
+ }()
+ }
+
+ if info1.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info1.Id, filename) {
+ t.Fatal("stored file at incorrect path", info1.Path)
+ }
+
+ info2, err := DoUploadFile(time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data)
+ if err != nil {
+ t.Fatal(err)
+ } else {
+ defer func() {
+ <-Srv.Store.FileInfo().PermanentDelete(info2.Id)
+ utils.RemoveFile(info2.Path)
+ }()
+ }
+
+ if info2.Path != fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info2.Id, filename) {
+ t.Fatal("stored file at incorrect path", info2.Path)
+ }
+
+ info3, err := DoUploadFile(time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamId, channelId, userId, filename, data)
+ if err != nil {
+ t.Fatal(err)
+ } else {
+ defer func() {
+ <-Srv.Store.FileInfo().PermanentDelete(info3.Id)
+ utils.RemoveFile(info3.Path)
+ }()
+ }
+
+ if info3.Path != fmt.Sprintf("20080305/teams/%v/channels/%v/users/%v/%v/%v", teamId, channelId, userId, info3.Id, filename) {
+ t.Fatal("stored file at incorrect path", info3.Path)
+ }
+}
diff --git a/app/import.go b/app/import.go
index fb7d43cdf..d404dbadd 100644
--- a/app/import.go
+++ b/app/import.go
@@ -12,9 +12,11 @@ import (
"regexp"
"strings"
"sync"
+ "time"
"unicode/utf8"
l4g "github.com/alecthomas/log4go"
+
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
"github.com/mattermost/platform/utils"
@@ -1486,12 +1488,12 @@ func OldImportChannel(channel *model.Channel) *model.Channel {
}
}
-func OldImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
+func OldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
data := buf.Bytes()
- fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data)
+ fileInfo, err := DoUploadFile(timestamp, teamId, channelId, userId, fileName, data)
if err != nil {
return nil, err
}
diff --git a/app/slackimport.go b/app/slackimport.go
index 4470b8323..e57a3a3d1 100644
--- a/app/slackimport.go
+++ b/app/slackimport.go
@@ -401,7 +401,8 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin
}
defer openFile.Close()
- uploadedFile, err := OldImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name))
+ timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(sPost.TimeStamp))
+ uploadedFile, err := OldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name))
if err != nil {
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
return nil, false
diff --git a/app/user.go b/app/user.go
index 813421a5c..40e32c282 100644
--- a/app/user.go
+++ b/app/user.go
@@ -758,7 +758,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
} else {
path := "users/" + user.Id + "/profile.png"
- if data, err := ReadFile(path); err != nil {
+ if data, err := utils.ReadFile(path); err != nil {
readFailed = true
if img, err = CreateProfileImage(user.Username, user.Id); err != nil {
@@ -766,7 +766,7 @@ func GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
}
if user.LastPictureUpdate == 0 {
- if err := WriteFile(img, path); err != nil {
+ if err := utils.WriteFile(img, path); err != nil {
return nil, false, err
}
}
@@ -819,7 +819,7 @@ func SetProfileImage(userId string, imageData *multipart.FileHeader) *model.AppE
path := "users/" + userId + "/profile.png"
- if err := WriteFile(buf.Bytes(), path); err != nil {
+ if err := utils.WriteFile(buf.Bytes(), path); err != nil {
return model.NewLocAppError("SetProfileImage", "api.user.upload_profile_user.upload_profile.app_error", nil, "")
}
diff --git a/cmd/platform/server.go b/cmd/platform/server.go
index ac8f3baff..f5be5a5fc 100644
--- a/cmd/platform/server.go
+++ b/cmd/platform/server.go
@@ -67,7 +67,7 @@ func runServer(configFileLocation string) {
*utils.Cfg.ServiceSettings.EnableDeveloper = true
}
- if err := app.TestFileConnection(); err != nil {
+ if err := utils.TestFileConnection(); err != nil {
l4g.Error("Problem with file storage settings: " + err.Error())
}
diff --git a/i18n/en.json b/i18n/en.json
index 900a6865f..338eba46e 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -1234,32 +1234,40 @@
"translation": "Unable to move file locally."
},
{
- "id": "api.file.open_file_write_stream.configured.app_error",
+ "id": "api.file.read_file.configured.app_error",
"translation": "File storage not configured properly. Please configure for either S3 or local server file storage."
},
{
- "id": "api.file.open_file_write_stream.creating_dir.app_error",
- "translation": "Encountered an error creating the directory for the new file"
+ "id": "api.file.read_file.get.app_error",
+ "translation": "Unable to get file from S3"
},
{
- "id": "api.file.open_file_write_stream.local_server.app_error",
- "translation": "Encountered an error writing to local server storage"
+ "id": "api.file.read_file.reading_local.app_error",
+ "translation": "Encountered an error reading from local server storage"
},
{
- "id": "api.file.open_file_write_stream.s3.app_error",
- "translation": "S3 is not supported."
+ "id": "utils.file.remove_file.local.app_error",
+ "translation": "Encountered an error removing file from local server file storage."
},
{
- "id": "api.file.read_file.configured.app_error",
+ "id": "utils.file.remove_file.s3.app_error",
+ "translation": "Encountered an error removing file from S3."
+ },
+ {
+ "id": "utils.file.remove_file.configured.app_error",
"translation": "File storage not configured properly. Please configure for either S3 or local server file storage."
},
{
- "id": "api.file.read_file.get.app_error",
- "translation": "Unable to get file from S3"
+ "id": "utils.file.remove_directory.local.app_error",
+ "translation": "Encountered an error removing directory from local server file storage."
},
{
- "id": "api.file.read_file.reading_local.app_error",
- "translation": "Encountered an error reading from local server storage"
+ "id": "utils.file.remove_directory.s3.app_error",
+ "translation": "Encountered an error removing directory from S3."
+ },
+ {
+ "id": "utils.file.remove_directory.configured.app_error",
+ "translation": "File storage not configured properly. Please configure for either S3 or local server file storage."
},
{
"id": "api.file.upload_file.bad_parse.app_error",
@@ -5576,6 +5584,10 @@
"translation": "We couldn't save or update the file info"
},
{
+ "id": "store.sql_file_info.permanent_delete.app_error",
+ "translation": "We couldn't permanently delete the file info"
+ },
+ {
"id": "store.sql_job.delete.app_error",
"translation": "We couldn't delete the job"
},
diff --git a/model/file_info.go b/model/file_info.go
index 8b5684127..3071e6d4b 100644
--- a/model/file_info.go
+++ b/model/file_info.go
@@ -80,6 +80,9 @@ func (o *FileInfo) PreSave() {
if o.CreateAt == 0 {
o.CreateAt = GetMillis()
+ }
+
+ if o.UpdateAt < o.CreateAt {
o.UpdateAt = o.CreateAt
}
}
diff --git a/store/sql_file_info_store.go b/store/sql_file_info_store.go
index 3fe62121c..4bf6c9c73 100644
--- a/store/sql_file_info_store.go
+++ b/store/sql_file_info_store.go
@@ -97,7 +97,7 @@ func (fs SqlFileInfoStore) Get(id string) StoreChannel {
if err == sql.ErrNoRows {
result.Err = model.NewAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error(), http.StatusNotFound)
} else {
- result.Err = model.NewLocAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error())
+ result.Err = model.NewAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error(), http.StatusInternalServerError)
}
} else {
result.Data = info
@@ -127,7 +127,7 @@ func (fs SqlFileInfoStore) GetByPath(path string) StoreChannel {
Path = :Path
AND DeleteAt = 0
LIMIT 1`, map[string]interface{}{"Path": path}); err != nil {
- result.Err = model.NewLocAppError("SqlFileInfoStore.GetByPath", "store.sql_file_info.get_by_path.app_error", nil, "path="+path+", "+err.Error())
+ result.Err = model.NewAppError("SqlFileInfoStore.GetByPath", "store.sql_file_info.get_by_path.app_error", nil, "path="+path+", "+err.Error(), http.StatusInternalServerError)
} else {
result.Data = info
}
@@ -139,7 +139,7 @@ func (fs SqlFileInfoStore) GetByPath(path string) StoreChannel {
return storeChannel
}
-func (s SqlFileInfoStore) InvalidateFileInfosForPostCache(postId string) {
+func (fs SqlFileInfoStore) InvalidateFileInfosForPostCache(postId string) {
fileInfoCache.Remove(postId)
}
@@ -190,8 +190,8 @@ func (fs SqlFileInfoStore) GetForPost(postId string, readFromMaster bool, allowF
AND DeleteAt = 0
ORDER BY
CreateAt`, map[string]interface{}{"PostId": postId}); err != nil {
- result.Err = model.NewLocAppError("SqlFileInfoStore.GetForPost",
- "store.sql_file_info.get_for_post.app_error", nil, "post_id="+postId+", "+err.Error())
+ result.Err = model.NewAppError("SqlFileInfoStore.GetForPost",
+ "store.sql_file_info.get_for_post.app_error", nil, "post_id="+postId+", "+err.Error(), http.StatusInternalServerError)
} else {
if len(infos) > 0 {
fileInfoCache.AddWithExpiresInSecs(postId, infos, FILE_INFO_CACHE_SEC)
@@ -221,8 +221,8 @@ func (fs SqlFileInfoStore) AttachToPost(fileId, postId string) StoreChannel {
WHERE
Id = :Id
AND PostId = ''`, map[string]interface{}{"PostId": postId, "Id": fileId}); err != nil {
- result.Err = model.NewLocAppError("SqlFileInfoStore.AttachToPost",
- "store.sql_file_info.attach_to_post.app_error", nil, "post_id="+postId+", file_id="+fileId+", err="+err.Error())
+ result.Err = model.NewAppError("SqlFileInfoStore.AttachToPost",
+ "store.sql_file_info.attach_to_post.app_error", nil, "post_id="+postId+", file_id="+fileId+", err="+err.Error(), http.StatusInternalServerError)
}
storeChannel <- result
@@ -245,8 +245,8 @@ func (fs SqlFileInfoStore) DeleteForPost(postId string) StoreChannel {
DeleteAt = :DeleteAt
WHERE
PostId = :PostId`, map[string]interface{}{"DeleteAt": model.GetMillis(), "PostId": postId}); err != nil {
- result.Err = model.NewLocAppError("SqlFileInfoStore.DeleteForPost",
- "store.sql_file_info.delete_for_post.app_error", nil, "post_id="+postId+", err="+err.Error())
+ result.Err = model.NewAppError("SqlFileInfoStore.DeleteForPost",
+ "store.sql_file_info.delete_for_post.app_error", nil, "post_id="+postId+", err="+err.Error(), http.StatusInternalServerError)
} else {
result.Data = postId
}
@@ -257,3 +257,25 @@ func (fs SqlFileInfoStore) DeleteForPost(postId string) StoreChannel {
return storeChannel
}
+
+func (fs SqlFileInfoStore) PermanentDelete(fileId string) StoreChannel {
+ storeChannel := make(StoreChannel, 1)
+
+ go func() {
+ result := StoreResult{}
+
+ if _, err := fs.GetMaster().Exec(
+ `DELETE FROM
+ FileInfo
+ WHERE
+ Id = :FileId`, map[string]interface{}{"FileId": fileId}); err != nil {
+ result.Err = model.NewAppError("SqlFileInfoStore.PermanentDelete",
+ "store.sql_file_info.permanent_delete.app_error", nil, "file_id="+fileId+", err="+err.Error(), http.StatusInternalServerError)
+ }
+
+ storeChannel <- result
+ close(storeChannel)
+ }()
+
+ return storeChannel
+}
diff --git a/store/sql_file_info_store_test.go b/store/sql_file_info_store_test.go
index daec06269..fecd862c0 100644
--- a/store/sql_file_info_store_test.go
+++ b/store/sql_file_info_store_test.go
@@ -25,6 +25,9 @@ func TestFileInfoSaveGet(t *testing.T) {
} else {
info = returned
}
+ defer func() {
+ <-store.FileInfo().PermanentDelete(info.Id)
+ }()
if result := <-store.FileInfo().Get(info.Id); result.Err != nil {
t.Fatal(result.Err)
@@ -43,6 +46,9 @@ func TestFileInfoSaveGet(t *testing.T) {
if result := <-store.FileInfo().Get(info2.Id); result.Err == nil {
t.Fatal("shouldn't have gotten deleted file")
}
+ defer func() {
+ <-store.FileInfo().PermanentDelete(info2.Id)
+ }()
}
func TestFileInfoSaveGetByPath(t *testing.T) {
@@ -60,6 +66,9 @@ func TestFileInfoSaveGetByPath(t *testing.T) {
} else {
info = returned
}
+ defer func() {
+ <-store.FileInfo().PermanentDelete(info.Id)
+ }()
if result := <-store.FileInfo().GetByPath(info.Path); result.Err != nil {
t.Fatal(result.Err)
@@ -78,6 +87,9 @@ func TestFileInfoSaveGetByPath(t *testing.T) {
if result := <-store.FileInfo().GetByPath(info2.Id); result.Err == nil {
t.Fatal("shouldn't have gotten deleted file")
}
+ defer func() {
+ <-store.FileInfo().PermanentDelete(info2.Id)
+ }()
}
func TestFileInfoGetForPost(t *testing.T) {
@@ -112,6 +124,9 @@ func TestFileInfoGetForPost(t *testing.T) {
for i, info := range infos {
infos[i] = Must(store.FileInfo().Save(info)).(*model.FileInfo)
+ defer func(id string) {
+ <-store.FileInfo().PermanentDelete(id)
+ }(infos[i].Id)
}
if result := <-store.FileInfo().GetForPost(postId, true, false); result.Err != nil {
@@ -143,6 +158,9 @@ func TestFileInfoAttachToPost(t *testing.T) {
CreatorId: userId,
Path: "file.txt",
})).(*model.FileInfo)
+ defer func() {
+ <-store.FileInfo().PermanentDelete(info1.Id)
+ }()
if len(info1.PostId) != 0 {
t.Fatal("file shouldn't have a PostId")
@@ -162,6 +180,9 @@ func TestFileInfoAttachToPost(t *testing.T) {
CreatorId: userId,
Path: "file.txt",
})).(*model.FileInfo)
+ defer func() {
+ <-store.FileInfo().PermanentDelete(info2.Id)
+ }()
if result := <-store.FileInfo().AttachToPost(info2.Id, postId); result.Err != nil {
t.Fatal(result.Err)
@@ -208,6 +229,9 @@ func TestFileInfoDeleteForPost(t *testing.T) {
for i, info := range infos {
infos[i] = Must(store.FileInfo().Save(info)).(*model.FileInfo)
+ defer func(id string) {
+ <-store.FileInfo().PermanentDelete(id)
+ }(infos[i].Id)
}
if result := <-store.FileInfo().DeleteForPost(postId); result.Err != nil {
@@ -218,3 +242,17 @@ func TestFileInfoDeleteForPost(t *testing.T) {
t.Fatal("shouldn't have returned any file infos")
}
}
+
+func TestFileInfoPermanentDelete(t *testing.T) {
+ Setup()
+
+ info := Must(store.FileInfo().Save(&model.FileInfo{
+ PostId: model.NewId(),
+ CreatorId: model.NewId(),
+ Path: "file.txt",
+ })).(*model.FileInfo)
+
+ if result := <-store.FileInfo().PermanentDelete(info.Id); result.Err != nil {
+ t.Fatal(result.Err)
+ }
+}
diff --git a/store/store.go b/store/store.go
index e86b5f116..0aa4d5c41 100644
--- a/store/store.go
+++ b/store/store.go
@@ -386,6 +386,7 @@ type FileInfoStore interface {
InvalidateFileInfosForPostCache(postId string)
AttachToPost(fileId string, postId string) StoreChannel
DeleteForPost(postId string) StoreChannel
+ PermanentDelete(fileId string) StoreChannel
}
type ReactionStore interface {
diff --git a/utils/file.go b/utils/file.go
new file mode 100644
index 000000000..efed1e954
--- /dev/null
+++ b/utils/file.go
@@ -0,0 +1,313 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package utils
+
+import (
+ "bytes"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path/filepath"
+
+ l4g "github.com/alecthomas/log4go"
+ s3 "github.com/minio/minio-go"
+ "github.com/minio/minio-go/pkg/credentials"
+
+ "github.com/mattermost/platform/model"
+)
+
+const (
+ TEST_FILE_PATH = "/testfile"
+)
+
+// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
+// If signV2 input is false, function always returns signature v4.
+//
+// Additionally this function also takes a user defined region, if set
+// disables automatic region lookup.
+func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) {
+ var creds *credentials.Credentials
+ if signV2 {
+ creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2)
+ } else {
+ creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4)
+ }
+ return s3.NewWithCredentials(endpoint, creds, secure, region)
+}
+
+func TestFileConnection() *model.AppError {
+ if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
+ endpoint := Cfg.FileSettings.AmazonS3Endpoint
+ accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
+ secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
+ secure := *Cfg.FileSettings.AmazonS3SSL
+ signV2 := *Cfg.FileSettings.AmazonS3SignV2
+ region := Cfg.FileSettings.AmazonS3Region
+ bucket := Cfg.FileSettings.AmazonS3Bucket
+
+ s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
+ if err != nil {
+ return model.NewLocAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error())
+ }
+
+ exists, err := s3Clnt.BucketExists(bucket)
+ if err != nil {
+ return model.NewLocAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error())
+ }
+
+ if !exists {
+ l4g.Warn("Bucket specified does not exist. Attempting to create...")
+ err := s3Clnt.MakeBucket(bucket, region)
+ if err != nil {
+ l4g.Error("Unable to create bucket.")
+ return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError)
+ }
+ }
+ l4g.Info("Connection to S3 or minio is good. Bucket exists.")
+ } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
+ f := []byte("testingwrite")
+ if err := writeFileLocally(f, Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil {
+ return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError)
+ }
+ os.Remove(Cfg.FileSettings.Directory + TEST_FILE_PATH)
+ l4g.Info("Able to write files to local storage.")
+ } else {
+ return model.NewLocAppError("TestFileConnection", "No file driver selected.", nil, "")
+ }
+
+ return nil
+}
+
+func ReadFile(path string) ([]byte, *model.AppError) {
+ if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
+ endpoint := Cfg.FileSettings.AmazonS3Endpoint
+ accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
+ secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
+ secure := *Cfg.FileSettings.AmazonS3SSL
+ signV2 := *Cfg.FileSettings.AmazonS3SignV2
+ region := Cfg.FileSettings.AmazonS3Region
+ s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
+ if err != nil {
+ return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
+ }
+ bucket := Cfg.FileSettings.AmazonS3Bucket
+ minioObject, err := s3Clnt.GetObject(bucket, path)
+ defer minioObject.Close()
+ if err != nil {
+ return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
+ }
+ if f, err := ioutil.ReadAll(minioObject); err != nil {
+ return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
+ } else {
+ return f, nil
+ }
+ } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
+ if f, err := ioutil.ReadFile(Cfg.FileSettings.Directory + path); err != nil {
+ return nil, model.NewLocAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error())
+ } else {
+ return f, nil
+ }
+ } else {
+ return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented)
+ }
+}
+
+func MoveFile(oldPath, newPath string) *model.AppError {
+ if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
+ endpoint := Cfg.FileSettings.AmazonS3Endpoint
+ accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
+ secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
+ secure := *Cfg.FileSettings.AmazonS3SSL
+ signV2 := *Cfg.FileSettings.AmazonS3SignV2
+ region := Cfg.FileSettings.AmazonS3Region
+ encrypt := false
+ if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance {
+ encrypt = true
+ }
+ s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
+ if err != nil {
+ return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
+ }
+ bucket := Cfg.FileSettings.AmazonS3Bucket
+
+ source := s3.NewSourceInfo(bucket, oldPath, nil)
+ destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt))
+ if err != nil {
+ return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
+ }
+ if err = s3Clnt.CopyObject(destination, source); err != nil {
+ return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
+ }
+ if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil {
+ return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
+ }
+ } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
+ if err := os.MkdirAll(filepath.Dir(Cfg.FileSettings.Directory+newPath), 0774); err != nil {
+ return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
+ }
+
+ if err := os.Rename(Cfg.FileSettings.Directory+oldPath, Cfg.FileSettings.Directory+newPath); err != nil {
+ return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
+ }
+ } else {
+ return model.NewLocAppError("moveFile", "api.file.move_file.configured.app_error", nil, "")
+ }
+
+ return nil
+}
+
+func WriteFile(f []byte, path string) *model.AppError {
+ if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
+ endpoint := Cfg.FileSettings.AmazonS3Endpoint
+ accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
+ secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
+ secure := *Cfg.FileSettings.AmazonS3SSL
+ signV2 := *Cfg.FileSettings.AmazonS3SignV2
+ region := Cfg.FileSettings.AmazonS3Region
+ encrypt := false
+ if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance {
+ encrypt = true
+ }
+
+ s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
+ if err != nil {
+ return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
+ }
+
+ bucket := Cfg.FileSettings.AmazonS3Bucket
+ ext := filepath.Ext(path)
+ metaData := S3Metadata(encrypt, "binary/octet-stream")
+ if model.IsFileExtImage(ext) {
+ metaData = S3Metadata(encrypt, model.GetImageMimeType(ext))
+ }
+
+ _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil)
+ if err != nil {
+ return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
+ }
+ } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
+ if err := writeFileLocally(f, Cfg.FileSettings.Directory+path); err != nil {
+ return err
+ }
+ } else {
+ return model.NewLocAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "")
+ }
+
+ return nil
+}
+
+func writeFileLocally(f []byte, path string) *model.AppError {
+ if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil {
+ directory, _ := filepath.Abs(filepath.Dir(path))
+ return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error())
+ }
+
+ if err := ioutil.WriteFile(path, f, 0644); err != nil {
+ return model.NewLocAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error())
+ }
+
+ return nil
+}
+
+func RemoveFile(path string) *model.AppError {
+ if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
+ endpoint := Cfg.FileSettings.AmazonS3Endpoint
+ accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
+ secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
+ secure := *Cfg.FileSettings.AmazonS3SSL
+ signV2 := *Cfg.FileSettings.AmazonS3SignV2
+ region := Cfg.FileSettings.AmazonS3Region
+
+ s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
+ if err != nil {
+ return model.NewLocAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error())
+ }
+
+ bucket := Cfg.FileSettings.AmazonS3Bucket
+ if err := s3Clnt.RemoveObject(bucket, path); err != nil {
+ return model.NewLocAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error())
+ }
+ } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
+ if err := os.Remove(Cfg.FileSettings.Directory + path); err != nil {
+ return model.NewLocAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error())
+ }
+ } else {
+ return model.NewLocAppError("RemoveFile", "utils.file.remove_file.configured.app_error", nil, "")
+ }
+
+ return nil
+}
+
+func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string {
+ out := make(chan string, 1)
+
+ go func() {
+ defer close(out)
+
+ for {
+ info, done := <-in
+
+ if !done {
+ break
+ }
+
+ out <- info.Key
+ }
+ }()
+
+ return out
+}
+
+func RemoveDirectory(path string) *model.AppError {
+ if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
+ endpoint := Cfg.FileSettings.AmazonS3Endpoint
+ accessKey := Cfg.FileSettings.AmazonS3AccessKeyId
+ secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey
+ secure := *Cfg.FileSettings.AmazonS3SSL
+ signV2 := *Cfg.FileSettings.AmazonS3SignV2
+ region := Cfg.FileSettings.AmazonS3Region
+
+ s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region)
+ if err != nil {
+ return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error())
+ }
+
+ doneCh := make(chan struct{})
+
+ bucket := Cfg.FileSettings.AmazonS3Bucket
+ for err := range s3Clnt.RemoveObjects(bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(bucket, path, true, doneCh))) {
+ if err.Err != nil {
+ doneCh <- struct{}{}
+ return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error())
+ }
+ }
+
+ close(doneCh)
+ } else if Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
+ if err := os.RemoveAll(Cfg.FileSettings.Directory + path); err != nil {
+ return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error())
+ }
+ } else {
+ return model.NewLocAppError("RemoveDirectory", "utils.file.remove_directory.configured.app_error", nil, "")
+ }
+
+ return nil
+}
+
+func S3Metadata(encrypt bool, contentType string) map[string][]string {
+ metaData := make(map[string][]string)
+ if contentType != "" {
+ metaData["Content-Type"] = []string{"contentType"}
+ }
+ if encrypt {
+ metaData["x-amz-server-side-encryption"] = []string{"AES256"}
+ }
+ return metaData
+}
+
+func CopyMetadata(encrypt bool) map[string]string {
+ metaData := make(map[string]string)
+ metaData["x-amz-server-side-encryption"] = "AES256"
+ return metaData
+}
diff --git a/utils/file_test.go b/utils/file_test.go
new file mode 100644
index 000000000..202a5354e
--- /dev/null
+++ b/utils/file_test.go
@@ -0,0 +1,122 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package utils
+
+import (
+ "testing"
+
+ "github.com/mattermost/platform/model"
+)
+
+func TestReadWriteFile(t *testing.T) {
+ TranslationsPreInit()
+ LoadConfig("config.json")
+ InitTranslations(Cfg.LocalizationSettings)
+
+ b := []byte("test")
+ path := "tests/" + model.NewId()
+
+ if err := WriteFile(b, path); err != nil {
+ t.Fatal(err)
+ }
+ defer RemoveFile(path)
+
+ if read, err := ReadFile(path); err != nil {
+ t.Fatal(err)
+ } else if readString := string(read); readString != "test" {
+ t.Fatal("should've read back contents of file")
+ }
+}
+
+func TestMoveFile(t *testing.T) {
+ TranslationsPreInit()
+ LoadConfig("config.json")
+ InitTranslations(Cfg.LocalizationSettings)
+
+ b := []byte("test")
+ path1 := "tests/" + model.NewId()
+ path2 := "tests/" + model.NewId()
+
+ if err := WriteFile(b, path1); err != nil {
+ t.Fatal(err)
+ }
+ defer RemoveFile(path1)
+
+ if err := MoveFile(path1, path2); err != nil {
+ t.Fatal(err)
+ }
+ defer RemoveFile(path2)
+
+ if _, err := ReadFile(path1); err == nil {
+ t.Fatal("file should no longer exist at old path")
+ }
+
+ if _, err := ReadFile(path2); err != nil {
+ t.Fatal("file should exist at new path", err)
+ }
+}
+
+func TestRemoveFile(t *testing.T) {
+ TranslationsPreInit()
+ LoadConfig("config.json")
+ InitTranslations(Cfg.LocalizationSettings)
+
+ b := []byte("test")
+ path := "tests/" + model.NewId()
+
+ if err := WriteFile(b, path); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := RemoveFile(path); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := ReadFile(path); err == nil {
+ t.Fatal("should've removed file")
+ }
+
+ if err := WriteFile(b, "tests2/foo"); err != nil {
+ t.Fatal(err)
+ }
+ if err := WriteFile(b, "tests2/bar"); err != nil {
+ t.Fatal(err)
+ }
+ if err := WriteFile(b, "tests2/asdf"); err != nil {
+ t.Fatal(err)
+ }
+ if err := RemoveDirectory("tests2"); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestRemoveDirectory(t *testing.T) {
+ TranslationsPreInit()
+ LoadConfig("config.json")
+ InitTranslations(Cfg.LocalizationSettings)
+
+ b := []byte("test")
+
+ if err := WriteFile(b, "tests2/foo"); err != nil {
+ t.Fatal(err)
+ }
+ if err := WriteFile(b, "tests2/bar"); err != nil {
+ t.Fatal(err)
+ }
+ if err := WriteFile(b, "tests2/aaa"); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := RemoveDirectory("tests2"); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := ReadFile("tests2/foo"); err == nil {
+ t.Fatal("should've removed file")
+ } else if _, err := ReadFile("tests2/bar"); err == nil {
+ t.Fatal("should've removed file")
+ } else if _, err := ReadFile("tests2/asdf"); err == nil {
+ t.Fatal("should've removed file")
+ }
+}
diff --git a/utils/time.go b/utils/time.go
index 7d5afdf8f..baddd7329 100644
--- a/utils/time.go
+++ b/utils/time.go
@@ -8,6 +8,10 @@ func MillisFromTime(t time.Time) int64 {
return t.UnixNano() / int64(time.Millisecond)
}
+func TimeFromMillis(millis int64) time.Time {
+ return time.Unix(0, millis*int64(time.Millisecond))
+}
+
func StartOfDay(t time.Time) time.Time {
year, month, day := t.Date()
return time.Date(year, month, day, 0, 0, 0, 0, t.Location())
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-05-17 06:46:42 +0000