diff options
Diffstat (limited to 'utils/file_backend_s3.go')
-rw-r--r-- | utils/file_backend_s3.go | 294 |
1 files changed, 0 insertions, 294 deletions
diff --git a/utils/file_backend_s3.go b/utils/file_backend_s3.go deleted file mode 100644 index f5f96f878..000000000 --- a/utils/file_backend_s3.go +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - - s3 "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/encrypt" - - "github.com/mattermost/mattermost-server/mlog" - "github.com/mattermost/mattermost-server/model" -) - -type S3FileBackend struct { - endpoint string - accessKey string - secretKey string - secure bool - signV2 bool - region string - bucket string - encrypt bool - trace bool -} - -// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. -// If signV2 input is false, function always returns signature v4. -// -// Additionally this function also takes a user defined region, if set -// disables automatic region lookup. -func (b *S3FileBackend) s3New() (*s3.Client, error) { - var creds *credentials.Credentials - - if b.accessKey == "" && b.secretKey == "" { - creds = credentials.NewIAM("") - } else if b.signV2 { - creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2) - } else { - creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) - } - - s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) - if err != nil { - return nil, err - } - - if b.trace { - s3Clnt.TraceOn(os.Stdout) - } - - return s3Clnt, nil -} - -func (b *S3FileBackend) TestConnection() *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.connection.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - exists, err := s3Clnt.BucketExists(b.bucket) - if err != nil { - return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucket_exists.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if !exists { - mlog.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(b.bucket, b.region) - if err != nil { - mlog.Error("Unable to create bucket.") - return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucked_create.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } - mlog.Info("Connection to S3 or minio is good. Bucket exists.") - return nil -} - -// Caller must close the first return value -func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) { - s3Clnt, err := b.s3New() - if err != nil { - return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) - if err != nil { - return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return minioObject, nil -} - -func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { - s3Clnt, err := b.s3New() - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - defer minioObject.Close() - if f, err := ioutil.ReadAll(minioObject); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } -} - -func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) { - s3Clnt, err := b.s3New() - - if err != nil { - return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - _, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{}) - - if err == nil { - return true, nil - } - - if err.(s3.ErrorResponse).Code == "NoSuchKey" { - return false, nil - } - - return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) -} - -func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - source := s3.NewSourceInfo(b.bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) - if err != nil { - return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil -} - -func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - source := s3.NewSourceInfo(b.bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - return nil -} - -func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) { - s3Clnt, err := b.s3New() - if err != nil { - return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - var contentType string - if ext := filepath.Ext(path); model.IsFileExtImage(ext) { - contentType = model.GetImageMimeType(ext) - } else { - contentType = "binary/octet-stream" - } - - options := s3PutOptions(b.encrypt, contentType) - var buf bytes.Buffer - _, err = buf.ReadFrom(fr) - if err != nil { - return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - written, err := s3Clnt.PutObject(b.bucket, path, &buf, int64(buf.Len()), options) - if err != nil { - return written, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return written, nil -} - -func (b *S3FileBackend) RemoveFile(path string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return nil -} - -func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { - out := make(chan string, 1) - - go func() { - defer close(out) - - for { - info, done := <-in - - if !done { - break - } - - out <- info.Key - } - }() - - return out -} - -func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) { - var paths []string - - s3Clnt, err := b.s3New() - if err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - defer close(doneCh) - - for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { - if object.Err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) - } - paths = append(paths, strings.Trim(object.Key, "/")) - } - - return &paths, nil -} - -func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { - s3Clnt, err := b.s3New() - if err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { - if err.Err != nil { - doneCh <- struct{}{} - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) - } - } - - close(doneCh) - return nil -} - -func s3PutOptions(encrypted bool, contentType string) s3.PutObjectOptions { - options := s3.PutObjectOptions{} - if encrypted { - options.ServerSideEncryption = encrypt.NewSSE() - } - options.ContentType = contentType - - return options -} - -func CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError { - if len(settings.AmazonS3Bucket) == 0 { - return model.NewAppError("S3File", "api.admin.test_s3.missing_s3_bucket", nil, "", http.StatusBadRequest) - } - - // if S3 endpoint is not set call the set defaults to set that - if len(settings.AmazonS3Endpoint) == 0 { - settings.SetDefaults() - } - - return nil -} |