From f02620b291b988848392c455a7719699f6b5c00f Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 26 Oct 2016 05:21:07 -0700 Subject: Moving away from goamz to use minio-go instead. (#4193) minio-go does fully managed way of handling S3 API requests - Automatic bucket location management across all s3 regions. - Transparently upload large files in multipart if file 64MB or larger. - Right GetObject() API provides compatibility with io.ReadWriteSeeker interface. - Various other APIs including bulk deletes, server side object copy, bucket policies and bucket notifications. Fixes #4182 --- api/file.go | 110 ++++++++++++++++++++++++------------------------------------ 1 file changed, 43 insertions(+), 67 deletions(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index a71bc71da..3fc61f469 100644 --- a/api/file.go +++ b/api/file.go @@ -18,21 +18,21 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "strconv" "strings" "sync" - "time" l4g "github.com/alecthomas/log4go" "github.com/disintegration/imaging" - "github.com/goamz/goamz/aws" - "github.com/goamz/goamz/s3" "github.com/gorilla/mux" "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" "github.com/rwcarlsen/goexif/exif" _ "golang.org/x/image/bmp" + + s3 "github.com/minio/minio-go" ) const ( @@ -738,25 +738,22 @@ func getInfoForFilename(post *model.Post, teamId string, filename string) *model func WriteFile(f []byte, path string) *model.AppError { if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - var auth aws.Auth - auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId - auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey - - s := s3.New(auth, awsRegion()) - bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket) - + endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint + accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *utils.Cfg.FileSettings.AmazonS3SSL + s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure) + if err != nil { + return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) + } + bucket := utils.Cfg.FileSettings.AmazonS3Bucket ext := filepath.Ext(path) - var err error if model.IsFileExtImage(ext) { - options := s3.Options{} - err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options) - + _, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), model.GetImageMimeType(ext)) } else { - options := s3.Options{} - err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options) + _, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), "binary/octet-stream") } - if err != nil { return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) } @@ -773,28 +770,24 @@ func WriteFile(f []byte, path string) *model.AppError { func MoveFile(oldPath, newPath string) *model.AppError { if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - fileBytes, _ := ReadFile(oldPath) - - if fileBytes == nil { - return model.NewLocAppError("moveFile", "api.file.move_file.get_from_s3.app_error", nil, "") + endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint + accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *utils.Cfg.FileSettings.AmazonS3SSL + s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure) + if err != nil { + return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error()) } + bucket := utils.Cfg.FileSettings.AmazonS3Bucket - var auth aws.Auth - auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId - auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey - - s := s3.New(auth, awsRegion()) - bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket) - - if err := bucket.Del(oldPath); err != nil { + var copyConds = s3.NewCopyConditions() + if err = s3Clnt.CopyObject(bucket, newPath, "/"+path.Join(bucket, oldPath), copyConds); err != nil { return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) } - - if err := WriteFile(fileBytes, newPath); err != nil { - return err + if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { + return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error()) } } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil { return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error()) } @@ -824,26 +817,23 @@ func writeFileLocally(f []byte, path string) *model.AppError { func ReadFile(path string) ([]byte, *model.AppError) { if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - var auth aws.Auth - auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId - auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey - - s := s3.New(auth, awsRegion()) - bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket) - - // try to get the file from S3 with some basic retry logic - tries := 0 - for { - tries++ - - f, err := bucket.Get(path) - - if f != nil { - return f, nil - } else if tries >= 3 { - return nil, model.NewLocAppError("ReadFile", "api.file.read_file.get.app_error", nil, "path="+path+", err="+err.Error()) - } - time.Sleep(3000 * time.Millisecond) + endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint + accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId + secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey + secure := *utils.Cfg.FileSettings.AmazonS3SSL + s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure) + if err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } + bucket := utils.Cfg.FileSettings.AmazonS3Bucket + reader, err := s3Clnt.GetObject(bucket, path) + if err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } + if f, err := ioutil.ReadAll(reader); err != nil { + return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error()) + } else { + return f, nil } } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil { @@ -878,17 +868,3 @@ func openFileWriteStream(path string) (io.Writer, *model.AppError) { func closeFileWriteStream(file io.Writer) { file.(*os.File).Close() } - -func awsRegion() aws.Region { - if region, ok := aws.Regions[utils.Cfg.FileSettings.AmazonS3Region]; ok { - return region - } - - return aws.Region{ - Name: utils.Cfg.FileSettings.AmazonS3Region, - S3Endpoint: utils.Cfg.FileSettings.AmazonS3Endpoint, - S3BucketEndpoint: utils.Cfg.FileSettings.AmazonS3BucketEndpoint, - S3LocationConstraint: *utils.Cfg.FileSettings.AmazonS3LocationConstraint, - S3LowercaseBucket: *utils.Cfg.FileSettings.AmazonS3LowercaseBucket, - } -} -- cgit v1.2.3-1-g7c22