Moving away from goamz to use minio-go instead. (#4193)

minio-go does fully managed way of handling S3 API requests

  - Automatic bucket location management across all s3 regions.
  - Transparently upload large files in multipart if file 64MB
    or larger.
  - Right GetObject() API provides compatibility with
    io.ReadWriteSeeker interface.
  - Various other APIs including bulk deletes, server side object
    copy, bucket policies and bucket notifications.

Fixes #4182

4 files changed, 84 insertions, 102 deletions

diff --git a/api/file.go b/api/file.go
index a71bc71da..3fc61f469 100644
--- a/api/file.go
+++ b/api/file.go
@@ -18,21 +18,21 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
 	l4g "github.com/alecthomas/log4go"
 	"github.com/disintegration/imaging"
-	"github.com/goamz/goamz/aws"
-	"github.com/goamz/goamz/s3"
 	"github.com/gorilla/mux"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
 	"github.com/rwcarlsen/goexif/exif"
 	_ "golang.org/x/image/bmp"
+
+	s3 "github.com/minio/minio-go"
 )
 
 const (
@@ -738,25 +738,22 @@ func getInfoForFilename(post *model.Post, teamId string, filename string) *model
 
 func WriteFile(f []byte, path string) *model.AppError {
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
-		var auth aws.Auth
-		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
-		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
-
-		s := s3.New(auth, awsRegion())
-		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
-
+		endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
+		accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
+		secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
+		secure := *utils.Cfg.FileSettings.AmazonS3SSL
+		s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
+		if err != nil {
+			return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
+		}
+		bucket := utils.Cfg.FileSettings.AmazonS3Bucket
 		ext := filepath.Ext(path)
 
-		var err error
 		if model.IsFileExtImage(ext) {
-			options := s3.Options{}
-			err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)
-
+			_, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), model.GetImageMimeType(ext))
 		} else {
-			options := s3.Options{}
-			err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
+			_, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), "binary/octet-stream")
 		}
-
 		if err != nil {
 			return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
 		}
@@ -773,28 +770,24 @@ func WriteFile(f []byte, path string) *model.AppError {
 
 func MoveFile(oldPath, newPath string) *model.AppError {
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
-		fileBytes, _ := ReadFile(oldPath)
-
-		if fileBytes == nil {
-			return model.NewLocAppError("moveFile", "api.file.move_file.get_from_s3.app_error", nil, "")
+		endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
+		accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
+		secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
+		secure := *utils.Cfg.FileSettings.AmazonS3SSL
+		s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
+		if err != nil {
+			return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
 		}
+		bucket := utils.Cfg.FileSettings.AmazonS3Bucket
 
-		var auth aws.Auth
-		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
-		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
-
-		s := s3.New(auth, awsRegion())
-		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
-
-		if err := bucket.Del(oldPath); err != nil {
+		var copyConds = s3.NewCopyConditions()
+		if err = s3Clnt.CopyObject(bucket, newPath, "/"+path.Join(bucket, oldPath), copyConds); err != nil {
 			return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
 		}
-
-		if err := WriteFile(fileBytes, newPath); err != nil {
-			return err
+		if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil {
+			return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
 		}
 	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
-
 		if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil {
 			return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
 		}
@@ -824,26 +817,23 @@ func writeFileLocally(f []byte, path string) *model.AppError {
 
 func ReadFile(path string) ([]byte, *model.AppError) {
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
-		var auth aws.Auth
-		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
-		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
-
-		s := s3.New(auth, awsRegion())
-		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
-
-		// try to get the file from S3 with some basic retry logic
-		tries := 0
-		for {
-			tries++
-
-			f, err := bucket.Get(path)
-
-			if f != nil {
-				return f, nil
-			} else if tries >= 3 {
-				return nil, model.NewLocAppError("ReadFile", "api.file.read_file.get.app_error", nil, "path="+path+", err="+err.Error())
-			}
-			time.Sleep(3000 * time.Millisecond)
+		endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
+		accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
+		secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
+		secure := *utils.Cfg.FileSettings.AmazonS3SSL
+		s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
+		if err != nil {
+			return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
+		}
+		bucket := utils.Cfg.FileSettings.AmazonS3Bucket
+		reader, err := s3Clnt.GetObject(bucket, path)
+		if err != nil {
+			return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
+		}
+		if f, err := ioutil.ReadAll(reader); err != nil {
+			return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
+		} else {
+			return f, nil
 		}
 	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
 		if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil {
@@ -878,17 +868,3 @@ func openFileWriteStream(path string) (io.Writer, *model.AppError) {
 func closeFileWriteStream(file io.Writer) {
 	file.(*os.File).Close()
 }
-
-func awsRegion() aws.Region {
-	if region, ok := aws.Regions[utils.Cfg.FileSettings.AmazonS3Region]; ok {
-		return region
-	}
-
-	return aws.Region{
-		Name:                 utils.Cfg.FileSettings.AmazonS3Region,
-		S3Endpoint:           utils.Cfg.FileSettings.AmazonS3Endpoint,
-		S3BucketEndpoint:     utils.Cfg.FileSettings.AmazonS3BucketEndpoint,
-		S3LocationConstraint: *utils.Cfg.FileSettings.AmazonS3LocationConstraint,
-		S3LowercaseBucket:    *utils.Cfg.FileSettings.AmazonS3LowercaseBucket,
-	}
-}
diff --git a/api/file_test.go b/api/file_test.go
index ded866ab6..5d440f112 100644
--- a/api/file_test.go
+++ b/api/file_test.go
@@ -6,11 +6,6 @@ package api
 import (
 	"bytes"
 	"fmt"
-	"github.com/goamz/goamz/aws"
-	"github.com/goamz/goamz/s3"
-	"github.com/mattermost/platform/model"
-	"github.com/mattermost/platform/store"
-	"github.com/mattermost/platform/utils"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -18,6 +13,12 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/mattermost/platform/model"
+	"github.com/mattermost/platform/store"
+	"github.com/mattermost/platform/utils"
+
+	s3 "github.com/minio/minio-go"
 )
 
 func TestUploadFile(t *testing.T) {
@@ -830,25 +831,27 @@ func readTestFile(name string) ([]byte, error) {
 
 func cleanupTestFile(info *model.FileInfo) error {
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
-		var auth aws.Auth
-		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
-		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
-
-		s := s3.New(auth, aws.Regions[utils.Cfg.FileSettings.AmazonS3Region])
-		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
-
-		if err := bucket.Del(info.Path); err != nil {
+		endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
+		accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
+		secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
+		secure := *utils.Cfg.FileSettings.AmazonS3SSL
+		s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
+		if err != nil {
+			return err
+		}
+		bucket := utils.Cfg.FileSettings.AmazonS3Bucket
+		if err := s3Clnt.RemoveObject(bucket, info.Path); err != nil {
 			return err
 		}
 
 		if info.ThumbnailPath != "" {
-			if err := bucket.Del(info.ThumbnailPath); err != nil {
+			if err := s3Clnt.RemoveObject(bucket, info.ThumbnailPath); err != nil {
 				return err
 			}
 		}
 
 		if info.PreviewPath != "" {
-			if err := bucket.Del(info.PreviewPath); err != nil {
+			if err := s3Clnt.RemoveObject(bucket, info.PreviewPath); err != nil {
 				return err
 			}
 		}
diff --git a/api/user.go b/api/user.go
index 12e57a33f..37f8d6818 100644
--- a/api/user.go
+++ b/api/user.go
@@ -1238,7 +1238,7 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		} else {
-			path := "/users/" + id + "/profile.png"
+			path := "users/" + id + "/profile.png"
 
 			if data, err := ReadFile(path); err != nil {
 
diff --git a/api/user_test.go b/api/user_test.go
index 3b6fcb1fb..57f4729da 100644
--- a/api/user_test.go
+++ b/api/user_test.go
@@ -16,12 +16,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/goamz/goamz/aws"
-	"github.com/goamz/goamz/s3"
-
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/store"
 	"github.com/mattermost/platform/utils"
+
+	s3 "github.com/minio/minio-go"
 )
 
 func TestCreateUser(t *testing.T) {
@@ -674,14 +673,16 @@ func TestUserCreateImage(t *testing.T) {
 	Client.DoApiGet("/users/"+user.Id+"/image", "", "")
 
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
-		var auth aws.Auth
-		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
-		auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
-
-		s := s3.New(auth, aws.Regions[utils.Cfg.FileSettings.AmazonS3Region])
-		bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
-
-		if err := bucket.Del("/users/" + user.Id + "/profile.png"); err != nil {
+		endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
+		accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
+		secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
+		secure := *utils.Cfg.FileSettings.AmazonS3SSL
+		s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bucket := utils.Cfg.FileSettings.AmazonS3Bucket
+		if err = s3Clnt.RemoveObject(bucket, "/users/"+user.Id+"/profile.png"); err != nil {
 			t.Fatal(err)
 		}
 	} else {
@@ -774,14 +775,16 @@ func TestUserUploadProfileImage(t *testing.T) {
 		Client.DoApiGet("/users/"+user.Id+"/image", "", "")
 
 		if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
-			var auth aws.Auth
-			auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
-			auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
-
-			s := s3.New(auth, aws.Regions[utils.Cfg.FileSettings.AmazonS3Region])
-			bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
-
-			if err := bucket.Del("users/" + user.Id + "/profile.png"); err != nil {
+			endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
+			accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
+			secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
+			secure := *utils.Cfg.FileSettings.AmazonS3SSL
+			s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
+			if err != nil {
+				t.Fatal(err)
+			}
+			bucket := utils.Cfg.FileSettings.AmazonS3Bucket
+			if err = s3Clnt.RemoveObject(bucket, "/users/"+user.Id+"/profile.png"); err != nil {
 				t.Fatal(err)
 			}
 		} else {