diff options
Diffstat (limited to 'utils/file.go')
-rw-r--r-- | utils/file.go | 359 |
1 files changed, 0 insertions, 359 deletions
diff --git a/utils/file.go b/utils/file.go index 6472770a0..13b25bdab 100644 --- a/utils/file.go +++ b/utils/file.go @@ -4,372 +4,13 @@ package utils import ( - "bytes" "fmt" "io" "io/ioutil" - "net/http" "os" "path/filepath" - "strings" - - l4g "github.com/alecthomas/log4go" - s3 "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/credentials" - - "github.com/mattermost/mattermost-server/model" -) - -const ( - TEST_FILE_PATH = "/testfile" ) -// Similar to s3.New() but allows initialization of signature v2 or signature v4 client. -// If signV2 input is false, function always returns signature v4. -// -// Additionally this function also takes a user defined region, if set -// disables automatic region lookup. -func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, region string) (*s3.Client, error) { - var creds *credentials.Credentials - if signV2 { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV2) - } else { - creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4) - } - - s3Clnt, err := s3.NewWithCredentials(endpoint, creds, secure, region) - if err != nil { - return nil, err - } - - if *Cfg.FileSettings.AmazonS3Trace { - s3Clnt.TraceOn(os.Stdout) - } - - return s3Clnt, nil -} - -func TestFileConnection() *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - bucket := Cfg.FileSettings.AmazonS3Bucket - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("TestFileConnection", "Bad connection to S3 or minio.", nil, err.Error(), http.StatusInternalServerError) - } - - exists, err := s3Clnt.BucketExists(bucket) - if err != nil { - return model.NewAppError("TestFileConnection", "Error checking if bucket exists.", nil, err.Error(), http.StatusInternalServerError) - } - - if !exists { - l4g.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(bucket, region) - if err != nil { - l4g.Error("Unable to create bucket.") - return model.NewAppError("TestFileConnection", "Unable to create bucket", nil, err.Error(), http.StatusInternalServerError) - } - } - l4g.Info("Connection to S3 or minio is good. Bucket exists.") - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - f := []byte("testingwrite") - if err := writeFileLocally(f, Cfg.FileSettings.Directory+TEST_FILE_PATH); err != nil { - return model.NewAppError("TestFileConnection", "Don't have permissions to write to local path specified or other error.", nil, err.Error(), http.StatusInternalServerError) - } - os.Remove(Cfg.FileSettings.Directory + TEST_FILE_PATH) - l4g.Info("Able to write files to local storage.") - } else { - return model.NewAppError("TestFileConnection", "No file driver selected.", nil, "", http.StatusInternalServerError) - } - - return nil -} - -func ReadFile(path string) ([]byte, *model.AppError) { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - bucket := Cfg.FileSettings.AmazonS3Bucket - minioObject, err := s3Clnt.GetObject(bucket, path) - if err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - defer minioObject.Close() - if f, err := ioutil.ReadAll(minioObject); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if f, err := ioutil.ReadFile(Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - return f, nil - } - } else { - return nil, model.NewAppError("ReadFile", "api.file.read_file.configured.app_error", nil, "", http.StatusNotImplemented) - } -} - -func MoveFile(oldPath, newPath string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - encrypt := false - if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { - encrypt = true - } - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - bucket := Cfg.FileSettings.AmazonS3Bucket - - source := s3.NewSourceInfo(bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(bucket, newPath, nil, CopyMetadata(encrypt)) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.CopyObject(destination, source); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.MkdirAll(filepath.Dir(Cfg.FileSettings.Directory+newPath), 0774); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - if err := os.Rename(Cfg.FileSettings.Directory+oldPath, Cfg.FileSettings.Directory+newPath); err != nil { - return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("moveFile", "api.file.move_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func WriteFile(f []byte, path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - encrypt := false - if *Cfg.FileSettings.AmazonS3SSE && IsLicensed() && *License().Features.Compliance { - encrypt = true - } - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - bucket := Cfg.FileSettings.AmazonS3Bucket - ext := filepath.Ext(path) - metaData := S3Metadata(encrypt, "binary/octet-stream") - if model.IsFileExtImage(ext) { - metaData = S3Metadata(encrypt, model.GetImageMimeType(ext)) - } - - _, err = s3Clnt.PutObjectWithMetadata(bucket, path, bytes.NewReader(f), metaData, nil) - if err != nil { - return model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := writeFileLocally(f, Cfg.FileSettings.Directory+path); err != nil { - return err - } - } else { - return model.NewAppError("WriteFile", "api.file.write_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func writeFileLocally(f []byte, path string) *model.AppError { - if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { - directory, _ := filepath.Abs(filepath.Dir(path)) - return model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError) - } - - if err := ioutil.WriteFile(path, f, 0644); err != nil { - return model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - return nil -} - -func RemoveFile(path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - bucket := Cfg.FileSettings.AmazonS3Bucket - if err := s3Clnt.RemoveObject(bucket, path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.Remove(Cfg.FileSettings.Directory + path); err != nil { - return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("RemoveFile", "utils.file.remove_file.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { - out := make(chan string, 1) - - go func() { - defer close(out) - - for { - info, done := <-in - - if !done { - break - } - - out <- info.Key - } - }() - - return out -} - -// Returns a list of all the directories within the path directory provided. -func ListDirectory(path string) (*[]string, *model.AppError) { - var paths []string - - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - defer close(doneCh) - - bucket := Cfg.FileSettings.AmazonS3Bucket - for object := range s3Clnt.ListObjects(bucket, path, false, doneCh) { - if object.Err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) - } - paths = append(paths, strings.Trim(object.Key, "/")) - } - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if fileInfos, err := ioutil.ReadDir(Cfg.FileSettings.Directory + path); err != nil { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } else { - for _, fileInfo := range fileInfos { - if fileInfo.IsDir() { - paths = append(paths, filepath.Join(path, fileInfo.Name())) - } - } - } - } else { - return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.configured.app_error", nil, "", http.StatusInternalServerError) - } - - return &paths, nil -} - -func RemoveDirectory(path string) *model.AppError { - if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - endpoint := Cfg.FileSettings.AmazonS3Endpoint - accessKey := Cfg.FileSettings.AmazonS3AccessKeyId - secretKey := Cfg.FileSettings.AmazonS3SecretAccessKey - secure := *Cfg.FileSettings.AmazonS3SSL - signV2 := *Cfg.FileSettings.AmazonS3SignV2 - region := Cfg.FileSettings.AmazonS3Region - - s3Clnt, err := s3New(endpoint, accessKey, secretKey, secure, signV2, region) - if err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) - } - - doneCh := make(chan struct{}) - - bucket := Cfg.FileSettings.AmazonS3Bucket - for err := range s3Clnt.RemoveObjects(bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(bucket, path, true, doneCh))) { - if err.Err != nil { - doneCh <- struct{}{} - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) - } - } - - close(doneCh) - } else if *Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := os.RemoveAll(Cfg.FileSettings.Directory + path); err != nil { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError) - } - } else { - return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.configured.app_error", nil, "", http.StatusNotImplemented) - } - - return nil -} - -func S3Metadata(encrypt bool, contentType string) map[string][]string { - metaData := make(map[string][]string) - if contentType != "" { - metaData["Content-Type"] = []string{"contentType"} - } - if encrypt { - metaData["x-amz-server-side-encryption"] = []string{"AES256"} - } - return metaData -} - -func CopyMetadata(encrypt bool) map[string]string { - metaData := make(map[string]string) - metaData["x-amz-server-side-encryption"] = "AES256" - return metaData -} - // CopyFile will copy a file from src path to dst path. // Overwrites any existing files at dst. // Permissions are copied from file at src to the new file at dst. |