From f5837c1b64994a15537a0b8df109bed504d0d20a Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Tue, 21 Jul 2015 16:15:24 -0400 Subject: Old files are saved with full paths, this changes so that new files are not saved with absolute paths and detects old files saved and fixes them. --- api/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index 362cdf896..5d676b9fd 100644 --- a/api/file.go +++ b/api/file.go @@ -115,7 +115,7 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - fileUrl := c.GetSiteURL() + "/api/v1/files/get/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename) + fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename) resStruct.Filenames = append(resStruct.Filenames, fileUrl) } -- cgit v1.2.3-1-g7c22 From ada84835eec1d69a962769afb590088d2f5a7d0a Mon Sep 17 00:00:00 2001 From: JoramWilander Date: Thu, 16 Jul 2015 08:54:09 -0400 Subject: initial implementation of local server storage for files --- api/file.go | 169 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 97 insertions(+), 72 deletions(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index 5d676b9fd..d6f9e6c1d 100644 --- a/api/file.go +++ b/api/file.go @@ -18,8 +18,10 @@ import ( _ "image/gif" "image/jpeg" "io" + "io/ioutil" "net/http" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -27,7 +29,7 @@ import ( ) func InitFile(r *mux.Router) { - l4g.Debug("Initializing post api routes") + l4g.Debug("Initializing file api routes") sr := r.PathPrefix("/files").Subrouter() sr.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST") @@ -36,8 +38,8 @@ func InitFile(r *mux.Router) { } func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("uploadFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -48,13 +50,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - m := r.MultipartForm props := m.Value @@ -94,25 +89,18 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) - ext := filepath.Ext(files[i].Filename) - uid := model.NewId() path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename - if model.IsFileExtImage(ext) { - options := s3.Options{} - err = bucket.Put(path, buf.Bytes(), model.GetImageMimeType(ext), s3.Private, options) - imageNameList = append(imageNameList, uid+"/"+files[i].Filename) - imageDataList = append(imageDataList, buf.Bytes()) - } else { - options := s3.Options{} - err = bucket.Put(path, buf.Bytes(), "binary/octet-stream", s3.Private, options) + if err := writeFile(buf.Bytes(), path); err != nil { + c.Err = err + return } - if err != nil { - c.Err = model.NewAppError("uploadFile", "Unable to upload file. ", err.Error()) - return + if model.IsFileExtImage(filepath.Ext(files[i].Filename)) { + imageNameList = append(imageNameList, uid+"/"+files[i].Filename) + imageDataList = append(imageDataList, buf.Bytes()) } fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename) @@ -127,13 +115,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) { go func() { - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" for i, filename := range filenames { @@ -169,9 +150,7 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch return } - // Upload thumbnail to S3 - options := s3.Options{} - err = bucket.Put(dest+name+"_thumb.jpg", buf.Bytes(), "image/jpeg", s3.Private, options) + err = writeFile(buf.Bytes(), dest+name+"_thumb.jpg") if err != nil { l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return @@ -188,17 +167,14 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch } buf := new(bytes.Buffer) - err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}) - //err = png.Encode(buf, preview) + err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}) if err != nil { l4g.Error("Unable to encode image as preview jpg channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } - // Upload preview to S3 - options := s3.Options{} - err = bucket.Put(dest+name+"_preview.jpg", buf.Bytes(), "image/jpeg", s3.Private, options) + err = writeFile(buf.Bytes(), dest+name+"_preview.jpg") if err != nil { l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return @@ -215,8 +191,8 @@ type ImageGetResult struct { } func getFile(c *Context, w http.ResponseWriter, r *http.Request) { - if !utils.IsS3Configured() { - c.Err = model.NewAppError("getFile", "Unable to get file. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("getFile", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -247,13 +223,6 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId) - var auth aws.Auth - auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId - auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey - - s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) - bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) - path := "" if len(teamId) == 26 { path = "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename @@ -262,7 +231,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { } fileData := make(chan []byte) - asyncGetFile(bucket, path, fileData) + asyncGetFile(path, fileData) if len(hash) > 0 && len(data) > 0 && len(teamId) == 26 { if !model.ComparePassword(hash, fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) { @@ -283,26 +252,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { f := <-fileData if f == nil { - var f2 []byte - tries := 0 - for { - time.Sleep(3000 * time.Millisecond) - tries++ - - asyncGetFile(bucket, path, fileData) - f2 = <-fileData - - if f2 != nil { - w.Header().Set("Cache-Control", "max-age=2592000, public") - w.Header().Set("Content-Length", strconv.Itoa(len(f2))) - w.Write(f2) - return - } else if tries >= 2 { - break - } - } - - c.Err = model.NewAppError("getFile", "Could not find file.", "url extenstion: "+path) + c.Err = model.NewAppError("getFile", "Could not find file.", "path="+path) c.Err.StatusCode = http.StatusNotFound return } @@ -312,9 +262,9 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { w.Write(f) } -func asyncGetFile(bucket *s3.Bucket, path string, fileData chan []byte) { +func asyncGetFile(path string, fileData chan []byte) { go func() { - data, getErr := bucket.Get(path) + data, getErr := readFile(path) if getErr != nil { fileData <- nil } else { @@ -329,8 +279,8 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { c.Err.StatusCode = http.StatusForbidden } - if !utils.IsS3Configured() { - c.Err = model.NewAppError("getPublicLink", "Unable to get link. Amazon S3 not configured. ", "") + if !utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + c.Err = model.NewAppError("getPublicLink", "Unable to upload file. Amazon S3 not configured and local server storage turned off. ", "") c.Err.StatusCode = http.StatusNotImplemented return } @@ -374,3 +324,78 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { w.Write([]byte(model.MapToJson(rData))) } + +func writeFile(f []byte, path string) *model.AppError { + + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + ext := filepath.Ext(path) + + var err error + if model.IsFileExtImage(ext) { + options := s3.Options{} + err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options) + + } else { + options := s3.Options{} + err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options) + } + + if err != nil { + return model.NewAppError("writeFile", "Encountered an error writing to S3", err.Error()) + } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + if err := os.MkdirAll(filepath.Dir(utils.Cfg.ServiceSettings.StorageDirectory+path), 0774); err != nil { + return model.NewAppError("writeFile", "Encountered an error creating the directory for the new file", err.Error()) + } + + if err := ioutil.WriteFile(utils.Cfg.ServiceSettings.StorageDirectory+path, f, 0644); err != nil { + return model.NewAppError("writeFile", "Encountered an error writing to local server storage", err.Error()) + } + } else { + return model.NewAppError("writeFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "") + } + + return nil +} + +func readFile(path string) ([]byte, *model.AppError) { + + if utils.IsS3Configured() && !utils.Cfg.ServiceSettings.UseLocalStorage { + var auth aws.Auth + auth.AccessKey = utils.Cfg.AWSSettings.S3AccessKeyId + auth.SecretKey = utils.Cfg.AWSSettings.S3SecretAccessKey + + s := s3.New(auth, aws.Regions[utils.Cfg.AWSSettings.S3Region]) + bucket := s.Bucket(utils.Cfg.AWSSettings.S3Bucket) + + // try to get the file from S3 with some basic retry logic + tries := 0 + for { + tries++ + + f, err := bucket.Get(path) + + if f != nil { + return f, nil + } else if tries >= 3 { + return nil, model.NewAppError("readFile", "Unable to get file from S3", "path="+path+", err="+err.Error()) + } + time.Sleep(3000 * time.Millisecond) + } + } else if utils.Cfg.ServiceSettings.UseLocalStorage && len(utils.Cfg.ServiceSettings.StorageDirectory) > 0 { + if f, err := ioutil.ReadFile(utils.Cfg.ServiceSettings.StorageDirectory + path); err != nil { + return nil, model.NewAppError("readFile", "Encountered an error reading from local server storage", err.Error()) + } else { + return f, nil + } + } else { + return nil, model.NewAppError("readFile", "File storage not configured properly. Please configure for either S3 or local server file storage.", "") + } +} -- cgit v1.2.3-1-g7c22 From a6fc129a01bf760aa163c8f842a3f2b67b375e3e Mon Sep 17 00:00:00 2001 From: JoramWilander Date: Thu, 16 Jul 2015 12:50:38 -0400 Subject: update file unit tests --- api/file.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index d6f9e6c1d..4fead4627 100644 --- a/api/file.go +++ b/api/file.go @@ -150,9 +150,8 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch return } - err = writeFile(buf.Bytes(), dest+name+"_thumb.jpg") - if err != nil { - l4g.Error("Unable to upload thumbnail to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) + if err := writeFile(buf.Bytes(), dest+name+"_thumb.jpg"); err != nil { + l4g.Error("Unable to upload thumbnail channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } }() @@ -174,9 +173,8 @@ func fireAndForgetHandleImages(filenames []string, fileData [][]byte, teamId, ch return } - err = writeFile(buf.Bytes(), dest+name+"_preview.jpg") - if err != nil { - l4g.Error("Unable to upload preview to S3 channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) + if err := writeFile(buf.Bytes(), dest+name+"_preview.jpg"); err != nil { + l4g.Error("Unable to upload preview channelId=%v userId=%v filename=%v err=%v", channelId, userId, filename, err) return } }() -- cgit v1.2.3-1-g7c22 From c63fbd4ccc5e7a11c4ce15fe7d19a3daf4e5c45e Mon Sep 17 00:00:00 2001 From: JoramWilander Date: Fri, 17 Jul 2015 15:55:06 -0400 Subject: add proper url encoding for filenames --- api/file.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index 4fead4627..2abaca709 100644 --- a/api/file.go +++ b/api/file.go @@ -103,7 +103,9 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { imageDataList = append(imageDataList, buf.Bytes()) } - fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + url.QueryEscape(files[i].Filename) + encName := utils.UrlEncode(files[i].Filename) + + fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName resStruct.Filenames = append(resStruct.Filenames, fileUrl) } @@ -264,6 +266,7 @@ func asyncGetFile(path string, fileData chan []byte) { go func() { data, getErr := readFile(path) if getErr != nil { + l4g.Error(getErr) fileData <- nil } else { fileData <- data -- cgit v1.2.3-1-g7c22 From 39abf24708870cec71a84c01063e647b859b2b67 Mon Sep 17 00:00:00 2001 From: JoramWilander Date: Tue, 21 Jul 2015 15:18:17 -0400 Subject: added sanitization to filenames to remove the possibility of relative paths --- api/file.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index 2abaca709..1dd179422 100644 --- a/api/file.go +++ b/api/file.go @@ -89,9 +89,11 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) + filename := filepath.Base(files[i].Filename) + uid := model.NewId() - path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + files[i].Filename + path := "teams/" + c.Session.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + filename if err := writeFile(buf.Bytes(), path); err != nil { c.Err = err @@ -99,11 +101,11 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { } if model.IsFileExtImage(filepath.Ext(files[i].Filename)) { - imageNameList = append(imageNameList, uid+"/"+files[i].Filename) + imageNameList = append(imageNameList, uid+"/"+filename) imageDataList = append(imageDataList, buf.Bytes()) } - encName := utils.UrlEncode(files[i].Filename) + encName := utils.UrlEncode(filename) fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName resStruct.Filenames = append(resStruct.Filenames, fileUrl) -- cgit v1.2.3-1-g7c22 From b821d23ed71c89b14aa294debcf390057de27b37 Mon Sep 17 00:00:00 2001 From: JoramWilander Date: Tue, 21 Jul 2015 19:21:05 -0400 Subject: fixed unit tests to work with team domain changes and update partial url regex for files --- api/file.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index 1dd179422..82cee9d1e 100644 --- a/api/file.go +++ b/api/file.go @@ -297,15 +297,14 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { } matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1) - if len(matches) == 0 || len(matches[0]) < 5 { + if len(matches) == 0 || len(matches[0]) < 4 { c.SetInvalidParam("getPublicLink", "filename") return } - getType := matches[0][1] - channelId := matches[0][2] - userId := matches[0][3] - filename = matches[0][4] + channelId := matches[0][1] + userId := matches[0][2] + filename = matches[0][3] cchan := Srv.Store.Channel().CheckPermissionsTo(c.Session.TeamId, channelId, c.Session.UserId) @@ -316,7 +315,7 @@ func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { data := model.MapToJson(newProps) hash := model.HashPassword(fmt.Sprintf("%v:%v", data, utils.Cfg.ServiceSettings.PublicLinkSalt)) - url := fmt.Sprintf("%s/api/v1/files/%s/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), getType, channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId) + url := fmt.Sprintf("%s/api/v1/files/get/%s/%s/%s?d=%s&h=%s&t=%s", c.GetSiteURL(), channelId, userId, filename, url.QueryEscape(data), url.QueryEscape(hash), c.Session.TeamId) if !c.HasPermissionsToChannel(cchan, "getPublicLink") { return -- cgit v1.2.3-1-g7c22