From 8a0e649f989a824bb3bbfd1900a5b8e5383b47e1 Mon Sep 17 00:00:00 2001 From: Harrison Healey Date: Fri, 30 Sep 2016 11:06:30 -0400 Subject: PLT-3105 Files table migration (#4068) * Implemented initial changes for files table * Removed *_benchmark_test.go files * Re-implemented GetPublicFile and added support for old path * Localization for files table * Moved file system code into utils package * Finished server-side changes and added initial upgrade script * Added getPostFiles api * Re-add Extension and HasPreviewImage fields to FileInfo * Removed unused translation * Fixed merge conflicts left over after permissions changes * Forced FileInfo.extension to be lower case * Changed FileUploadResponse to contain the FileInfos instead of FileIds * Fixed permissions on getFile* calls * Fixed notifications for file uploads * Added initial version of client code for files changes * Permanently added FileIds field to Post object and removed Post.HasFiles * Updated PostStore.Update to be usable in more circumstances * Re-added Filenames field and switched file migration to be entirely lazy-loaded * Increased max listener count for FileStore * Removed unused fileInfoCache * Moved file system code back into api * Removed duplicate test case * Fixed unit test running on ports other than 8065 * Renamed HasPermissionToPostContext to HasPermissionToChannelByPostContext * Refactored handleImages to make it more easily understandable * Renamed getPostFiles to getFileInfosForPost * Re-added pre-FileIds posts to analytics * Changed files to be saved as their ids as opposed to id/filename.ext * Renamed FileInfo.UserId to FileInfo.CreatorId * Fixed detection of language in CodePreview * Fixed switching between threads in the RHS not loading new files * Add serverside protection against a rare bug where the client sends the same file twice for a single post * Refactored the important parts of uploadFile api call into a function that can be called without a web context --- api/file.go | 649 ++++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 413 insertions(+), 236 deletions(-) (limited to 'api/file.go') diff --git a/api/file.go b/api/file.go index dd99a8caf..9cf513ebf 100644 --- a/api/file.go +++ b/api/file.go @@ -16,6 +16,7 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "os" "path/filepath" "strconv" @@ -57,17 +58,19 @@ const ( MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image ) -var fileInfoCache *utils.Cache = utils.NewLru(1000) - func InitFile() { l4g.Debug(utils.T("api.file.init.debug")) - BaseRoutes.Files.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST") - BaseRoutes.Files.Handle("/get/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiUserRequiredTrustRequester(getFile)).Methods("GET") - BaseRoutes.Files.Handle("/get_info/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiUserRequired(getFileInfo)).Methods("GET") - BaseRoutes.Files.Handle("/get_public_link", ApiUserRequired(getPublicLink)).Methods("POST") + BaseRoutes.TeamFiles.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST") + + BaseRoutes.NeedFile.Handle("/get", ApiUserRequiredTrustRequester(getFile)).Methods("GET") + BaseRoutes.NeedFile.Handle("/get_thumbnail", ApiUserRequiredTrustRequester(getFileThumbnail)).Methods("GET") + BaseRoutes.NeedFile.Handle("/get_preview", ApiUserRequiredTrustRequester(getFilePreview)).Methods("GET") + BaseRoutes.NeedFile.Handle("/get_info", ApiUserRequired(getFileInfo)).Methods("GET") + BaseRoutes.NeedFile.Handle("/get_public_link", ApiUserRequired(getPublicLink)).Methods("GET") - BaseRoutes.Public.Handle("/files/get/{team_id:[A-Za-z0-9]+}/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiAppHandlerTrustRequesterIndependent(getPublicFile)).Methods("GET") + BaseRoutes.Public.Handle("/files/{file_id:[A-Za-z0-9]+}/get", ApiAppHandlerTrustRequesterIndependent(getPublicFile)).Methods("GET") + BaseRoutes.Public.Handle("/files/get/{team_id:[A-Za-z0-9]+}/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiAppHandlerTrustRequesterIndependent(getPublicFileOld)).Methods("GET") } func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { @@ -83,8 +86,7 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - err := r.ParseMultipartForm(*utils.Cfg.FileSettings.MaxFileSize) - if err != nil { + if err := r.ParseMultipartForm(*utils.Cfg.FileSettings.MaxFileSize); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -92,7 +94,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { m := r.MultipartForm props := m.Value - if len(props["channel_id"]) == 0 { c.SetInvalidParam("uploadFile", "channel_id") return @@ -103,91 +104,108 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - files := m.File["files"] + if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_UPLOAD_FILE) { + return + } resStruct := &model.FileUploadResponse{ - Filenames: []string{}, + FileInfos: []*model.FileInfo{}, ClientIds: []string{}, } - imageNameList := []string{} + previewPathList := []string{} + thumbnailPathList := []string{} imageDataList := [][]byte{} - if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_UPLOAD_FILE) { - return - } - - for i := range files { - file, err := files[i].Open() + for i, fileHeader := range m.File["files"] { + file, fileErr := fileHeader.Open() defer file.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if fileErr != nil { + http.Error(w, fileErr.Error(), http.StatusInternalServerError) return } buf := bytes.NewBuffer(nil) io.Copy(buf, file) + data := buf.Bytes() - filename := filepath.Base(files[i].Filename) + info, err := doUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data) + if err != nil { + c.Err = err + return + } - uid := model.NewId() + if info.PreviewPath != "" || info.ThumbnailPath != "" { + previewPathList = append(previewPathList, info.PreviewPath) + thumbnailPathList = append(thumbnailPathList, info.ThumbnailPath) + imageDataList = append(imageDataList, data) + } - if model.IsFileExtImage(filepath.Ext(files[i].Filename)) { - imageNameList = append(imageNameList, uid+"/"+filename) - imageDataList = append(imageDataList, buf.Bytes()) + resStruct.FileInfos = append(resStruct.FileInfos, info) - // Decode image config first to check dimensions before loading the whole thing into memory later on - config, _, err := image.DecodeConfig(bytes.NewReader(buf.Bytes())) - if err != nil { - c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.image.app_error", nil, err.Error()) - return - } else if config.Width*config.Height > MaxImageSize { - c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", nil, c.T("api.file.file_upload.exceeds")) - return - } + if len(m.Value["client_ids"]) > 0 { + resStruct.ClientIds = append(resStruct.ClientIds, m.Value["client_ids"][i]) } + } - path := "teams/" + c.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + filename + handleImages(previewPathList, thumbnailPathList, imageDataList) - if err := WriteFile(buf.Bytes(), path); err != nil { - c.Err = err - return - } + w.Write([]byte(resStruct.ToJson())) +} - encName := utils.UrlEncode(filename) +func doUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { + filename := filepath.Base(rawFilename) - fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName - resStruct.Filenames = append(resStruct.Filenames, fileUrl) + info, err := model.GetInfoForBytes(filename, data) + if err != nil { + err.StatusCode = http.StatusBadRequest + return nil, err } - for _, clientId := range props["client_ids"] { - resStruct.ClientIds = append(resStruct.ClientIds, clientId) + info.Id = model.NewId() + info.CreatorId = userId + + pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/" + info.Path = pathPrefix + filename + + if info.IsImage() { + // Check dimensions before loading the whole thing into memory later on + if info.Width*info.Height > MaxImageSize { + err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", nil, "") + err.StatusCode = http.StatusBadRequest + return nil, err + } + + nameWithoutExtension := filename[:strings.LastIndex(filename, ".")] + info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" + info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" } - go handleImages(imageNameList, imageDataList, c.TeamId, channelId, c.Session.UserId) + if err := WriteFile(data, info.Path); err != nil { + return nil, err + } - w.Write([]byte(resStruct.ToJson())) -} + if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil { + return nil, result.Err + } -func handleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) { - dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + return info, nil +} - for i, filename := range filenames { - name := filename[:strings.LastIndex(filename, ".")] +func handleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) { + for i := range fileData { go func() { // Decode image bytes into Image object img, imgType, err := image.Decode(bytes.NewReader(fileData[i])) if err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), channelId, userId, filename, err) + l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err) return } width := img.Bounds().Dx() height := img.Bounds().Dy() - // Get the image's orientation and ignore any errors since not all images will have orientation data - orientation, _ := getImageOrientation(fileData[i]) - + // Fill in the background of a potentially-transparent png file as white if imgType == "png" { dst := image.NewRGBA(img.Bounds()) draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src) @@ -195,6 +213,9 @@ func handleImages(filenames []string, fileData [][]byte, teamId, channelId, user img = dst } + // Flip the image to be upright + orientation, _ := getImageOrientation(fileData[i]) + switch orientation { case UprightMirrored: img = imaging.FlipH(img) @@ -212,57 +233,8 @@ func handleImages(filenames []string, fileData [][]byte, teamId, channelId, user img = imaging.Rotate90(img) } - // Create thumbnail - go func() { - thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth) - thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight) - imgWidth := float64(width) - imgHeight := float64(height) - - var thumbnail image.Image - if imgHeight < thumbHeight && imgWidth < thumbWidth { - thumbnail = img - } else if imgHeight/imgWidth < thumbHeight/thumbWidth { - thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos) - } else { - thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos) - } - - buf := new(bytes.Buffer) - err = jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}) - if err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), channelId, userId, filename, err) - return - } - - if err := WriteFile(buf.Bytes(), dest+name+"_thumb.jpg"); err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), channelId, userId, filename, err) - return - } - }() - - // Create preview - go func() { - var preview image.Image - if width > int(utils.Cfg.FileSettings.PreviewWidth) { - preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos) - } else { - preview = img - } - - buf := new(bytes.Buffer) - - err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}) - if err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), channelId, userId, filename, err) - return - } - - if err := WriteFile(buf.Bytes(), dest+name+"_preview.jpg"); err != nil { - l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), channelId, userId, filename, err) - return - } - }() + go generateThumbnailImage(img, thumbnailPathList[i], width, height) + go generatePreviewImage(img, previewPathList[i], width) }() } } @@ -284,105 +256,143 @@ func getImageOrientation(imageData []byte) (int, error) { } } -type ImageGetResult struct { - Error error - ImageData []byte -} +func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) { + thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth) + thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight) + imgWidth := float64(width) + imgHeight := float64(height) + + var thumbnail image.Image + if imgHeight < thumbHeight && imgWidth < thumbWidth { + thumbnail = img + } else if imgHeight/imgWidth < thumbHeight/thumbWidth { + thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos) + } else { + thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos) + } -func getFileInfo(c *Context, w http.ResponseWriter, r *http.Request) { - if len(utils.Cfg.FileSettings.DriverName) == 0 { - c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.storage.app_error", nil, "") - c.Err.StatusCode = http.StatusNotImplemented + buf := new(bytes.Buffer) + if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err) return } - params := mux.Vars(r) - - channelId := params["channel_id"] - if len(channelId) != 26 { - c.SetInvalidParam("getFileInfo", "channel_id") + if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err) return } +} - userId := params["user_id"] - if len(userId) != 26 { - c.SetInvalidParam("getFileInfo", "user_id") - return +func generatePreviewImage(img image.Image, previewPath string, width int) { + var preview image.Image + if width > int(utils.Cfg.FileSettings.PreviewWidth) { + preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos) + } else { + preview = img } - filename := params["filename"] - if len(filename) == 0 { - c.SetInvalidParam("getFileInfo", "filename") + buf := new(bytes.Buffer) + + if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err) return } - if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) { + if err := WriteFile(buf.Bytes(), previewPath); err != nil { + l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err) return } +} - path := "teams/" + c.TeamId + "/channels/" + channelId + "/users/" + userId + "/" + filename - var info *model.FileInfo +func getFile(c *Context, w http.ResponseWriter, r *http.Request) { + info, err := getFileInfoForRequest(c, r, true) + if err != nil { + c.Err = err + return + } - if cached, ok := fileInfoCache.Get(path); ok { - info = cached.(*model.FileInfo) - } else { - fileData := make(chan []byte) - go readFile(path, fileData) + if data, err := ReadFile(info.Path); err != nil { + c.Err = err + c.Err.StatusCode = http.StatusNotFound + } else if err := writeFileResponse(info.Name, data, w, r); err != nil { + c.Err = err + return + } +} - newInfo, err := model.GetInfoForBytes(filename, <-fileData) - if err != nil { - c.Err = err - return - } else { - fileInfoCache.Add(path, newInfo) - info = newInfo - } +func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) { + info, err := getFileInfoForRequest(c, r, true) + if err != nil { + c.Err = err + return } - w.Header().Set("Cache-Control", "max-age=2592000, public") + if info.ThumbnailPath == "" { + c.Err = model.NewLocAppError("getFileThumbnail", "api.file.get_file_thumbnail.no_thumbnail.app_error", nil, "file_id="+info.Id) + c.Err.StatusCode = http.StatusBadRequest + return + } - w.Write([]byte(info.ToJson())) + if data, err := ReadFile(info.ThumbnailPath); err != nil { + c.Err = err + c.Err.StatusCode = http.StatusNotFound + } else if err := writeFileResponse(info.Name, data, w, r); err != nil { + c.Err = err + return + } } -func getFile(c *Context, w http.ResponseWriter, r *http.Request) { - params := mux.Vars(r) - - teamId := c.TeamId - channelId := params["channel_id"] - userId := params["user_id"] - filename := params["filename"] +func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) { + info, err := getFileInfoForRequest(c, r, true) + if err != nil { + c.Err = err + return + } - if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) { + if info.PreviewPath == "" { + c.Err = model.NewLocAppError("getFilePreview", "api.file.get_file_preview.no_preview.app_error", nil, "file_id="+info.Id) + c.Err.StatusCode = http.StatusBadRequest return } - if err, bytes := getFileData(teamId, channelId, userId, filename); err != nil { + if data, err := ReadFile(info.PreviewPath); err != nil { c.Err = err - return - } else if err := writeFileResponse(filename, bytes, w, r); err != nil { + c.Err.StatusCode = http.StatusNotFound + } else if err := writeFileResponse(info.Name, data, w, r); err != nil { c.Err = err return } } -func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { - params := mux.Vars(r) +func getFileInfo(c *Context, w http.ResponseWriter, r *http.Request) { + info, err := getFileInfoForRequest(c, r, true) + if err != nil { + c.Err = err + return + } - teamId := params["team_id"] - channelId := params["channel_id"] - userId := params["user_id"] - filename := params["filename"] + w.Header().Set("Cache-Control", "max-age=2592000, public") - hash := r.URL.Query().Get("h") + w.Write([]byte(info.ToJson())) +} +func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { if !utils.Cfg.FileSettings.EnablePublicLink { c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_disabled.app_error", nil, "") c.Err.StatusCode = http.StatusNotImplemented return } + info, err := getFileInfoForRequest(c, r, false) + if err != nil { + c.Err = err + return + } + + hash := r.URL.Query().Get("h") + if len(hash) > 0 { - correctHash := generatePublicLinkHash(filename, *utils.Cfg.FileSettings.PublicLinkSalt) + correctHash := generatePublicLinkHash(info.Id, *utils.Cfg.FileSettings.PublicLinkSalt) if hash != correctHash { c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_invalid.app_error", nil, "") @@ -395,49 +405,110 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { return } - if err, bytes := getFileData(teamId, channelId, userId, filename); err != nil { + if data, err := ReadFile(info.Path); err != nil { c.Err = err - return - } else if err := writeFileResponse(filename, bytes, w, r); err != nil { + c.Err.StatusCode = http.StatusNotFound + } else if err := writeFileResponse(info.Name, data, w, r); err != nil { c.Err = err return } } -func getFileData(teamId string, channelId string, userId string, filename string) (*model.AppError, []byte) { +func getFileInfoForRequest(c *Context, r *http.Request, requireFileVisible bool) (*model.FileInfo, *model.AppError) { if len(utils.Cfg.FileSettings.DriverName) == 0 { - err := model.NewLocAppError("getFileData", "api.file.upload_file.storage.app_error", nil, "") + err := model.NewLocAppError("getFileInfoForRequest", "api.file.get_file_info_for_request.storage.app_error", nil, "") err.StatusCode = http.StatusNotImplemented - return err, nil + return nil, err } - if len(teamId) != 26 { - return NewInvalidParamError("getFileData", "team_id"), nil + params := mux.Vars(r) + + fileId := params["file_id"] + if len(fileId) != 26 { + return nil, NewInvalidParamError("getFileInfoForRequest", "file_id") } - if len(channelId) != 26 { - return NewInvalidParamError("getFileData", "channel_id"), nil + var info *model.FileInfo + if result := <-Srv.Store.FileInfo().Get(fileId); result.Err != nil { + return nil, result.Err + } else { + info = result.Data.(*model.FileInfo) } - if len(userId) != 26 { - return NewInvalidParamError("getFileData", "user_id"), nil + // only let users access files visible in a channel, unless they're the one who uploaded the file + if info.CreatorId != c.Session.UserId { + if len(info.PostId) == 0 { + err := model.NewLocAppError("getFileInfoForRequest", "api.file.get_file_info_for_request.no_post.app_error", nil, "file_id="+fileId) + err.StatusCode = http.StatusBadRequest + return nil, err + } + + if requireFileVisible { + if !HasPermissionToChannelByPostContext(c, info.PostId, model.PERMISSION_READ_CHANNEL) { + return nil, c.Err + } + } } - if len(filename) == 0 { - return NewInvalidParamError("getFileData", "filename"), nil + return info, nil +} + +func getPublicFileOld(c *Context, w http.ResponseWriter, r *http.Request) { + if len(utils.Cfg.FileSettings.DriverName) == 0 { + c.Err = model.NewLocAppError("getPublicFile", "api.file.get_public_file_old.storage.app_error", nil, "") + c.Err.StatusCode = http.StatusNotImplemented + return + } else if !utils.Cfg.FileSettings.EnablePublicLink { + c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_disabled.app_error", nil, "") + c.Err.StatusCode = http.StatusNotImplemented + return } - path := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename + params := mux.Vars(r) - fileChan := make(chan []byte) - go readFile(path, fileChan) + teamId := params["team_id"] + channelId := params["channel_id"] + userId := params["user_id"] + filename := params["filename"] - if bytes := <-fileChan; bytes == nil { - err := model.NewLocAppError("writeFileResponse", "api.file.get_file.not_found.app_error", nil, "path="+path) - err.StatusCode = http.StatusNotFound - return err, nil + hash := r.URL.Query().Get("h") + + if len(hash) > 0 { + correctHash := generatePublicLinkHash(filename, *utils.Cfg.FileSettings.PublicLinkSalt) + + if hash != correctHash { + c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_invalid.app_error", nil, "") + c.Err.StatusCode = http.StatusBadRequest + return + } } else { - return nil, bytes + c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_invalid.app_error", nil, "") + c.Err.StatusCode = http.StatusBadRequest + return + } + + path := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename + + var info *model.FileInfo + if result := <-Srv.Store.FileInfo().GetByPath(path); result.Err != nil { + c.Err = result.Err + return + } else { + info = result.Data.(*model.FileInfo) + } + + if len(info.PostId) == 0 { + c.Err = model.NewLocAppError("getPublicFileOld", "api.file.get_public_file_old.no_post.app_error", nil, "file_id="+info.Id) + c.Err.StatusCode = http.StatusBadRequest + return + } + + if data, err := ReadFile(info.Path); err != nil { + c.Err = err + c.Err.StatusCode = http.StatusNotFound + } else if err := writeFileResponse(info.Name, data, w, r); err != nil { + c.Err = err + return } } @@ -450,9 +521,7 @@ func writeFileResponse(filename string, bytes []byte, w http.ResponseWriter, r * ua := user_agent.New(r.UserAgent()) bname, _ := ua.Browser() - parts := strings.Split(filename, "/") - filePart := strings.Split(parts[len(parts)-1], "?")[0] - w.Header().Set("Content-Disposition", "attachment;filename=\""+filePart+"\"") + w.Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"") if bname == "Edge" || bname == "Internet Explorer" || bname == "Safari" { w.Header().Set("Content-Type", "application/octet-stream") @@ -467,71 +536,183 @@ func writeFileResponse(filename string, bytes []byte, w http.ResponseWriter, r * return nil } -func readFile(path string, fileData chan []byte) { - data, getErr := ReadFile(path) - if getErr != nil { - l4g.Error(getErr) - fileData <- nil - } else { - fileData <- data - } -} - func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) { - if len(utils.Cfg.FileSettings.DriverName) == 0 { - c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.storage.app_error", nil, "") - c.Err.StatusCode = http.StatusNotImplemented - return - } - if !utils.Cfg.FileSettings.EnablePublicLink { c.Err = model.NewLocAppError("getPublicLink", "api.file.get_public_link.disabled.app_error", nil, "") c.Err.StatusCode = http.StatusNotImplemented return } - props := model.MapFromJson(r.Body) - - filename := props["filename"] - if len(filename) == 0 { - c.SetInvalidParam("getPublicLink", "filename") + info, err := getFileInfoForRequest(c, r, true) + if err != nil { + c.Err = err return } - matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1) - if len(matches) == 0 || len(matches[0]) < 4 { - c.SetInvalidParam("getPublicLink", "filename") + if len(info.PostId) == 0 { + c.Err = model.NewLocAppError("getPublicLink", "api.file.get_public_link.no_post.app_error", nil, "file_id="+info.Id) + c.Err.StatusCode = http.StatusBadRequest return } - channelId := matches[0][1] - userId := matches[0][2] - filename = matches[0][3] + w.Write([]byte(model.StringToJson(generatePublicLink(c.GetSiteURL(), info)))) +} - if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_GET_PUBLIC_LINK) { - return +func generatePublicLink(siteURL string, info *model.FileInfo) string { + hash := generatePublicLinkHash(info.Id, *utils.Cfg.FileSettings.PublicLinkSalt) + return fmt.Sprintf("%s%s/public/files/%v/get?h=%s", siteURL, model.API_URL_SUFFIX, info.Id, hash) +} + +func generatePublicLinkHash(fileId, salt string) string { + hash := sha256.New() + hash.Write([]byte(salt)) + hash.Write([]byte(fileId)) + + return base64.RawURLEncoding.EncodeToString(hash.Sum(nil)) +} + +// Creates and stores FileInfos for a post created before the FileInfos table existed. +func migrateFilenamesToFileInfos(post *model.Post) []*model.FileInfo { + if len(post.Filenames) == 0 { + l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.no_filenames.warn"), post.Id) + return []*model.FileInfo{} + } + + cchan := Srv.Store.Channel().Get(post.ChannelId) + + // There's a weird bug that rarely happens where a post ends up with duplicate Filenames so remove those + filenames := utils.RemoveDuplicatesFromStringArray(post.Filenames) + + var channel *model.Channel + if result := <-cchan; result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.channel.app_error"), post.Id, post.ChannelId, result.Err) + return []*model.FileInfo{} + } else { + channel = result.Data.(*model.Channel) + } + + // Find the team that was used to make this post since its part of the file path that isn't saved in the Filename + var teamId string + if channel.TeamId == "" { + // This post was made in a cross-team DM channel so we need to find where its files were saved + teamId = findTeamIdForFilename(post, filenames[0]) + } else { + teamId = channel.TeamId + } + + // Create FileInfo objects for this post + infos := make([]*model.FileInfo, 0, len(filenames)) + fileIds := make([]string, 0, len(filenames)) + if teamId == "" { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.team_id.error"), post.Id, filenames) + } else { + for _, filename := range filenames { + info := getInfoForFilename(post, teamId, filename) + if info == nil { + continue + } + + if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.save_file_info.app_error"), post.Id, info.Id, filename, result.Err) + continue + } + + fileIds = append(fileIds, info.Id) + infos = append(infos, info) + } } - url := generatePublicLink(c.GetSiteURL(), c.TeamId, channelId, userId, filename) + // Copy and save the updated post + newPost := &model.Post{} + *newPost = *post - w.Write([]byte(model.StringToJson(url))) + newPost.Filenames = []string{} + newPost.FileIds = fileIds + + // Update Posts to clear Filenames and set FileIds + if result := <-Srv.Store.Post().Update(newPost, post); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.save_post.app_error"), post.Id, newPost.FileIds, post.Filenames, result.Err) + return []*model.FileInfo{} + } else { + return infos + } } -func generatePublicLink(siteURL, teamId, channelId, userId, filename string) string { - hash := generatePublicLinkHash(filename, *utils.Cfg.FileSettings.PublicLinkSalt) - return fmt.Sprintf("%s%s/public/files/get/%s/%s/%s/%s?h=%s", siteURL, model.API_URL_SUFFIX, teamId, channelId, userId, filename, hash) +func findTeamIdForFilename(post *model.Post, filename string) string { + split := strings.SplitN(filename, "/", 5) + id := split[3] + name, _ := url.QueryUnescape(split[4]) + + // This post is in a direct channel so we need to figure out what team the files are stored under. + if result := <-Srv.Store.Team().GetTeamsByUserId(post.UserId); result.Err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.teams.app_error"), post.Id, result.Err) + } else if teams := result.Data.([]*model.Team); len(teams) == 1 { + // The user has only one team so the post must've been sent from it + return teams[0].Id + } else { + for _, team := range teams { + path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name) + if _, err := ReadFile(path); err == nil { + // Found the team that this file was posted from + return team.Id + } + } + } + + return "" } -func generatePublicLinkHash(filename, salt string) string { - hash := sha256.New() - hash.Write([]byte(salt)) - hash.Write([]byte(filename)) +func getInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo { + // Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension} + split := strings.SplitN(filename, "/", 5) + if len(split) < 5 { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.unexpected_filename.error"), post.Id, filename) + return nil + } - return base64.RawURLEncoding.EncodeToString(hash.Sum(nil)) + channelId := split[1] + userId := split[2] + oldId := split[3] + name, _ := url.QueryUnescape(split[4]) + + if split[0] != "" || split[1] != post.ChannelId || split[2] != post.UserId || strings.Contains(split[4], "/") { + l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.mismatched_filename.warn"), post.Id, post.ChannelId, post.UserId, filename) + } + + pathPrefix := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/", teamId, channelId, userId, oldId) + path := pathPrefix + name + + // Open the file and populate the fields of the FileInfo + var info *model.FileInfo + if data, err := ReadFile(path); err != nil { + l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err) + return nil + } else { + var err *model.AppError + info, err = model.GetInfoForBytes(name, data) + if err != nil { + l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.info.app_error"), post.Id, filename, err) + } + } + + // Generate a new ID because with the old system, you could very rarely get multiple posts referencing the same file + info.Id = model.NewId() + info.CreatorId = post.UserId + info.PostId = post.Id + info.CreateAt = post.CreateAt + info.UpdateAt = post.UpdateAt + info.Path = path + + if info.IsImage() { + nameWithoutExtension := name[:strings.LastIndex(name, ".")] + info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg" + info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg" + } + + return info } func WriteFile(f []byte, path string) *model.AppError { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { var auth aws.Auth auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId @@ -556,7 +737,7 @@ func WriteFile(f []byte, path string) *model.AppError { return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error()) } } else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL { - if err := WriteFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil { + if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil { return err } } else { @@ -568,9 +749,7 @@ func WriteFile(f []byte, path string) *model.AppError { func MoveFile(oldPath, newPath string) *model.AppError { if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { - fileData := make(chan []byte) - go readFile(oldPath, fileData) - fileBytes := <-fileData + fileBytes, _ := ReadFile(oldPath) if fileBytes == nil { return model.NewLocAppError("moveFile", "api.file.move_file.get_from_s3.app_error", nil, "") @@ -606,7 +785,7 @@ func MoveFile(oldPath, newPath string) *model.AppError { return nil } -func WriteFileLocally(f []byte, path string) *model.AppError { +func writeFileLocally(f []byte, path string) *model.AppError { if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil { directory, _ := filepath.Abs(filepath.Dir(path)) return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error()) @@ -620,7 +799,6 @@ func WriteFileLocally(f []byte, path string) *model.AppError { } func ReadFile(path string) ([]byte, *model.AppError) { - if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 { var auth aws.Auth auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId @@ -668,7 +846,6 @@ func openFileWriteStream(path string) (io.Writer, *model.AppError) { fileHandle.Chmod(0644) return fileHandle, nil } - } return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.configured.app_error", nil, "") -- cgit v1.2.3-1-g7c22