PLT-3105 Files table migration (#4068)

* Implemented initial changes for files table

* Removed *_benchmark_test.go files

* Re-implemented GetPublicFile and added support for old path

* Localization for files table

* Moved file system code into utils package

* Finished server-side changes and added initial upgrade script

* Added getPostFiles api

* Re-add Extension and HasPreviewImage fields to FileInfo

* Removed unused translation

* Fixed merge conflicts left over after permissions changes

* Forced FileInfo.extension to be lower case

* Changed FileUploadResponse to contain the FileInfos instead of FileIds

* Fixed permissions on getFile* calls

* Fixed notifications for file uploads

* Added initial version of client code for files changes

* Permanently added FileIds field to Post object and removed Post.HasFiles

* Updated PostStore.Update to be usable in more circumstances

* Re-added Filenames field and switched file migration to be entirely lazy-loaded

* Increased max listener count for FileStore

* Removed unused fileInfoCache

* Moved file system code back into api

* Removed duplicate test case

* Fixed unit test running on ports other than 8065

* Renamed HasPermissionToPostContext to HasPermissionToChannelByPostContext

* Refactored handleImages to make it more easily understandable

* Renamed getPostFiles to getFileInfosForPost

* Re-added pre-FileIds posts to analytics

* Changed files to be saved as their ids as opposed to id/filename.ext

* Renamed FileInfo.UserId to FileInfo.CreatorId

* Fixed detection of language in CodePreview

* Fixed switching between threads in the RHS not loading new files

* Add serverside protection against a rare bug where the client sends the same file twice for a single post

* Refactored the important parts of uploadFile api call into a function that can be called without a web context

1 files changed, 413 insertions, 236 deletions

diff --git a/api/file.go b/api/file.go
index dd99a8caf..9cf513ebf 100644
--- a/api/file.go
+++ b/api/file.go
@@ -16,6 +16,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -57,17 +58,19 @@ const (
 	MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image
 )
 
-var fileInfoCache *utils.Cache = utils.NewLru(1000)
-
 func InitFile() {
 	l4g.Debug(utils.T("api.file.init.debug"))
 
-	BaseRoutes.Files.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
-	BaseRoutes.Files.Handle("/get/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiUserRequiredTrustRequester(getFile)).Methods("GET")
-	BaseRoutes.Files.Handle("/get_info/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiUserRequired(getFileInfo)).Methods("GET")
-	BaseRoutes.Files.Handle("/get_public_link", ApiUserRequired(getPublicLink)).Methods("POST")
+	BaseRoutes.TeamFiles.Handle("/upload", ApiUserRequired(uploadFile)).Methods("POST")
+
+	BaseRoutes.NeedFile.Handle("/get", ApiUserRequiredTrustRequester(getFile)).Methods("GET")
+	BaseRoutes.NeedFile.Handle("/get_thumbnail", ApiUserRequiredTrustRequester(getFileThumbnail)).Methods("GET")
+	BaseRoutes.NeedFile.Handle("/get_preview", ApiUserRequiredTrustRequester(getFilePreview)).Methods("GET")
+	BaseRoutes.NeedFile.Handle("/get_info", ApiUserRequired(getFileInfo)).Methods("GET")
+	BaseRoutes.NeedFile.Handle("/get_public_link", ApiUserRequired(getPublicLink)).Methods("GET")
 
-	BaseRoutes.Public.Handle("/files/get/{team_id:[A-Za-z0-9]+}/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiAppHandlerTrustRequesterIndependent(getPublicFile)).Methods("GET")
+	BaseRoutes.Public.Handle("/files/{file_id:[A-Za-z0-9]+}/get", ApiAppHandlerTrustRequesterIndependent(getPublicFile)).Methods("GET")
+	BaseRoutes.Public.Handle("/files/get/{team_id:[A-Za-z0-9]+}/{channel_id:[A-Za-z0-9]+}/{user_id:[A-Za-z0-9]+}/{filename:([A-Za-z0-9]+/)?.+(\\.[A-Za-z0-9]{3,})?}", ApiAppHandlerTrustRequesterIndependent(getPublicFileOld)).Methods("GET")
 }
 
 func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -83,8 +86,7 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err := r.ParseMultipartForm(*utils.Cfg.FileSettings.MaxFileSize)
-	if err != nil {
+	if err := r.ParseMultipartForm(*utils.Cfg.FileSettings.MaxFileSize); err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -92,7 +94,6 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
 	m := r.MultipartForm
 
 	props := m.Value
-
 	if len(props["channel_id"]) == 0 {
 		c.SetInvalidParam("uploadFile", "channel_id")
 		return
@@ -103,91 +104,108 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	files := m.File["files"]
+	if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_UPLOAD_FILE) {
+		return
+	}
 
 	resStruct := &model.FileUploadResponse{
-		Filenames: []string{},
+		FileInfos: []*model.FileInfo{},
 		ClientIds: []string{},
 	}
 
-	imageNameList := []string{}
+	previewPathList := []string{}
+	thumbnailPathList := []string{}
 	imageDataList := [][]byte{}
 
-	if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_UPLOAD_FILE) {
-		return
-	}
-
-	for i := range files {
-		file, err := files[i].Open()
+	for i, fileHeader := range m.File["files"] {
+		file, fileErr := fileHeader.Open()
 		defer file.Close()
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+		if fileErr != nil {
+			http.Error(w, fileErr.Error(), http.StatusInternalServerError)
 			return
 		}
 
 		buf := bytes.NewBuffer(nil)
 		io.Copy(buf, file)
+		data := buf.Bytes()
 
-		filename := filepath.Base(files[i].Filename)
+		info, err := doUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
+		if err != nil {
+			c.Err = err
+			return
+		}
 
-		uid := model.NewId()
+		if info.PreviewPath != "" || info.ThumbnailPath != "" {
+			previewPathList = append(previewPathList, info.PreviewPath)
+			thumbnailPathList = append(thumbnailPathList, info.ThumbnailPath)
+			imageDataList = append(imageDataList, data)
+		}
 
-		if model.IsFileExtImage(filepath.Ext(files[i].Filename)) {
-			imageNameList = append(imageNameList, uid+"/"+filename)
-			imageDataList = append(imageDataList, buf.Bytes())
+		resStruct.FileInfos = append(resStruct.FileInfos, info)
 
-			// Decode image config first to check dimensions before loading the whole thing into memory later on
-			config, _, err := image.DecodeConfig(bytes.NewReader(buf.Bytes()))
-			if err != nil {
-				c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.image.app_error", nil, err.Error())
-				return
-			} else if config.Width*config.Height > MaxImageSize {
-				c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", nil, c.T("api.file.file_upload.exceeds"))
-				return
-			}
+		if len(m.Value["client_ids"]) > 0 {
+			resStruct.ClientIds = append(resStruct.ClientIds, m.Value["client_ids"][i])
 		}
+	}
 
-		path := "teams/" + c.TeamId + "/channels/" + channelId + "/users/" + c.Session.UserId + "/" + uid + "/" + filename
+	handleImages(previewPathList, thumbnailPathList, imageDataList)
 
-		if err := WriteFile(buf.Bytes(), path); err != nil {
-			c.Err = err
-			return
-		}
+	w.Write([]byte(resStruct.ToJson()))
+}
 
-		encName := utils.UrlEncode(filename)
+func doUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
+	filename := filepath.Base(rawFilename)
 
-		fileUrl := "/" + channelId + "/" + c.Session.UserId + "/" + uid + "/" + encName
-		resStruct.Filenames = append(resStruct.Filenames, fileUrl)
+	info, err := model.GetInfoForBytes(filename, data)
+	if err != nil {
+		err.StatusCode = http.StatusBadRequest
+		return nil, err
 	}
 
-	for _, clientId := range props["client_ids"] {
-		resStruct.ClientIds = append(resStruct.ClientIds, clientId)
+	info.Id = model.NewId()
+	info.CreatorId = userId
+
+	pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
+	info.Path = pathPrefix + filename
+
+	if info.IsImage() {
+		// Check dimensions before loading the whole thing into memory later on
+		if info.Width*info.Height > MaxImageSize {
+			err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", nil, "")
+			err.StatusCode = http.StatusBadRequest
+			return nil, err
+		}
+
+		nameWithoutExtension := filename[:strings.LastIndex(filename, ".")]
+		info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
+		info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
 	}
 
-	go handleImages(imageNameList, imageDataList, c.TeamId, channelId, c.Session.UserId)
+	if err := WriteFile(data, info.Path); err != nil {
+		return nil, err
+	}
 
-	w.Write([]byte(resStruct.ToJson()))
-}
+	if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil {
+		return nil, result.Err
+	}
 
-func handleImages(filenames []string, fileData [][]byte, teamId, channelId, userId string) {
-	dest := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/"
+	return info, nil
+}
 
-	for i, filename := range filenames {
-		name := filename[:strings.LastIndex(filename, ".")]
+func handleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
+	for i := range fileData {
 		go func() {
 			// Decode image bytes into Image object
 			img, imgType, err := image.Decode(bytes.NewReader(fileData[i]))
 			if err != nil {
-				l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), channelId, userId, filename, err)
+				l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err)
 				return
 			}
 
 			width := img.Bounds().Dx()
 			height := img.Bounds().Dy()
 
-			// Get the image's orientation and ignore any errors since not all images will have orientation data
-			orientation, _ := getImageOrientation(fileData[i])
-
+			// Fill in the background of a potentially-transparent png file as white
 			if imgType == "png" {
 				dst := image.NewRGBA(img.Bounds())
 				draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
@@ -195,6 +213,9 @@ func handleImages(filenames []string, fileData [][]byte, teamId, channelId, user
 				img = dst
 			}
 
+			// Flip the image to be upright
+			orientation, _ := getImageOrientation(fileData[i])
+
 			switch orientation {
 			case UprightMirrored:
 				img = imaging.FlipH(img)
@@ -212,57 +233,8 @@ func handleImages(filenames []string, fileData [][]byte, teamId, channelId, user
 				img = imaging.Rotate90(img)
 			}
 
-			// Create thumbnail
-			go func() {
-				thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
-				thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
-				imgWidth := float64(width)
-				imgHeight := float64(height)
-
-				var thumbnail image.Image
-				if imgHeight < thumbHeight && imgWidth < thumbWidth {
-					thumbnail = img
-				} else if imgHeight/imgWidth < thumbHeight/thumbWidth {
-					thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
-				} else {
-					thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
-				}
-
-				buf := new(bytes.Buffer)
-				err = jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90})
-				if err != nil {
-					l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), channelId, userId, filename, err)
-					return
-				}
-
-				if err := WriteFile(buf.Bytes(), dest+name+"_thumb.jpg"); err != nil {
-					l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), channelId, userId, filename, err)
-					return
-				}
-			}()
-
-			// Create preview
-			go func() {
-				var preview image.Image
-				if width > int(utils.Cfg.FileSettings.PreviewWidth) {
-					preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
-				} else {
-					preview = img
-				}
-
-				buf := new(bytes.Buffer)
-
-				err = jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90})
-				if err != nil {
-					l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), channelId, userId, filename, err)
-					return
-				}
-
-				if err := WriteFile(buf.Bytes(), dest+name+"_preview.jpg"); err != nil {
-					l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), channelId, userId, filename, err)
-					return
-				}
-			}()
+			go generateThumbnailImage(img, thumbnailPathList[i], width, height)
+			go generatePreviewImage(img, previewPathList[i], width)
 		}()
 	}
 }
@@ -284,105 +256,143 @@ func getImageOrientation(imageData []byte) (int, error) {
 	}
 }
 
-type ImageGetResult struct {
-	Error     error
-	ImageData []byte
-}
+func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
+	thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
+	thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
+	imgWidth := float64(width)
+	imgHeight := float64(height)
+
+	var thumbnail image.Image
+	if imgHeight < thumbHeight && imgWidth < thumbWidth {
+		thumbnail = img
+	} else if imgHeight/imgWidth < thumbHeight/thumbWidth {
+		thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
+	} else {
+		thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
+	}
 
-func getFileInfo(c *Context, w http.ResponseWriter, r *http.Request) {
-	if len(utils.Cfg.FileSettings.DriverName) == 0 {
-		c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.storage.app_error", nil, "")
-		c.Err.StatusCode = http.StatusNotImplemented
+	buf := new(bytes.Buffer)
+	if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err)
 		return
 	}
 
-	params := mux.Vars(r)
-
-	channelId := params["channel_id"]
-	if len(channelId) != 26 {
-		c.SetInvalidParam("getFileInfo", "channel_id")
+	if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
 		return
 	}
+}
 
-	userId := params["user_id"]
-	if len(userId) != 26 {
-		c.SetInvalidParam("getFileInfo", "user_id")
-		return
+func generatePreviewImage(img image.Image, previewPath string, width int) {
+	var preview image.Image
+	if width > int(utils.Cfg.FileSettings.PreviewWidth) {
+		preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
+	} else {
+		preview = img
 	}
 
-	filename := params["filename"]
-	if len(filename) == 0 {
-		c.SetInvalidParam("getFileInfo", "filename")
+	buf := new(bytes.Buffer)
+
+	if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err)
 		return
 	}
 
-	if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) {
+	if err := WriteFile(buf.Bytes(), previewPath); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
 		return
 	}
+}
 
-	path := "teams/" + c.TeamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
-	var info *model.FileInfo
+func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
+	info, err := getFileInfoForRequest(c, r, true)
+	if err != nil {
+		c.Err = err
+		return
+	}
 
-	if cached, ok := fileInfoCache.Get(path); ok {
-		info = cached.(*model.FileInfo)
-	} else {
-		fileData := make(chan []byte)
-		go readFile(path, fileData)
+	if data, err := ReadFile(info.Path); err != nil {
+		c.Err = err
+		c.Err.StatusCode = http.StatusNotFound
+	} else if err := writeFileResponse(info.Name, data, w, r); err != nil {
+		c.Err = err
+		return
+	}
+}
 
-		newInfo, err := model.GetInfoForBytes(filename, <-fileData)
-		if err != nil {
-			c.Err = err
-			return
-		} else {
-			fileInfoCache.Add(path, newInfo)
-			info = newInfo
-		}
+func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) {
+	info, err := getFileInfoForRequest(c, r, true)
+	if err != nil {
+		c.Err = err
+		return
 	}
 
-	w.Header().Set("Cache-Control", "max-age=2592000, public")
+	if info.ThumbnailPath == "" {
+		c.Err = model.NewLocAppError("getFileThumbnail", "api.file.get_file_thumbnail.no_thumbnail.app_error", nil, "file_id="+info.Id)
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
 
-	w.Write([]byte(info.ToJson()))
+	if data, err := ReadFile(info.ThumbnailPath); err != nil {
+		c.Err = err
+		c.Err.StatusCode = http.StatusNotFound
+	} else if err := writeFileResponse(info.Name, data, w, r); err != nil {
+		c.Err = err
+		return
+	}
 }
 
-func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
-	params := mux.Vars(r)
-
-	teamId := c.TeamId
-	channelId := params["channel_id"]
-	userId := params["user_id"]
-	filename := params["filename"]
+func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) {
+	info, err := getFileInfoForRequest(c, r, true)
+	if err != nil {
+		c.Err = err
+		return
+	}
 
-	if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_READ_CHANNEL) {
+	if info.PreviewPath == "" {
+		c.Err = model.NewLocAppError("getFilePreview", "api.file.get_file_preview.no_preview.app_error", nil, "file_id="+info.Id)
+		c.Err.StatusCode = http.StatusBadRequest
 		return
 	}
 
-	if err, bytes := getFileData(teamId, channelId, userId, filename); err != nil {
+	if data, err := ReadFile(info.PreviewPath); err != nil {
 		c.Err = err
-		return
-	} else if err := writeFileResponse(filename, bytes, w, r); err != nil {
+		c.Err.StatusCode = http.StatusNotFound
+	} else if err := writeFileResponse(info.Name, data, w, r); err != nil {
 		c.Err = err
 		return
 	}
 }
 
-func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
-	params := mux.Vars(r)
+func getFileInfo(c *Context, w http.ResponseWriter, r *http.Request) {
+	info, err := getFileInfoForRequest(c, r, true)
+	if err != nil {
+		c.Err = err
+		return
+	}
 
-	teamId := params["team_id"]
-	channelId := params["channel_id"]
-	userId := params["user_id"]
-	filename := params["filename"]
+	w.Header().Set("Cache-Control", "max-age=2592000, public")
 
-	hash := r.URL.Query().Get("h")
+	w.Write([]byte(info.ToJson()))
+}
 
+func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
 	if !utils.Cfg.FileSettings.EnablePublicLink {
 		c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_disabled.app_error", nil, "")
 		c.Err.StatusCode = http.StatusNotImplemented
 		return
 	}
 
+	info, err := getFileInfoForRequest(c, r, false)
+	if err != nil {
+		c.Err = err
+		return
+	}
+
+	hash := r.URL.Query().Get("h")
+
 	if len(hash) > 0 {
-		correctHash := generatePublicLinkHash(filename, *utils.Cfg.FileSettings.PublicLinkSalt)
+		correctHash := generatePublicLinkHash(info.Id, *utils.Cfg.FileSettings.PublicLinkSalt)
 
 		if hash != correctHash {
 			c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_invalid.app_error", nil, "")
@@ -395,49 +405,110 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err, bytes := getFileData(teamId, channelId, userId, filename); err != nil {
+	if data, err := ReadFile(info.Path); err != nil {
 		c.Err = err
-		return
-	} else if err := writeFileResponse(filename, bytes, w, r); err != nil {
+		c.Err.StatusCode = http.StatusNotFound
+	} else if err := writeFileResponse(info.Name, data, w, r); err != nil {
 		c.Err = err
 		return
 	}
 }
 
-func getFileData(teamId string, channelId string, userId string, filename string) (*model.AppError, []byte) {
+func getFileInfoForRequest(c *Context, r *http.Request, requireFileVisible bool) (*model.FileInfo, *model.AppError) {
 	if len(utils.Cfg.FileSettings.DriverName) == 0 {
-		err := model.NewLocAppError("getFileData", "api.file.upload_file.storage.app_error", nil, "")
+		err := model.NewLocAppError("getFileInfoForRequest", "api.file.get_file_info_for_request.storage.app_error", nil, "")
 		err.StatusCode = http.StatusNotImplemented
-		return err, nil
+		return nil, err
 	}
 
-	if len(teamId) != 26 {
-		return NewInvalidParamError("getFileData", "team_id"), nil
+	params := mux.Vars(r)
+
+	fileId := params["file_id"]
+	if len(fileId) != 26 {
+		return nil, NewInvalidParamError("getFileInfoForRequest", "file_id")
 	}
 
-	if len(channelId) != 26 {
-		return NewInvalidParamError("getFileData", "channel_id"), nil
+	var info *model.FileInfo
+	if result := <-Srv.Store.FileInfo().Get(fileId); result.Err != nil {
+		return nil, result.Err
+	} else {
+		info = result.Data.(*model.FileInfo)
 	}
 
-	if len(userId) != 26 {
-		return NewInvalidParamError("getFileData", "user_id"), nil
+	// only let users access files visible in a channel, unless they're the one who uploaded the file
+	if info.CreatorId != c.Session.UserId {
+		if len(info.PostId) == 0 {
+			err := model.NewLocAppError("getFileInfoForRequest", "api.file.get_file_info_for_request.no_post.app_error", nil, "file_id="+fileId)
+			err.StatusCode = http.StatusBadRequest
+			return nil, err
+		}
+
+		if requireFileVisible {
+			if !HasPermissionToChannelByPostContext(c, info.PostId, model.PERMISSION_READ_CHANNEL) {
+				return nil, c.Err
+			}
+		}
 	}
 
-	if len(filename) == 0 {
-		return NewInvalidParamError("getFileData", "filename"), nil
+	return info, nil
+}
+
+func getPublicFileOld(c *Context, w http.ResponseWriter, r *http.Request) {
+	if len(utils.Cfg.FileSettings.DriverName) == 0 {
+		c.Err = model.NewLocAppError("getPublicFile", "api.file.get_public_file_old.storage.app_error", nil, "")
+		c.Err.StatusCode = http.StatusNotImplemented
+		return
+	} else if !utils.Cfg.FileSettings.EnablePublicLink {
+		c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_disabled.app_error", nil, "")
+		c.Err.StatusCode = http.StatusNotImplemented
+		return
 	}
 
-	path := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
+	params := mux.Vars(r)
 
-	fileChan := make(chan []byte)
-	go readFile(path, fileChan)
+	teamId := params["team_id"]
+	channelId := params["channel_id"]
+	userId := params["user_id"]
+	filename := params["filename"]
 
-	if bytes := <-fileChan; bytes == nil {
-		err := model.NewLocAppError("writeFileResponse", "api.file.get_file.not_found.app_error", nil, "path="+path)
-		err.StatusCode = http.StatusNotFound
-		return err, nil
+	hash := r.URL.Query().Get("h")
+
+	if len(hash) > 0 {
+		correctHash := generatePublicLinkHash(filename, *utils.Cfg.FileSettings.PublicLinkSalt)
+
+		if hash != correctHash {
+			c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_invalid.app_error", nil, "")
+			c.Err.StatusCode = http.StatusBadRequest
+			return
+		}
 	} else {
-		return nil, bytes
+		c.Err = model.NewLocAppError("getPublicFile", "api.file.get_file.public_invalid.app_error", nil, "")
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
+
+	path := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + filename
+
+	var info *model.FileInfo
+	if result := <-Srv.Store.FileInfo().GetByPath(path); result.Err != nil {
+		c.Err = result.Err
+		return
+	} else {
+		info = result.Data.(*model.FileInfo)
+	}
+
+	if len(info.PostId) == 0 {
+		c.Err = model.NewLocAppError("getPublicFileOld", "api.file.get_public_file_old.no_post.app_error", nil, "file_id="+info.Id)
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
+
+	if data, err := ReadFile(info.Path); err != nil {
+		c.Err = err
+		c.Err.StatusCode = http.StatusNotFound
+	} else if err := writeFileResponse(info.Name, data, w, r); err != nil {
+		c.Err = err
+		return
 	}
 }
 
@@ -450,9 +521,7 @@ func writeFileResponse(filename string, bytes []byte, w http.ResponseWriter, r *
 	ua := user_agent.New(r.UserAgent())
 	bname, _ := ua.Browser()
 
-	parts := strings.Split(filename, "/")
-	filePart := strings.Split(parts[len(parts)-1], "?")[0]
-	w.Header().Set("Content-Disposition", "attachment;filename=\""+filePart+"\"")
+	w.Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"")
 
 	if bname == "Edge" || bname == "Internet Explorer" || bname == "Safari" {
 		w.Header().Set("Content-Type", "application/octet-stream")
@@ -467,71 +536,183 @@ func writeFileResponse(filename string, bytes []byte, w http.ResponseWriter, r *
 	return nil
 }
 
-func readFile(path string, fileData chan []byte) {
-	data, getErr := ReadFile(path)
-	if getErr != nil {
-		l4g.Error(getErr)
-		fileData <- nil
-	} else {
-		fileData <- data
-	}
-}
-
 func getPublicLink(c *Context, w http.ResponseWriter, r *http.Request) {
-	if len(utils.Cfg.FileSettings.DriverName) == 0 {
-		c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.storage.app_error", nil, "")
-		c.Err.StatusCode = http.StatusNotImplemented
-		return
-	}
-
 	if !utils.Cfg.FileSettings.EnablePublicLink {
 		c.Err = model.NewLocAppError("getPublicLink", "api.file.get_public_link.disabled.app_error", nil, "")
 		c.Err.StatusCode = http.StatusNotImplemented
 		return
 	}
 
-	props := model.MapFromJson(r.Body)
-
-	filename := props["filename"]
-	if len(filename) == 0 {
-		c.SetInvalidParam("getPublicLink", "filename")
+	info, err := getFileInfoForRequest(c, r, true)
+	if err != nil {
+		c.Err = err
 		return
 	}
 
-	matches := model.PartialUrlRegex.FindAllStringSubmatch(filename, -1)
-	if len(matches) == 0 || len(matches[0]) < 4 {
-		c.SetInvalidParam("getPublicLink", "filename")
+	if len(info.PostId) == 0 {
+		c.Err = model.NewLocAppError("getPublicLink", "api.file.get_public_link.no_post.app_error", nil, "file_id="+info.Id)
+		c.Err.StatusCode = http.StatusBadRequest
 		return
 	}
 
-	channelId := matches[0][1]
-	userId := matches[0][2]
-	filename = matches[0][3]
+	w.Write([]byte(model.StringToJson(generatePublicLink(c.GetSiteURL(), info))))
+}
 
-	if !HasPermissionToChannelContext(c, channelId, model.PERMISSION_GET_PUBLIC_LINK) {
-		return
+func generatePublicLink(siteURL string, info *model.FileInfo) string {
+	hash := generatePublicLinkHash(info.Id, *utils.Cfg.FileSettings.PublicLinkSalt)
+	return fmt.Sprintf("%s%s/public/files/%v/get?h=%s", siteURL, model.API_URL_SUFFIX, info.Id, hash)
+}
+
+func generatePublicLinkHash(fileId, salt string) string {
+	hash := sha256.New()
+	hash.Write([]byte(salt))
+	hash.Write([]byte(fileId))
+
+	return base64.RawURLEncoding.EncodeToString(hash.Sum(nil))
+}
+
+// Creates and stores FileInfos for a post created before the FileInfos table existed.
+func migrateFilenamesToFileInfos(post *model.Post) []*model.FileInfo {
+	if len(post.Filenames) == 0 {
+		l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.no_filenames.warn"), post.Id)
+		return []*model.FileInfo{}
+	}
+
+	cchan := Srv.Store.Channel().Get(post.ChannelId)
+
+	// There's a weird bug that rarely happens where a post ends up with duplicate Filenames so remove those
+	filenames := utils.RemoveDuplicatesFromStringArray(post.Filenames)
+
+	var channel *model.Channel
+	if result := <-cchan; result.Err != nil {
+		l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.channel.app_error"), post.Id, post.ChannelId, result.Err)
+		return []*model.FileInfo{}
+	} else {
+		channel = result.Data.(*model.Channel)
+	}
+
+	// Find the team that was used to make this post since its part of the file path that isn't saved in the Filename
+	var teamId string
+	if channel.TeamId == "" {
+		// This post was made in a cross-team DM channel so we need to find where its files were saved
+		teamId = findTeamIdForFilename(post, filenames[0])
+	} else {
+		teamId = channel.TeamId
+	}
+
+	// Create FileInfo objects for this post
+	infos := make([]*model.FileInfo, 0, len(filenames))
+	fileIds := make([]string, 0, len(filenames))
+	if teamId == "" {
+		l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.team_id.error"), post.Id, filenames)
+	} else {
+		for _, filename := range filenames {
+			info := getInfoForFilename(post, teamId, filename)
+			if info == nil {
+				continue
+			}
+
+			if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil {
+				l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.save_file_info.app_error"), post.Id, info.Id, filename, result.Err)
+				continue
+			}
+
+			fileIds = append(fileIds, info.Id)
+			infos = append(infos, info)
+		}
 	}
 
-	url := generatePublicLink(c.GetSiteURL(), c.TeamId, channelId, userId, filename)
+	// Copy and save the updated post
+	newPost := &model.Post{}
+	*newPost = *post
 
-	w.Write([]byte(model.StringToJson(url)))
+	newPost.Filenames = []string{}
+	newPost.FileIds = fileIds
+
+	// Update Posts to clear Filenames and set FileIds
+	if result := <-Srv.Store.Post().Update(newPost, post); result.Err != nil {
+		l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.save_post.app_error"), post.Id, newPost.FileIds, post.Filenames, result.Err)
+		return []*model.FileInfo{}
+	} else {
+		return infos
+	}
 }
 
-func generatePublicLink(siteURL, teamId, channelId, userId, filename string) string {
-	hash := generatePublicLinkHash(filename, *utils.Cfg.FileSettings.PublicLinkSalt)
-	return fmt.Sprintf("%s%s/public/files/get/%s/%s/%s/%s?h=%s", siteURL, model.API_URL_SUFFIX, teamId, channelId, userId, filename, hash)
+func findTeamIdForFilename(post *model.Post, filename string) string {
+	split := strings.SplitN(filename, "/", 5)
+	id := split[3]
+	name, _ := url.QueryUnescape(split[4])
+
+	// This post is in a direct channel so we need to figure out what team the files are stored under.
+	if result := <-Srv.Store.Team().GetTeamsByUserId(post.UserId); result.Err != nil {
+		l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.teams.app_error"), post.Id, result.Err)
+	} else if teams := result.Data.([]*model.Team); len(teams) == 1 {
+		// The user has only one team so the post must've been sent from it
+		return teams[0].Id
+	} else {
+		for _, team := range teams {
+			path := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/%s", team.Id, post.ChannelId, post.UserId, id, name)
+			if _, err := ReadFile(path); err == nil {
+				// Found the team that this file was posted from
+				return team.Id
+			}
+		}
+	}
+
+	return ""
 }
 
-func generatePublicLinkHash(filename, salt string) string {
-	hash := sha256.New()
-	hash.Write([]byte(salt))
-	hash.Write([]byte(filename))
+func getInfoForFilename(post *model.Post, teamId string, filename string) *model.FileInfo {
+	// Find the path from the Filename of the form /{channelId}/{userId}/{uid}/{nameWithExtension}
+	split := strings.SplitN(filename, "/", 5)
+	if len(split) < 5 {
+		l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.unexpected_filename.error"), post.Id, filename)
+		return nil
+	}
 
-	return base64.RawURLEncoding.EncodeToString(hash.Sum(nil))
+	channelId := split[1]
+	userId := split[2]
+	oldId := split[3]
+	name, _ := url.QueryUnescape(split[4])
+
+	if split[0] != "" || split[1] != post.ChannelId || split[2] != post.UserId || strings.Contains(split[4], "/") {
+		l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.mismatched_filename.warn"), post.Id, post.ChannelId, post.UserId, filename)
+	}
+
+	pathPrefix := fmt.Sprintf("teams/%s/channels/%s/users/%s/%s/", teamId, channelId, userId, oldId)
+	path := pathPrefix + name
+
+	// Open the file and populate the fields of the FileInfo
+	var info *model.FileInfo
+	if data, err := ReadFile(path); err != nil {
+		l4g.Error(utils.T("api.file.migrate_filenames_to_file_infos.file_not_found.error"), post.Id, filename, path, err)
+		return nil
+	} else {
+		var err *model.AppError
+		info, err = model.GetInfoForBytes(name, data)
+		if err != nil {
+			l4g.Warn(utils.T("api.file.migrate_filenames_to_file_infos.info.app_error"), post.Id, filename, err)
+		}
+	}
+
+	// Generate a new ID because with the old system, you could very rarely get multiple posts referencing the same file
+	info.Id = model.NewId()
+	info.CreatorId = post.UserId
+	info.PostId = post.Id
+	info.CreateAt = post.CreateAt
+	info.UpdateAt = post.UpdateAt
+	info.Path = path
+
+	if info.IsImage() {
+		nameWithoutExtension := name[:strings.LastIndex(name, ".")]
+		info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
+		info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
+	}
+
+	return info
 }
 
 func WriteFile(f []byte, path string) *model.AppError {
-
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
 		var auth aws.Auth
 		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
@@ -556,7 +737,7 @@ func WriteFile(f []byte, path string) *model.AppError {
 			return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
 		}
 	} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
-		if err := WriteFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil {
+		if err := writeFileLocally(f, utils.Cfg.FileSettings.Directory+path); err != nil {
 			return err
 		}
 	} else {
@@ -568,9 +749,7 @@ func WriteFile(f []byte, path string) *model.AppError {
 
 func MoveFile(oldPath, newPath string) *model.AppError {
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
-		fileData := make(chan []byte)
-		go readFile(oldPath, fileData)
-		fileBytes := <-fileData
+		fileBytes, _ := ReadFile(oldPath)
 
 		if fileBytes == nil {
 			return model.NewLocAppError("moveFile", "api.file.move_file.get_from_s3.app_error", nil, "")
@@ -606,7 +785,7 @@ func MoveFile(oldPath, newPath string) *model.AppError {
 	return nil
 }
 
-func WriteFileLocally(f []byte, path string) *model.AppError {
+func writeFileLocally(f []byte, path string) *model.AppError {
 	if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil {
 		directory, _ := filepath.Abs(filepath.Dir(path))
 		return model.NewLocAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error())
@@ -620,7 +799,6 @@ func WriteFileLocally(f []byte, path string) *model.AppError {
 }
 
 func ReadFile(path string) ([]byte, *model.AppError) {
-
 	if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
 		var auth aws.Auth
 		auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
@@ -668,7 +846,6 @@ func openFileWriteStream(path string) (io.Writer, *model.AppError) {
 			fileHandle.Chmod(0644)
 			return fileHandle, nil
 		}
-
 	}
 
 	return nil, model.NewLocAppError("openFileWriteStream", "api.file.open_file_write_stream.configured.app_error", nil, "")