PLT-3105 Files table migration (#4068)

* Implemented initial changes for files table

* Removed *_benchmark_test.go files

* Re-implemented GetPublicFile and added support for old path

* Localization for files table

* Moved file system code into utils package

* Finished server-side changes and added initial upgrade script

* Added getPostFiles api

* Re-add Extension and HasPreviewImage fields to FileInfo

* Removed unused translation

* Fixed merge conflicts left over after permissions changes

* Forced FileInfo.extension to be lower case

* Changed FileUploadResponse to contain the FileInfos instead of FileIds

* Fixed permissions on getFile* calls

* Fixed notifications for file uploads

* Added initial version of client code for files changes

* Permanently added FileIds field to Post object and removed Post.HasFiles

* Updated PostStore.Update to be usable in more circumstances

* Re-added Filenames field and switched file migration to be entirely lazy-loaded

* Increased max listener count for FileStore

* Removed unused fileInfoCache

* Moved file system code back into api

* Removed duplicate test case

* Fixed unit test running on ports other than 8065

* Renamed HasPermissionToPostContext to HasPermissionToChannelByPostContext

* Refactored handleImages to make it more easily understandable

* Renamed getPostFiles to getFileInfosForPost

* Re-added pre-FileIds posts to analytics

* Changed files to be saved as their ids as opposed to id/filename.ext

* Renamed FileInfo.UserId to FileInfo.CreatorId

* Fixed detection of language in CodePreview

* Fixed switching between threads in the RHS not loading new files

* Add serverside protection against a rare bug where the client sends the same file twice for a single post

* Refactored the important parts of uploadFile api call into a function that can be called without a web context

1 files changed, 137 insertions, 28 deletions

diff --git a/model/file_info_test.go b/model/file_info_test.go
index 90256aed7..d3671f252 100644
--- a/model/file_info_test.go
+++ b/model/file_info_test.go
@@ -5,56 +5,137 @@ package model
 
 import (
 	"encoding/base64"
+	_ "image/gif"
+	_ "image/png"
 	"io/ioutil"
 	"strings"
 	"testing"
 )
 
-func TestGetInfoForBytes(t *testing.T) {
+func TestFileInfoIsValid(t *testing.T) {
+	info := &FileInfo{
+		Id:        NewId(),
+		CreatorId: NewId(),
+		CreateAt:  1234,
+		UpdateAt:  1234,
+		PostId:    "",
+		Path:      "fake/path.png",
+	}
+
+	if err := info.IsValid(); err != nil {
+		t.Fatal(err)
+	}
+
+	info.Id = ""
+	if err := info.IsValid(); err == nil {
+		t.Fatal("empty Id isn't valid")
+	}
+
+	info.Id = NewId()
+	info.CreateAt = 0
+	if err := info.IsValid(); err == nil {
+		t.Fatal("empty CreateAt isn't valid")
+	}
+
+	info.CreateAt = 1234
+	info.UpdateAt = 0
+	if err := info.IsValid(); err == nil {
+		t.Fatal("empty UpdateAt isn't valid")
+	}
+
+	info.UpdateAt = 1234
+	info.PostId = NewId()
+	if err := info.IsValid(); err != nil {
+		t.Fatal(err)
+	}
+
+	info.Path = ""
+	if err := info.IsValid(); err == nil {
+		t.Fatal("empty Path isn't valid")
+	}
+
+	info.Path = "fake/path.png"
+	if err := info.IsValid(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFileInfoIsImage(t *testing.T) {
+	info := &FileInfo{
+		MimeType: "image/png",
+	}
+
+	if !info.IsImage() {
+		t.Fatal("file is an image")
+	}
+
+	info.MimeType = "text/plain"
+	if info.IsImage() {
+		t.Fatal("file is not an image")
+	}
+}
+
+func TestGetInfoForFile(t *testing.T) {
 	fakeFile := make([]byte, 1000)
 
 	if info, err := GetInfoForBytes("file.txt", fakeFile); err != nil {
 		t.Fatal(err)
-	} else if info.Filename != "file.txt" {
-		t.Fatalf("Got incorrect filename: %v", info.Filename)
+	} else if info.Name != "file.txt" {
+		t.Fatalf("Got incorrect filename: %v", info.Name)
+	} else if info.Extension != "txt" {
+		t.Fatalf("Got incorrect extension: %v", info.Extension)
 	} else if info.Size != 1000 {
 		t.Fatalf("Got incorrect size: %v", info.Size)
-	} else if info.Extension != "txt" {
-		t.Fatalf("Got incorrect file extension: %v", info.Extension)
 	} else if !strings.HasPrefix(info.MimeType, "text/plain") {
 		t.Fatalf("Got incorrect mime type: %v", info.MimeType)
+	} else if info.Width != 0 {
+		t.Fatalf("Got incorrect width: %v", info.Width)
+	} else if info.Height != 0 {
+		t.Fatalf("Got incorrect height: %v", info.Height)
 	} else if info.HasPreviewImage {
-		t.Fatalf("Got HasPreviewImage = true for non-image file")
+		t.Fatalf("Got incorrect has preview image: %v", info.HasPreviewImage)
 	}
 
-	if info, err := GetInfoForBytes("file.png", fakeFile); err != nil {
+	pngFile, err := ioutil.ReadFile("../tests/test.png")
+	if err != nil {
+		t.Fatalf("Failed to load test.png: %v", err.Error())
+	}
+	if info, err := GetInfoForBytes("test.png", pngFile); err != nil {
 		t.Fatal(err)
-	} else if info.Filename != "file.png" {
-		t.Fatalf("Got incorrect filename: %v", info.Filename)
-	} else if info.Size != 1000 {
-		t.Fatalf("Got incorrect size: %v", info.Size)
+	} else if info.Name != "test.png" {
+		t.Fatalf("Got incorrect filename: %v", info.Name)
 	} else if info.Extension != "png" {
-		t.Fatalf("Got incorrect file extension: %v", info.Extension)
+		t.Fatalf("Got incorrect extension: %v", info.Extension)
+	} else if info.Size != 279591 {
+		t.Fatalf("Got incorrect size: %v", info.Size)
 	} else if info.MimeType != "image/png" {
 		t.Fatalf("Got incorrect mime type: %v", info.MimeType)
+	} else if info.Width != 408 {
+		t.Fatalf("Got incorrect width: %v", info.Width)
+	} else if info.Height != 336 {
+		t.Fatalf("Got incorrect height: %v", info.Height)
 	} else if !info.HasPreviewImage {
-		t.Fatalf("Got HasPreviewImage = false for image")
+		t.Fatalf("Got incorrect has preview image: %v", info.HasPreviewImage)
 	}
 
 	// base 64 encoded version of handtinywhite.gif from http://probablyprogramming.com/2009/03/15/the-tiniest-gif-ever
 	gifFile, _ := base64.StdEncoding.DecodeString("R0lGODlhAQABAIABAP///wAAACwAAAAAAQABAAACAkQBADs=")
 	if info, err := GetInfoForBytes("handtinywhite.gif", gifFile); err != nil {
 		t.Fatal(err)
-	} else if info.Filename != "handtinywhite.gif" {
-		t.Fatalf("Got incorrect filename: %v", info.Filename)
+	} else if info.Name != "handtinywhite.gif" {
+		t.Fatalf("Got incorrect filename: %v", info.Name)
+	} else if info.Extension != "gif" {
+		t.Fatalf("Got incorrect extension: %v", info.Extension)
 	} else if info.Size != 35 {
 		t.Fatalf("Got incorrect size: %v", info.Size)
-	} else if info.Extension != "gif" {
-		t.Fatalf("Got incorrect file extension: %v", info.Extension)
 	} else if info.MimeType != "image/gif" {
 		t.Fatalf("Got incorrect mime type: %v", info.MimeType)
+	} else if info.Width != 1 {
+		t.Fatalf("Got incorrect width: %v", info.Width)
+	} else if info.Height != 1 {
+		t.Fatalf("Got incorrect height: %v", info.Height)
 	} else if !info.HasPreviewImage {
-		t.Fatalf("Got HasPreviewImage = false for static gif")
+		t.Fatalf("Got incorrect has preview image: %v", info.HasPreviewImage)
 	}
 
 	animatedGifFile, err := ioutil.ReadFile("../tests/testgif.gif")
@@ -63,29 +144,57 @@ func TestGetInfoForBytes(t *testing.T) {
 	}
 	if info, err := GetInfoForBytes("testgif.gif", animatedGifFile); err != nil {
 		t.Fatal(err)
-	} else if info.Filename != "testgif.gif" {
-		t.Fatalf("Got incorrect filename: %v", info.Filename)
+	} else if info.Name != "testgif.gif" {
+		t.Fatalf("Got incorrect filename: %v", info.Name)
+	} else if info.Extension != "gif" {
+		t.Fatalf("Got incorrect extension: %v", info.Extension)
 	} else if info.Size != 38689 {
 		t.Fatalf("Got incorrect size: %v", info.Size)
-	} else if info.Extension != "gif" {
-		t.Fatalf("Got incorrect file extension: %v", info.Extension)
 	} else if info.MimeType != "image/gif" {
 		t.Fatalf("Got incorrect mime type: %v", info.MimeType)
+	} else if info.Width != 118 {
+		t.Fatalf("Got incorrect width: %v", info.Width)
+	} else if info.Height != 118 {
+		t.Fatalf("Got incorrect height: %v", info.Height)
 	} else if info.HasPreviewImage {
-		t.Fatalf("Got HasPreviewImage = true for animated gif")
+		t.Fatalf("Got incorrect has preview image: %v", info.HasPreviewImage)
 	}
 
 	if info, err := GetInfoForBytes("filewithoutextension", fakeFile); err != nil {
 		t.Fatal(err)
-	} else if info.Filename != "filewithoutextension" {
-		t.Fatalf("Got incorrect filename: %v", info.Filename)
+	} else if info.Name != "filewithoutextension" {
+		t.Fatalf("Got incorrect filename: %v", info.Name)
+	} else if info.Extension != "" {
+		t.Fatalf("Got incorrect extension: %v", info.Extension)
 	} else if info.Size != 1000 {
 		t.Fatalf("Got incorrect size: %v", info.Size)
-	} else if info.Extension != "" {
-		t.Fatalf("Got incorrect file extension: %v", info.Extension)
 	} else if info.MimeType != "" {
 		t.Fatalf("Got incorrect mime type: %v", info.MimeType)
+	} else if info.Width != 0 {
+		t.Fatalf("Got incorrect width: %v", info.Width)
+	} else if info.Height != 0 {
+		t.Fatalf("Got incorrect height: %v", info.Height)
 	} else if info.HasPreviewImage {
-		t.Fatalf("Got HasPreviewImage = true for non-image file")
+		t.Fatalf("Got incorrect has preview image: %v", info.HasPreviewImage)
+	}
+
+	// Always make the extension lower case to make it easier to use in other places
+	if info, err := GetInfoForBytes("file.TXT", fakeFile); err != nil {
+		t.Fatal(err)
+	} else if info.Name != "file.TXT" {
+		t.Fatalf("Got incorrect filename: %v", info.Name)
+	} else if info.Extension != "txt" {
+		t.Fatalf("Got incorrect extension: %v", info.Extension)
+	}
+
+	// Don't error out for image formats we don't support
+	if info, err := GetInfoForBytes("file.tif", fakeFile); err != nil {
+		t.Fatal(err)
+	} else if info.Name != "file.tif" {
+		t.Fatalf("Got incorrect filename: %v", info.Name)
+	} else if info.Extension != "tif" {
+		t.Fatalf("Got incorrect extension: %v", info.Extension)
+	} else if info.MimeType != "image/tiff" && info.MimeType != "image/x-tiff" {
+		t.Fatalf("Got incorrect mime type: %v", info.MimeType)
 	}
 }