From 50fc6e1e9e8d286fd6a406cef75e48a28f9427ad Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Fri, 25 Aug 2017 15:38:13 +0100 Subject: PLT-???? Prepare file upload infrastructure for Data Retention. (#7266) * Prepare file upload infrastructure for Data Retention. This commit prepares the file upload infrastructure for the data retention feature that is under construction. Changes are: * Move file management code to utils to allow access to it from jobs. * From now on, store all file uploads in a top level folder which is the date of the day on which they were uploaded. This commit is based on Harrison Healey's branch, but updated to work with the latest master. * Use NewAppError --- store/sql_file_info_store.go | 40 ++++++++++++++++++++++++++++++--------- store/sql_file_info_store_test.go | 38 +++++++++++++++++++++++++++++++++++++ store/store.go | 1 + 3 files changed, 70 insertions(+), 9 deletions(-) (limited to 'store') diff --git a/store/sql_file_info_store.go b/store/sql_file_info_store.go index 3fe62121c..4bf6c9c73 100644 --- a/store/sql_file_info_store.go +++ b/store/sql_file_info_store.go @@ -97,7 +97,7 @@ func (fs SqlFileInfoStore) Get(id string) StoreChannel { if err == sql.ErrNoRows { result.Err = model.NewAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error(), http.StatusNotFound) } else { - result.Err = model.NewLocAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.Get", "store.sql_file_info.get.app_error", nil, "id="+id+", "+err.Error(), http.StatusInternalServerError) } } else { result.Data = info @@ -127,7 +127,7 @@ func (fs SqlFileInfoStore) GetByPath(path string) StoreChannel { Path = :Path AND DeleteAt = 0 LIMIT 1`, map[string]interface{}{"Path": path}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.GetByPath", "store.sql_file_info.get_by_path.app_error", nil, "path="+path+", "+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.GetByPath", "store.sql_file_info.get_by_path.app_error", nil, "path="+path+", "+err.Error(), http.StatusInternalServerError) } else { result.Data = info } @@ -139,7 +139,7 @@ func (fs SqlFileInfoStore) GetByPath(path string) StoreChannel { return storeChannel } -func (s SqlFileInfoStore) InvalidateFileInfosForPostCache(postId string) { +func (fs SqlFileInfoStore) InvalidateFileInfosForPostCache(postId string) { fileInfoCache.Remove(postId) } @@ -190,8 +190,8 @@ func (fs SqlFileInfoStore) GetForPost(postId string, readFromMaster bool, allowF AND DeleteAt = 0 ORDER BY CreateAt`, map[string]interface{}{"PostId": postId}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.GetForPost", - "store.sql_file_info.get_for_post.app_error", nil, "post_id="+postId+", "+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.GetForPost", + "store.sql_file_info.get_for_post.app_error", nil, "post_id="+postId+", "+err.Error(), http.StatusInternalServerError) } else { if len(infos) > 0 { fileInfoCache.AddWithExpiresInSecs(postId, infos, FILE_INFO_CACHE_SEC) @@ -221,8 +221,8 @@ func (fs SqlFileInfoStore) AttachToPost(fileId, postId string) StoreChannel { WHERE Id = :Id AND PostId = ''`, map[string]interface{}{"PostId": postId, "Id": fileId}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.AttachToPost", - "store.sql_file_info.attach_to_post.app_error", nil, "post_id="+postId+", file_id="+fileId+", err="+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.AttachToPost", + "store.sql_file_info.attach_to_post.app_error", nil, "post_id="+postId+", file_id="+fileId+", err="+err.Error(), http.StatusInternalServerError) } storeChannel <- result @@ -245,8 +245,8 @@ func (fs SqlFileInfoStore) DeleteForPost(postId string) StoreChannel { DeleteAt = :DeleteAt WHERE PostId = :PostId`, map[string]interface{}{"DeleteAt": model.GetMillis(), "PostId": postId}); err != nil { - result.Err = model.NewLocAppError("SqlFileInfoStore.DeleteForPost", - "store.sql_file_info.delete_for_post.app_error", nil, "post_id="+postId+", err="+err.Error()) + result.Err = model.NewAppError("SqlFileInfoStore.DeleteForPost", + "store.sql_file_info.delete_for_post.app_error", nil, "post_id="+postId+", err="+err.Error(), http.StatusInternalServerError) } else { result.Data = postId } @@ -257,3 +257,25 @@ func (fs SqlFileInfoStore) DeleteForPost(postId string) StoreChannel { return storeChannel } + +func (fs SqlFileInfoStore) PermanentDelete(fileId string) StoreChannel { + storeChannel := make(StoreChannel, 1) + + go func() { + result := StoreResult{} + + if _, err := fs.GetMaster().Exec( + `DELETE FROM + FileInfo + WHERE + Id = :FileId`, map[string]interface{}{"FileId": fileId}); err != nil { + result.Err = model.NewAppError("SqlFileInfoStore.PermanentDelete", + "store.sql_file_info.permanent_delete.app_error", nil, "file_id="+fileId+", err="+err.Error(), http.StatusInternalServerError) + } + + storeChannel <- result + close(storeChannel) + }() + + return storeChannel +} diff --git a/store/sql_file_info_store_test.go b/store/sql_file_info_store_test.go index daec06269..fecd862c0 100644 --- a/store/sql_file_info_store_test.go +++ b/store/sql_file_info_store_test.go @@ -25,6 +25,9 @@ func TestFileInfoSaveGet(t *testing.T) { } else { info = returned } + defer func() { + <-store.FileInfo().PermanentDelete(info.Id) + }() if result := <-store.FileInfo().Get(info.Id); result.Err != nil { t.Fatal(result.Err) @@ -43,6 +46,9 @@ func TestFileInfoSaveGet(t *testing.T) { if result := <-store.FileInfo().Get(info2.Id); result.Err == nil { t.Fatal("shouldn't have gotten deleted file") } + defer func() { + <-store.FileInfo().PermanentDelete(info2.Id) + }() } func TestFileInfoSaveGetByPath(t *testing.T) { @@ -60,6 +66,9 @@ func TestFileInfoSaveGetByPath(t *testing.T) { } else { info = returned } + defer func() { + <-store.FileInfo().PermanentDelete(info.Id) + }() if result := <-store.FileInfo().GetByPath(info.Path); result.Err != nil { t.Fatal(result.Err) @@ -78,6 +87,9 @@ func TestFileInfoSaveGetByPath(t *testing.T) { if result := <-store.FileInfo().GetByPath(info2.Id); result.Err == nil { t.Fatal("shouldn't have gotten deleted file") } + defer func() { + <-store.FileInfo().PermanentDelete(info2.Id) + }() } func TestFileInfoGetForPost(t *testing.T) { @@ -112,6 +124,9 @@ func TestFileInfoGetForPost(t *testing.T) { for i, info := range infos { infos[i] = Must(store.FileInfo().Save(info)).(*model.FileInfo) + defer func(id string) { + <-store.FileInfo().PermanentDelete(id) + }(infos[i].Id) } if result := <-store.FileInfo().GetForPost(postId, true, false); result.Err != nil { @@ -143,6 +158,9 @@ func TestFileInfoAttachToPost(t *testing.T) { CreatorId: userId, Path: "file.txt", })).(*model.FileInfo) + defer func() { + <-store.FileInfo().PermanentDelete(info1.Id) + }() if len(info1.PostId) != 0 { t.Fatal("file shouldn't have a PostId") @@ -162,6 +180,9 @@ func TestFileInfoAttachToPost(t *testing.T) { CreatorId: userId, Path: "file.txt", })).(*model.FileInfo) + defer func() { + <-store.FileInfo().PermanentDelete(info2.Id) + }() if result := <-store.FileInfo().AttachToPost(info2.Id, postId); result.Err != nil { t.Fatal(result.Err) @@ -208,6 +229,9 @@ func TestFileInfoDeleteForPost(t *testing.T) { for i, info := range infos { infos[i] = Must(store.FileInfo().Save(info)).(*model.FileInfo) + defer func(id string) { + <-store.FileInfo().PermanentDelete(id) + }(infos[i].Id) } if result := <-store.FileInfo().DeleteForPost(postId); result.Err != nil { @@ -218,3 +242,17 @@ func TestFileInfoDeleteForPost(t *testing.T) { t.Fatal("shouldn't have returned any file infos") } } + +func TestFileInfoPermanentDelete(t *testing.T) { + Setup() + + info := Must(store.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Path: "file.txt", + })).(*model.FileInfo) + + if result := <-store.FileInfo().PermanentDelete(info.Id); result.Err != nil { + t.Fatal(result.Err) + } +} diff --git a/store/store.go b/store/store.go index e86b5f116..0aa4d5c41 100644 --- a/store/store.go +++ b/store/store.go @@ -386,6 +386,7 @@ type FileInfoStore interface { InvalidateFileInfosForPostCache(postId string) AttachToPost(fileId string, postId string) StoreChannel DeleteForPost(postId string) StoreChannel + PermanentDelete(fileId string) StoreChannel } type ReactionStore interface { -- cgit v1.2.3-1-g7c22