From ee1d037ca6893ae57061a1eba25f624d7c163baa Mon Sep 17 00:00:00 2001 From: Pradeep Murugesan Date: Fri, 20 Jul 2018 10:49:49 +0200 Subject: Support attachments in post and replies - Bulk import (#9124) * 9006 - process the attachments of the post * 9006 enabling the import of attachments in the reply post * 9006 assert if the post and files are linked * 9006 fixed the typo --- app/import.go | 86 +++++++++++++++++++++--- app/import_test.go | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 265 insertions(+), 9 deletions(-) (limited to 'app') diff --git a/app/import.go b/app/import.go index 37bef01e7..dc4b65396 100644 --- a/app/import.go +++ b/app/import.go @@ -20,6 +20,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" + "github.com/mattermost/mattermost-server/utils" ) // Import Data Models @@ -132,8 +133,9 @@ type ReplyImportData struct { Message *string `json:"message"` CreateAt *int64 `json:"create_at"` - FlaggedBy *[]string `json:"flagged_by"` - Reactions *[]ReactionImportData `json:"reactions"` + FlaggedBy *[]string `json:"flagged_by"` + Reactions *[]ReactionImportData `json:"reactions"` + Attachments *[]AttachmentImportData `json:"attachments"` } type PostImportData struct { @@ -144,9 +146,10 @@ type PostImportData struct { Message *string `json:"message"` CreateAt *int64 `json:"create_at"` - FlaggedBy *[]string `json:"flagged_by"` - Reactions *[]ReactionImportData `json:"reactions"` - Replies *[]ReplyImportData `json:"replies"` + FlaggedBy *[]string `json:"flagged_by"` + Reactions *[]ReactionImportData `json:"reactions"` + Replies *[]ReplyImportData `json:"replies"` + Attachments *[]AttachmentImportData `json:"attachments"` } type DirectChannelImportData struct { @@ -196,6 +199,10 @@ type LineImportWorkerError struct { LineNumber int } +type AttachmentImportData struct { + Path *string `json:"path"` +} + // // -- Bulk Import Functions -- // These functions import data directly into the database. Security and permission checks are bypassed but validity is @@ -1391,7 +1398,7 @@ func (a *App) ImportReaction(data *ReactionImportData, post *model.Post, dryRun return nil } -func (a *App) ImportReply(data *ReplyImportData, post *model.Post, dryRun bool) *model.AppError { +func (a *App) ImportReply(data *ReplyImportData, post *model.Post, teamId string, dryRun bool) *model.AppError { if err := validateReplyImportData(data, post.CreateAt, a.MaxPostSize()); err != nil { return err } @@ -1429,6 +1436,14 @@ func (a *App) ImportReply(data *ReplyImportData, post *model.Post, dryRun bool) reply.Message = *data.Message reply.CreateAt = *data.CreateAt + if data.Attachments != nil { + fileIds, err := a.uploadAttachments(data.Attachments, reply, teamId, dryRun) + if err != nil { + return err + } + reply.FileIds = fileIds + } + if reply.Id == "" { if result := <-a.Srv.Store.Post().Save(reply); result.Err != nil { return result.Err @@ -1438,9 +1453,36 @@ func (a *App) ImportReply(data *ReplyImportData, post *model.Post, dryRun bool) return result.Err } } + + a.UpdateFileInfoWithPostId(reply) + return nil } +func (a *App) ImportAttachment(data *AttachmentImportData, post *model.Post, teamId string, dryRun bool) (*model.FileInfo, *model.AppError) { + fileUploadError := model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest) + file, err := os.Open(*data.Path) + if err != nil { + return nil, model.NewAppError("BulkImport", "app.import.attachment.bad_file.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest) + } + if file != nil { + timestamp := utils.TimeFromMillis(post.CreateAt) + buf := bytes.NewBuffer(nil) + io.Copy(buf, file) + + fileInfo, err := a.DoUploadFile(timestamp, teamId, post.ChannelId, post.UserId, file.Name(), buf.Bytes()) + + if err != nil { + fmt.Print(err) + return nil, fileUploadError + } + + mlog.Info(fmt.Sprintf("uploading file with name %s", file.Name())) + return fileInfo, nil + } + return nil, fileUploadError +} + func (a *App) ImportPost(data *PostImportData, dryRun bool) *model.AppError { if err := validatePostImportData(data, a.MaxPostSize()); err != nil { return err @@ -1499,6 +1541,14 @@ func (a *App) ImportPost(data *PostImportData, dryRun bool) *model.AppError { post.Hashtags, _ = model.ParseHashtags(post.Message) + if data.Attachments != nil { + fileIds, err := a.uploadAttachments(data.Attachments, post, team.Id, dryRun) + if err != nil { + return err + } + post.FileIds = fileIds + } + if post.Id == "" { if result := <-a.Srv.Store.Post().Save(post); result.Err != nil { return result.Err @@ -1546,15 +1596,35 @@ func (a *App) ImportPost(data *PostImportData, dryRun bool) *model.AppError { if data.Replies != nil { for _, reply := range *data.Replies { - if err := a.ImportReply(&reply, post, dryRun); err != nil { + if err := a.ImportReply(&reply, post, team.Id, dryRun); err != nil { return err } } } + a.UpdateFileInfoWithPostId(post) return nil } +func (a *App) uploadAttachments(attachments *[]AttachmentImportData, post *model.Post, teamId string, dryRun bool) ([]string, *model.AppError) { + fileIds := []string{} + for _, attachment := range *attachments { + fileInfo, err := a.ImportAttachment(&attachment, post, teamId, dryRun) + if err != nil { + return nil, err + } + fileIds = append(fileIds, fileInfo.Id) + } + return fileIds, nil +} + +func (a *App) UpdateFileInfoWithPostId(post *model.Post) { + for _, fileId := range post.FileIds { + if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil { + mlog.Error(fmt.Sprintf("Error attaching files to post. postId=%v, fileIds=%v, message=%v", post.Id, post.FileIds, result.Err), mlog.String("post_id", post.Id)) + } + } +} func validateReactionImportData(data *ReactionImportData, parentCreateAt int64) *model.AppError { if data.User == nil { return model.NewAppError("BulkImport", "app.import.validate_reaction_import_data.user_missing.error", nil, "", http.StatusBadRequest) @@ -1869,7 +1939,7 @@ func (a *App) ImportDirectPost(data *DirectPostImportData, dryRun bool) *model.A if data.Replies != nil { for _, reply := range *data.Replies { - if err := a.ImportReply(&reply, post, dryRun); err != nil { + if err := a.ImportReply(&reply, post, "noteam", dryRun); err != nil { return err } } diff --git a/app/import_test.go b/app/import_test.go index 8a88937f9..f99e100f1 100644 --- a/app/import_test.go +++ b/app/import_test.go @@ -3792,7 +3792,7 @@ func TestImportBulkImport(t *testing.T) { {"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}} {"type": "user", "user": {"username": "` + username2 + `", "email": "` + username2 + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}} {"type": "user", "user": {"username": "` + username3 + `", "email": "` + username3 + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}} -{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}} +{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012, "attachments":[{"path": "` + testImage + `"}]}} {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username2 + `"]}} {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username2 + `", "` + username3 + `"]}} {"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username2 + `"], "user": "` + username + `", "message": "Hello Direct Channel", "create_at": 123456789013}} @@ -3911,3 +3911,189 @@ func TestImportImportEmoji(t *testing.T) { err = th.App.ImportEmoji(&data, false) assert.Nil(t, err, "Second run should have succeeded apply mode") } + +func TestImportAttachment(t *testing.T) { + th := Setup() + defer th.TearDown() + + testsDir, _ := utils.FindDir("tests") + testImage := filepath.Join(testsDir, "test.png") + invalidPath := "some-invalid-path" + + userId := model.NewId() + data := AttachmentImportData{Path: &testImage} + _, err := th.App.ImportAttachment(&data, &model.Post{UserId: userId, ChannelId: "some-channel"}, "some-team", true) + assert.Nil(t, err, "sample run without errors") + + attachments := GetAttachments(userId, th, t) + assert.Equal(t, len(attachments), 1) + + data = AttachmentImportData{Path: &invalidPath} + _, err = th.App.ImportAttachment(&data, &model.Post{UserId: model.NewId(), ChannelId: "some-channel"}, "some-team", true) + assert.NotNil(t, err, "should have failed when opening the file") + assert.Equal(t, err.Id, "app.import.attachment.bad_file.error") +} + +func TestImportPostAndRepliesWithAttachments(t *testing.T) { + + th := Setup() + defer th.TearDown() + + // Create a Team. + teamName := model.NewId() + th.App.ImportTeam(&TeamImportData{ + Name: &teamName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + team, err := th.App.GetTeamByName(teamName) + if err != nil { + t.Fatalf("Failed to get team from database.") + } + + // Create a Channel. + channelName := model.NewId() + th.App.ImportChannel(&ChannelImportData{ + Team: &teamName, + Name: &channelName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + _, err = th.App.GetChannelByName(channelName, team.Id) + if err != nil { + t.Fatalf("Failed to get channel from database.") + } + + // Create a user3. + username := model.NewId() + th.App.ImportUser(&UserImportData{ + Username: &username, + Email: ptrStr(model.NewId() + "@example.com"), + }, false) + user3, err := th.App.GetUserByUsername(username) + if err != nil { + t.Fatalf("Failed to get user3 from database.") + } + + username2 := model.NewId() + th.App.ImportUser(&UserImportData{ + Username: &username2, + Email: ptrStr(model.NewId() + "@example.com"), + }, false) + user4, err := th.App.GetUserByUsername(username2) + if err != nil { + t.Fatalf("Failed to get user3 from database.") + } + + // Post with attachments. + time := model.GetMillis() + attachmentsPostTime := time + attachmentsReplyTime := time + 1 + testsDir, _ := utils.FindDir("tests") + testImage := filepath.Join(testsDir, "test.png") + testMarkDown := filepath.Join(testsDir, "test-attachments.md") + data := &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message with reply"), + CreateAt: &attachmentsPostTime, + Attachments: &[]AttachmentImportData{{Path: &testImage}, {Path: &testMarkDown}}, + Replies: &[]ReplyImportData{{ + User: &user4.Username, + Message: ptrStr("Message reply"), + CreateAt: &attachmentsReplyTime, + Attachments: &[]AttachmentImportData{{Path: &testImage}}, + }}, + } + + if err := th.App.ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + + attachments := GetAttachments(user3.Id, th, t) + assert.Equal(t, len(attachments), 2) + assert.Contains(t, attachments[0].Path, team.Id) + assert.Contains(t, attachments[1].Path, team.Id) + AssertFileIdsInPost(attachments, th, t) + + attachments = GetAttachments(user4.Id, th, t) + assert.Equal(t, len(attachments), 1) + assert.Contains(t, attachments[0].Path, team.Id) + AssertFileIdsInPost(attachments, th, t) + + // Reply with Attachments in Direct Post + + // Create direct post users. + + username3 := model.NewId() + th.App.ImportUser(&UserImportData{ + Username: &username3, + Email: ptrStr(model.NewId() + "@example.com"), + }, false) + user3, err = th.App.GetUserByUsername(username3) + if err != nil { + t.Fatalf("Failed to get user3 from database.") + } + + username4 := model.NewId() + th.App.ImportUser(&UserImportData{ + Username: &username4, + Email: ptrStr(model.NewId() + "@example.com"), + }, false) + + user4, err = th.App.GetUserByUsername(username4) + if err != nil { + t.Fatalf("Failed to get user3 from database.") + } + + directImportData := &DirectPostImportData{ + ChannelMembers: &[]string{ + user3.Username, + user4.Username, + }, + User: &user3.Username, + Message: ptrStr("Message with Replies"), + CreateAt: ptrInt64(model.GetMillis()), + Replies: &[]ReplyImportData{{ + User: &user4.Username, + Message: ptrStr("Message reply with attachment"), + CreateAt: ptrInt64(model.GetMillis()), + Attachments: &[]AttachmentImportData{{Path: &testImage}}, + }}, + } + + if err := th.App.ImportDirectPost(directImportData, false); err != nil { + t.Fatalf("Expected success.") + } + + attachments = GetAttachments(user4.Id, th, t) + assert.Equal(t, len(attachments), 1) + assert.Contains(t, attachments[0].Path, "noteam") + AssertFileIdsInPost(attachments, th, t) + +} + +func GetAttachments(userId string, th *TestHelper, t *testing.T) []*model.FileInfo { + if result := <-th.App.Srv.Store.FileInfo().GetForUser(userId); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + return result.Data.([]*model.FileInfo) + } + return nil +} + +func AssertFileIdsInPost(files []*model.FileInfo, th *TestHelper, t *testing.T) { + postId := files[0].PostId + assert.NotNil(t, postId) + + if result := <-th.App.Srv.Store.Post().GetPostsByIds([]string{postId}); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + posts := result.Data.([]*model.Post) + assert.Equal(t, len(posts), 1) + for _, file := range files { + assert.Contains(t, posts[0].FileIds, file.Id) + } + } +} -- cgit v1.2.3-1-g7c22