From f7682ad11a3f27925b46ffc37fa66c6d4b6feef5 Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Tue, 28 Feb 2017 23:55:21 +0000 Subject: PLT-5367: Basic post bulk importing. (#5562) --- app/import.go | 125 ++++++++++++++++++- app/import_test.go | 360 ++++++++++++++++++++++++++++++++++++++++++++++++++++- app/slackimport.go | 18 +-- 3 files changed, 487 insertions(+), 16 deletions(-) (limited to 'app') diff --git a/app/import.go b/app/import.go index c50c6d62d..1ca532902 100644 --- a/app/import.go +++ b/app/import.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/json" "io" + "net/http" "regexp" "strings" "unicode/utf8" @@ -15,7 +16,6 @@ import ( l4g "github.com/alecthomas/log4go" "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" - "net/http" ) // Import Data Models @@ -25,6 +25,7 @@ type LineImportData struct { Team *TeamImportData `json:"team"` Channel *ChannelImportData `json:"channel"` User *UserImportData `json:"user"` + Post *PostImportData `json:"post"` Version *int `json:"version"` } @@ -85,6 +86,15 @@ type UserChannelNotifyPropsImportData struct { MarkUnread *string `json:"mark_unread"` } +type PostImportData struct { + Team *string `json:"team"` + Channel *string `json:"channel"` + User *string `json:"user"` + + Message *string `json:"message"` + CreateAt *int64 `json:"create_at"` +} + // // -- Bulk Import Functions -- // These functions import data directly into the database. Security and permission checks are bypassed but validity is @@ -152,6 +162,12 @@ func ImportLine(line LineImportData, dryRun bool) *model.AppError { } else { return ImportUser(line.User, dryRun) } + case line.Type == "post": + if line.Post == nil { + return model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest) + } else { + return ImportPost(line.Post, dryRun) + } default: return model.NewLocAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "") } @@ -698,13 +714,112 @@ func validateUserChannelsImportData(data *[]UserChannelImportData) *model.AppErr return nil } +func ImportPost(data *PostImportData, dryRun bool) *model.AppError { + if err := validatePostImportData(data); err != nil { + return err + } + + // If this is a Dry Run, do not continue any further. + if dryRun { + return nil + } + + var team *model.Team + if result := <-Srv.Store.Team().GetByName(*data.Team); result.Err != nil { + return model.NewAppError("BulkImport", "app.import.import_post.team_not_found.error", map[string]interface{}{"TeamName": *data.Team}, "", http.StatusBadRequest) + } else { + team = result.Data.(*model.Team) + } + + var channel *model.Channel + if result := <-Srv.Store.Channel().GetByName(team.Id, *data.Channel, false); result.Err != nil { + return model.NewAppError("BulkImport", "app.import.import_post.channel_not_found.error", map[string]interface{}{"ChannelName": *data.Channel}, "", http.StatusBadRequest) + } else { + channel = result.Data.(*model.Channel) + } + + var user *model.User + if result := <-Srv.Store.User().GetByUsername(*data.User); result.Err != nil { + return model.NewAppError("BulkImport", "app.import.import_post.user_not_found.error", map[string]interface{}{"Username": *data.User}, "", http.StatusBadRequest) + } else { + user = result.Data.(*model.User) + } + + // Check if this post already exists. + var posts []*model.Post + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, *data.CreateAt); result.Err != nil { + return result.Err + } else { + posts = result.Data.([]*model.Post) + } + + var post *model.Post + for _, p := range posts { + if p.Message == *data.Message { + post = p + break + } + } + + if post == nil { + post = &model.Post{} + } + + post.ChannelId = channel.Id + post.Message = *data.Message + post.UserId = user.Id + post.CreateAt = *data.CreateAt + + post.Hashtags, _ = model.ParseHashtags(post.Message) + + if post.Id == "" { + if result := <-Srv.Store.Post().Save(post); result.Err != nil { + return result.Err + } + } else { + if result := <-Srv.Store.Post().Overwrite(post); result.Err != nil { + return result.Err + } + } + + return nil +} + +func validatePostImportData(data *PostImportData) *model.AppError { + if data.Team == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.team_missing.error", nil, "", http.StatusBadRequest) + } + + if data.Channel == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.channel_missing.error", nil, "", http.StatusBadRequest) + } + + if data.User == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.user_missing.error", nil, "", http.StatusBadRequest) + } + + if data.Message == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.message_missing.error", nil, "", http.StatusBadRequest) + } else if utf8.RuneCountInString(*data.Message) > model.POST_MESSAGE_MAX_RUNES { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.message_length.error", nil, "", http.StatusBadRequest) + } + + if data.CreateAt == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.create_at_missing.error", nil, "", http.StatusBadRequest) + } else if *data.CreateAt == 0 { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.create_at_zero.error", nil, "", http.StatusBadRequest) + } + + return nil +} + // // -- Old SlackImport Functions -- // Import functions are sutible for entering posts and users into the database without // some of the usual checks. (IsValid is still run) // -func ImportPost(post *model.Post) { +func OldImportPost(post *model.Post) { // Workaround for empty messages, which may be the case if they are webhook posts. firstIteration := true for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { @@ -768,7 +883,7 @@ func OldImportChannel(channel *model.Channel) *model.Channel { } } -func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { +func OldImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) data := buf.Bytes() @@ -787,7 +902,7 @@ func ImportFile(file io.Reader, teamId string, channelId string, userId string, return fileInfo, nil } -func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { +func OldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") @@ -838,5 +953,5 @@ func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { } } - ImportPost(post) + OldImportPost(post) } diff --git a/app/import_test.go b/app/import_test.go index ddcc2d06a..165c94875 100644 --- a/app/import_test.go +++ b/app/import_test.go @@ -6,6 +6,7 @@ package app import ( "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" + "runtime/debug" "strings" "testing" ) @@ -454,6 +455,108 @@ func TestImportValidateUserChannelsImportData(t *testing.T) { } } +func TestImportValidatePostImportData(t *testing.T) { + + // Test with minimum required valid properties. + data := PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err != nil { + t.Fatal("Validation failed but should have been valid.") + } + + // Test with missing required properties. + data = PostImportData{ + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + // Test with invalid message. + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr(strings.Repeat("1234567890", 500)), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to too long message.") + } + + // Test with invalid CreateAt + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(0), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to 0 create-at value.") + } + + // Test with valid all optional parameters. + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err != nil { + t.Fatal("Should have succeeded.") + } +} + func TestImportImportTeam(t *testing.T) { _ = Setup() @@ -1333,6 +1436,252 @@ func TestImportImportUser(t *testing.T) { } } +func AssertAllPostsCount(t *testing.T, initialCount int64, change int64, teamName string) { + if result := <-Srv.Store.Post().AnalyticsPostCount(teamName, false, false); result.Err != nil { + t.Fatal(result.Err) + } else { + if initialCount+change != result.Data.(int64) { + debug.PrintStack() + t.Fatalf("Did not find the expected number of posts.") + } + } +} + +func TestImportImportPost(t *testing.T) { + _ = Setup() + + // Create a Team. + teamName := model.NewId() + ImportTeam(&TeamImportData{ + Name: &teamName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + team, err := GetTeamByName(teamName) + if err != nil { + t.Fatalf("Failed to get team from database.") + } + + // Create a Channel. + channelName := model.NewId() + ImportChannel(&ChannelImportData{ + Team: &teamName, + Name: &channelName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + channel, err := GetChannelByName(channelName, team.Id) + if err != nil { + t.Fatalf("Failed to get channel from database.") + } + + // Create a user. + username := model.NewId() + ImportUser(&UserImportData{ + Username: &username, + Email: ptrStr(model.NewId() + "@example.com"), + }, false) + user, err := GetUserByUsername(username) + if err != nil { + t.Fatalf("Failed to get user from database.") + } + + // Count the number of posts in the testing team. + var initialPostCount int64 + if result := <-Srv.Store.Post().AnalyticsPostCount(team.Id, false, false); result.Err != nil { + t.Fatal(result.Err) + } else { + initialPostCount = result.Data.(int64) + } + + // Try adding an invalid post in dry run mode. + data := &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + } + if err := ImportPost(data, true); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post in dry run mode. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Hello"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, true); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding an invalid post in apply mode. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post with invalid team in apply mode. + data = &PostImportData{ + Team: ptrStr(model.NewId()), + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post with invalid channel in apply mode. + data = &PostImportData{ + Team: &teamName, + Channel: ptrStr(model.NewId()), + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post with invalid user in apply mode. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: ptrStr(model.NewId()), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post in apply mode. + time := model.GetMillis() + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &time, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 1, team.Id) + + // Check the post values. + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, time); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + posts := result.Data.([]*model.Post) + if len(posts) != 1 { + t.Fatal("Unexpected number of posts found.") + } + post := posts[0] + if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id { + t.Fatal("Post properties not as expected") + } + } + + // Update the post. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &time, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 1, team.Id) + + // Check the post values. + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, time); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + posts := result.Data.([]*model.Post) + if len(posts) != 1 { + t.Fatal("Unexpected number of posts found.") + } + post := posts[0] + if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id { + t.Fatal("Post properties not as expected") + } + } + + // Save the post with a different time. + newTime := time + 1 + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &newTime, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 2, team.Id) + + // Save the post with a different message. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2"), + CreateAt: &time, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 3, team.Id) + + // Test with hashtags + hashtagTime := time + 2 + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2 #hashtagmashupcity"), + CreateAt: &hashtagTime, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 4, team.Id) + + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, hashtagTime); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + posts := result.Data.([]*model.Post) + if len(posts) != 1 { + t.Fatal("Unexpected number of posts found.") + } + post := posts[0] + if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id { + t.Fatal("Post properties not as expected") + } + if post.Hashtags != "#hashtagmashupcity" { + t.Fatalf("Hashtags not as expected: %s", post.Hashtags) + } + } +} + func TestImportImportLine(t *testing.T) { _ = Setup() @@ -1362,6 +1711,12 @@ func TestImportImportLine(t *testing.T) { if err := ImportLine(line, false); err == nil { t.Fatalf("Expected an error when importing a line with type uesr with a nil user.") } + + // Try import line with post type but nil post. + line.Type = "post" + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line with type post with a nil post.") + } } func TestImportBulkImport(t *testing.T) { @@ -1369,13 +1724,14 @@ func TestImportBulkImport(t *testing.T) { teamName := model.NewId() channelName := model.NewId() + username := model.NewId() // Run bulk import with a valid 1 of everything. data1 := `{"type": "version", "version": 1} {"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} {"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}} -{"type": "user", "user": {"username": "kufjgnkxkrhhfgbrip6qxkfsaa", "email": "kufjgnkxkrhhfgbrip6qxkfsaa@example.com"}} -{"type": "user", "user": {"username": "bwshaim6qnc2ne7oqkd5b2s2rq", "email": "bwshaim6qnc2ne7oqkd5b2s2rq@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}` +{"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}} +{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}}` if err, line := BulkImport(strings.NewReader(data1), false); err != nil || line != 0 { t.Fatalf("BulkImport should have succeeded: %v, %v", err.Error(), line) diff --git a/app/slackimport.go b/app/slackimport.go index e54d0724b..c3d968907 100644 --- a/app/slackimport.go +++ b/app/slackimport.go @@ -242,7 +242,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use newPost.Message = sPost.File.Title } } - ImportPost(&newPost) + OldImportPost(&newPost) for _, fileId := range newPost.FileIds { if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil { l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err) @@ -266,7 +266,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use Message: sPost.Comment.Comment, CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "bot_message": if botUser == nil { l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn")) @@ -298,7 +298,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use Type: model.POST_SLACK_ATTACHMENT, } - ImportIncomingWebhookPost(post, props) + OldImportIncomingWebhookPost(post, props) case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -325,7 +325,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use "username": users[sPost.User].Username, }, } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "me_message": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug")) @@ -340,7 +340,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use Message: "*" + sPost.Text + "*", CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_topic": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -356,7 +356,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_HEADER_CHANGE, } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_purpose": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -372,7 +372,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_PURPOSE_CHANGE, } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_name": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -388,7 +388,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_DISPLAYNAME_CHANGE, } - ImportPost(&newPost) + OldImportPost(&newPost) default: l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType) } @@ -405,7 +405,7 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin } defer openFile.Close() - uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) + uploadedFile, err := OldImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) if err != nil { l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) return nil, false -- cgit v1.2.3-1-g7c22