From f7682ad11a3f27925b46ffc37fa66c6d4b6feef5 Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Tue, 28 Feb 2017 23:55:21 +0000 Subject: PLT-5367: Basic post bulk importing. (#5562) --- app/import.go | 125 ++++++++++++++- app/import_test.go | 360 ++++++++++++++++++++++++++++++++++++++++++- app/slackimport.go | 18 +-- i18n/en.json | 52 +++++++ store/sql_post_store.go | 52 +++++++ store/sql_post_store_test.go | 138 +++++++++++++++++ store/store.go | 2 + 7 files changed, 731 insertions(+), 16 deletions(-) diff --git a/app/import.go b/app/import.go index c50c6d62d..1ca532902 100644 --- a/app/import.go +++ b/app/import.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/json" "io" + "net/http" "regexp" "strings" "unicode/utf8" @@ -15,7 +16,6 @@ import ( l4g "github.com/alecthomas/log4go" "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" - "net/http" ) // Import Data Models @@ -25,6 +25,7 @@ type LineImportData struct { Team *TeamImportData `json:"team"` Channel *ChannelImportData `json:"channel"` User *UserImportData `json:"user"` + Post *PostImportData `json:"post"` Version *int `json:"version"` } @@ -85,6 +86,15 @@ type UserChannelNotifyPropsImportData struct { MarkUnread *string `json:"mark_unread"` } +type PostImportData struct { + Team *string `json:"team"` + Channel *string `json:"channel"` + User *string `json:"user"` + + Message *string `json:"message"` + CreateAt *int64 `json:"create_at"` +} + // // -- Bulk Import Functions -- // These functions import data directly into the database. Security and permission checks are bypassed but validity is @@ -152,6 +162,12 @@ func ImportLine(line LineImportData, dryRun bool) *model.AppError { } else { return ImportUser(line.User, dryRun) } + case line.Type == "post": + if line.Post == nil { + return model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest) + } else { + return ImportPost(line.Post, dryRun) + } default: return model.NewLocAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "") } @@ -698,13 +714,112 @@ func validateUserChannelsImportData(data *[]UserChannelImportData) *model.AppErr return nil } +func ImportPost(data *PostImportData, dryRun bool) *model.AppError { + if err := validatePostImportData(data); err != nil { + return err + } + + // If this is a Dry Run, do not continue any further. + if dryRun { + return nil + } + + var team *model.Team + if result := <-Srv.Store.Team().GetByName(*data.Team); result.Err != nil { + return model.NewAppError("BulkImport", "app.import.import_post.team_not_found.error", map[string]interface{}{"TeamName": *data.Team}, "", http.StatusBadRequest) + } else { + team = result.Data.(*model.Team) + } + + var channel *model.Channel + if result := <-Srv.Store.Channel().GetByName(team.Id, *data.Channel, false); result.Err != nil { + return model.NewAppError("BulkImport", "app.import.import_post.channel_not_found.error", map[string]interface{}{"ChannelName": *data.Channel}, "", http.StatusBadRequest) + } else { + channel = result.Data.(*model.Channel) + } + + var user *model.User + if result := <-Srv.Store.User().GetByUsername(*data.User); result.Err != nil { + return model.NewAppError("BulkImport", "app.import.import_post.user_not_found.error", map[string]interface{}{"Username": *data.User}, "", http.StatusBadRequest) + } else { + user = result.Data.(*model.User) + } + + // Check if this post already exists. + var posts []*model.Post + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, *data.CreateAt); result.Err != nil { + return result.Err + } else { + posts = result.Data.([]*model.Post) + } + + var post *model.Post + for _, p := range posts { + if p.Message == *data.Message { + post = p + break + } + } + + if post == nil { + post = &model.Post{} + } + + post.ChannelId = channel.Id + post.Message = *data.Message + post.UserId = user.Id + post.CreateAt = *data.CreateAt + + post.Hashtags, _ = model.ParseHashtags(post.Message) + + if post.Id == "" { + if result := <-Srv.Store.Post().Save(post); result.Err != nil { + return result.Err + } + } else { + if result := <-Srv.Store.Post().Overwrite(post); result.Err != nil { + return result.Err + } + } + + return nil +} + +func validatePostImportData(data *PostImportData) *model.AppError { + if data.Team == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.team_missing.error", nil, "", http.StatusBadRequest) + } + + if data.Channel == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.channel_missing.error", nil, "", http.StatusBadRequest) + } + + if data.User == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.user_missing.error", nil, "", http.StatusBadRequest) + } + + if data.Message == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.message_missing.error", nil, "", http.StatusBadRequest) + } else if utf8.RuneCountInString(*data.Message) > model.POST_MESSAGE_MAX_RUNES { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.message_length.error", nil, "", http.StatusBadRequest) + } + + if data.CreateAt == nil { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.create_at_missing.error", nil, "", http.StatusBadRequest) + } else if *data.CreateAt == 0 { + return model.NewAppError("BulkImport", "app.import.validate_post_import_data.create_at_zero.error", nil, "", http.StatusBadRequest) + } + + return nil +} + // // -- Old SlackImport Functions -- // Import functions are sutible for entering posts and users into the database without // some of the usual checks. (IsValid is still run) // -func ImportPost(post *model.Post) { +func OldImportPost(post *model.Post) { // Workaround for empty messages, which may be the case if they are webhook posts. firstIteration := true for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { @@ -768,7 +883,7 @@ func OldImportChannel(channel *model.Channel) *model.Channel { } } -func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { +func OldImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) data := buf.Bytes() @@ -787,7 +902,7 @@ func ImportFile(file io.Reader, teamId string, channelId string, userId string, return fileInfo, nil } -func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { +func OldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") @@ -838,5 +953,5 @@ func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { } } - ImportPost(post) + OldImportPost(post) } diff --git a/app/import_test.go b/app/import_test.go index ddcc2d06a..165c94875 100644 --- a/app/import_test.go +++ b/app/import_test.go @@ -6,6 +6,7 @@ package app import ( "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" + "runtime/debug" "strings" "testing" ) @@ -454,6 +455,108 @@ func TestImportValidateUserChannelsImportData(t *testing.T) { } } +func TestImportValidatePostImportData(t *testing.T) { + + // Test with minimum required valid properties. + data := PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err != nil { + t.Fatal("Validation failed but should have been valid.") + } + + // Test with missing required properties. + data = PostImportData{ + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to missing required property.") + } + + // Test with invalid message. + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr(strings.Repeat("1234567890", 500)), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to too long message.") + } + + // Test with invalid CreateAt + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(0), + } + if err := validatePostImportData(&data); err == nil { + t.Fatal("Should have failed due to 0 create-at value.") + } + + // Test with valid all optional parameters. + data = PostImportData{ + Team: ptrStr("teamname"), + Channel: ptrStr("channelname"), + User: ptrStr("username"), + Message: ptrStr("message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := validatePostImportData(&data); err != nil { + t.Fatal("Should have succeeded.") + } +} + func TestImportImportTeam(t *testing.T) { _ = Setup() @@ -1333,6 +1436,252 @@ func TestImportImportUser(t *testing.T) { } } +func AssertAllPostsCount(t *testing.T, initialCount int64, change int64, teamName string) { + if result := <-Srv.Store.Post().AnalyticsPostCount(teamName, false, false); result.Err != nil { + t.Fatal(result.Err) + } else { + if initialCount+change != result.Data.(int64) { + debug.PrintStack() + t.Fatalf("Did not find the expected number of posts.") + } + } +} + +func TestImportImportPost(t *testing.T) { + _ = Setup() + + // Create a Team. + teamName := model.NewId() + ImportTeam(&TeamImportData{ + Name: &teamName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + team, err := GetTeamByName(teamName) + if err != nil { + t.Fatalf("Failed to get team from database.") + } + + // Create a Channel. + channelName := model.NewId() + ImportChannel(&ChannelImportData{ + Team: &teamName, + Name: &channelName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + channel, err := GetChannelByName(channelName, team.Id) + if err != nil { + t.Fatalf("Failed to get channel from database.") + } + + // Create a user. + username := model.NewId() + ImportUser(&UserImportData{ + Username: &username, + Email: ptrStr(model.NewId() + "@example.com"), + }, false) + user, err := GetUserByUsername(username) + if err != nil { + t.Fatalf("Failed to get user from database.") + } + + // Count the number of posts in the testing team. + var initialPostCount int64 + if result := <-Srv.Store.Post().AnalyticsPostCount(team.Id, false, false); result.Err != nil { + t.Fatal(result.Err) + } else { + initialPostCount = result.Data.(int64) + } + + // Try adding an invalid post in dry run mode. + data := &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + } + if err := ImportPost(data, true); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post in dry run mode. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Hello"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, true); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding an invalid post in apply mode. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post with invalid team in apply mode. + data = &PostImportData{ + Team: ptrStr(model.NewId()), + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post with invalid channel in apply mode. + data = &PostImportData{ + Team: &teamName, + Channel: ptrStr(model.NewId()), + User: &username, + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post with invalid user in apply mode. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: ptrStr(model.NewId()), + Message: ptrStr("Message"), + CreateAt: ptrInt64(model.GetMillis()), + } + if err := ImportPost(data, false); err == nil { + t.Fatalf("Expected error.") + } + AssertAllPostsCount(t, initialPostCount, 0, team.Id) + + // Try adding a valid post in apply mode. + time := model.GetMillis() + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &time, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 1, team.Id) + + // Check the post values. + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, time); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + posts := result.Data.([]*model.Post) + if len(posts) != 1 { + t.Fatal("Unexpected number of posts found.") + } + post := posts[0] + if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id { + t.Fatal("Post properties not as expected") + } + } + + // Update the post. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &time, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 1, team.Id) + + // Check the post values. + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, time); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + posts := result.Data.([]*model.Post) + if len(posts) != 1 { + t.Fatal("Unexpected number of posts found.") + } + post := posts[0] + if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id { + t.Fatal("Post properties not as expected") + } + } + + // Save the post with a different time. + newTime := time + 1 + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message"), + CreateAt: &newTime, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 2, team.Id) + + // Save the post with a different message. + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2"), + CreateAt: &time, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 3, team.Id) + + // Test with hashtags + hashtagTime := time + 2 + data = &PostImportData{ + Team: &teamName, + Channel: &channelName, + User: &username, + Message: ptrStr("Message 2 #hashtagmashupcity"), + CreateAt: &hashtagTime, + } + if err := ImportPost(data, false); err != nil { + t.Fatalf("Expected success.") + } + AssertAllPostsCount(t, initialPostCount, 4, team.Id) + + if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, hashtagTime); result.Err != nil { + t.Fatal(result.Err.Error()) + } else { + posts := result.Data.([]*model.Post) + if len(posts) != 1 { + t.Fatal("Unexpected number of posts found.") + } + post := posts[0] + if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id { + t.Fatal("Post properties not as expected") + } + if post.Hashtags != "#hashtagmashupcity" { + t.Fatalf("Hashtags not as expected: %s", post.Hashtags) + } + } +} + func TestImportImportLine(t *testing.T) { _ = Setup() @@ -1362,6 +1711,12 @@ func TestImportImportLine(t *testing.T) { if err := ImportLine(line, false); err == nil { t.Fatalf("Expected an error when importing a line with type uesr with a nil user.") } + + // Try import line with post type but nil post. + line.Type = "post" + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line with type post with a nil post.") + } } func TestImportBulkImport(t *testing.T) { @@ -1369,13 +1724,14 @@ func TestImportBulkImport(t *testing.T) { teamName := model.NewId() channelName := model.NewId() + username := model.NewId() // Run bulk import with a valid 1 of everything. data1 := `{"type": "version", "version": 1} {"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} {"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}} -{"type": "user", "user": {"username": "kufjgnkxkrhhfgbrip6qxkfsaa", "email": "kufjgnkxkrhhfgbrip6qxkfsaa@example.com"}} -{"type": "user", "user": {"username": "bwshaim6qnc2ne7oqkd5b2s2rq", "email": "bwshaim6qnc2ne7oqkd5b2s2rq@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}` +{"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}} +{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}}` if err, line := BulkImport(strings.NewReader(data1), false); err != nil || line != 0 { t.Fatalf("BulkImport should have succeeded: %v, %v", err.Error(), line) diff --git a/app/slackimport.go b/app/slackimport.go index e54d0724b..c3d968907 100644 --- a/app/slackimport.go +++ b/app/slackimport.go @@ -242,7 +242,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use newPost.Message = sPost.File.Title } } - ImportPost(&newPost) + OldImportPost(&newPost) for _, fileId := range newPost.FileIds { if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil { l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err) @@ -266,7 +266,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use Message: sPost.Comment.Comment, CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "bot_message": if botUser == nil { l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn")) @@ -298,7 +298,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use Type: model.POST_SLACK_ATTACHMENT, } - ImportIncomingWebhookPost(post, props) + OldImportIncomingWebhookPost(post, props) case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -325,7 +325,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use "username": users[sPost.User].Username, }, } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "me_message": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug")) @@ -340,7 +340,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use Message: "*" + sPost.Text + "*", CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_topic": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -356,7 +356,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_HEADER_CHANGE, } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_purpose": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -372,7 +372,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_PURPOSE_CHANGE, } - ImportPost(&newPost) + OldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_name": if sPost.User == "" { l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug")) @@ -388,7 +388,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_DISPLAYNAME_CHANGE, } - ImportPost(&newPost) + OldImportPost(&newPost) default: l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType) } @@ -405,7 +405,7 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin } defer openFile.Close() - uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) + uploadedFile, err := OldImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name)) if err != nil { l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()})) return nil, false diff --git a/i18n/en.json b/i18n/en.json index d46812bd8..e6efcafda 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -2983,6 +2983,50 @@ "id": "app.import.validate_user_teams_import_data.team_name_missing.error", "translation": "Team name missing from User's Team Membership." }, + { + "id": "app.import.import_post.team_not_found.error", + "translation": "Error importing post. Team with name \"{{.TeamName}}\" could not be found." + }, + { + "id": "app.import.import_post.channel_not_found.error", + "translation": "Error importing post. Channel with name \"{{.ChannelName}}\" could not be found." + }, + { + "id": "app.import.import_post.user_not_found.error", + "translation": "Error importing post. User with username \"{{.Username}}\" could not be found." + }, + { + "id": "app.import.validate_post_import_data.team_missing.error", + "translation": "Missing required Post property: Team." + }, + { + "id": "app.import.validate_post_import_data.channel_missing.error", + "translation": "Missing required Post property: Channel." + }, + { + "id": "app.import.validate_post_import_data.user_missing.error", + "translation": "Missing required Post property: User." + }, + { + "id": "app.import.validate_post_import_data.message_missing.error", + "translation": "Missing required Post property: Message." + }, + { + "id": "app.import.validate_post_import_data.create_at_missing.error", + "translation": "Missing required Post property: create_at." + }, + { + "id": "app.import.validate_post_import_data.message_length.error", + "translation": "Post Message property is longer than the maximum permitted length." + }, + { + "id": "app.import.validate_post_import_data.create_at_zero.error", + "translation": "Post CreateAt must not be zero if it is provided." + }, + { + "id": "app.import.import_line.null_post.error", + "translation": "Import data line has type \"post\" but the post object is null." + }, { "id": "authentication.permissions.create_team_roles.description", "translation": "Ability to create new teams" @@ -4943,6 +4987,10 @@ "id": "store.sql_post.get_posts_around.get.app_error", "translation": "We couldn't get the posts for the channel" }, + { + "id": "store.sql_post.get_posts_created_att.app_error", + "translation": "We couldn't get the posts for the channel" + }, { "id": "store.sql_post.get_posts_around.get_parent.app_error", "translation": "We couldn't get the parent posts for the channel" @@ -4991,6 +5039,10 @@ "id": "store.sql_post.update.app_error", "translation": "We couldn't update the Post" }, + { + "id": "store.sql_post.overwrite.app_error", + "translation": "We couldn't overwrite the Post" + }, { "id": "store.sql_preference.delete.app_error", "translation": "We encountered an error while deleting preferences" diff --git a/store/sql_post_store.go b/store/sql_post_store.go index 060a3b1cf..e224f60bd 100644 --- a/store/sql_post_store.go +++ b/store/sql_post_store.go @@ -12,6 +12,7 @@ import ( "github.com/mattermost/platform/einterfaces" "github.com/mattermost/platform/model" "github.com/mattermost/platform/utils" + "net/http" ) type SqlPostStore struct { @@ -157,6 +158,33 @@ func (s SqlPostStore) Update(newPost *model.Post, oldPost *model.Post) StoreChan return storeChannel } +func (s SqlPostStore) Overwrite(post *model.Post) StoreChannel { + storeChannel := make(StoreChannel, 1) + + go func() { + result := StoreResult{} + + post.UpdateAt = model.GetMillis() + + if result.Err = post.IsValid(); result.Err != nil { + storeChannel <- result + close(storeChannel) + return + } + + if _, err := s.GetMaster().Update(post); err != nil { + result.Err = model.NewLocAppError("SqlPostStore.Overwrite", "store.sql_post.overwrite.app_error", nil, "id="+post.Id+", "+err.Error()) + } else { + result.Data = post + } + + storeChannel <- result + close(storeChannel) + }() + + return storeChannel +} + func (s SqlPostStore) GetFlaggedPosts(userId string, offset int, limit int) StoreChannel { storeChannel := make(StoreChannel, 1) go func() { @@ -1132,3 +1160,27 @@ func (s SqlPostStore) AnalyticsPostCount(teamId string, mustHaveFile bool, mustH return storeChannel } + +func (s SqlPostStore) GetPostsCreatedAt(channelId string, time int64) StoreChannel { + storeChannel := make(StoreChannel, 1) + + go func() { + result := StoreResult{} + + query := `SELECT * FROM Posts WHERE CreateAt = :CreateAt` + + var posts []*model.Post + _, err := s.GetReplica().Select(&posts, query, map[string]interface{}{"CreateAt": time}) + + if err != nil { + result.Err = model.NewAppError("SqlPostStore.GetPostsCreatedAt", "store.sql_post.get_posts_created_att.app_error", nil, "channelId="+channelId+err.Error(), http.StatusInternalServerError) + } else { + result.Data = posts + } + + storeChannel <- result + close(storeChannel) + }() + + return storeChannel +} diff --git a/store/sql_post_store_test.go b/store/sql_post_store_test.go index 08fe1282e..82490fffd 100644 --- a/store/sql_post_store_test.go +++ b/store/sql_post_store_test.go @@ -1127,3 +1127,141 @@ func TestPostStoreGetFlaggedPosts(t *testing.T) { t.Fatal("should have 2 posts") } } + +func TestPostStoreGetPostsCreatedAt(t *testing.T) { + Setup() + + createTime := model.GetMillis() + + o0 := &model.Post{} + o0.ChannelId = model.NewId() + o0.UserId = model.NewId() + o0.Message = "a" + model.NewId() + "b" + o0.CreateAt = createTime + o0 = (<-store.Post().Save(o0)).Data.(*model.Post) + + o1 := &model.Post{} + o1.ChannelId = o0.Id + o1.UserId = model.NewId() + o1.Message = "a" + model.NewId() + "b" + o0.CreateAt = createTime + o1 = (<-store.Post().Save(o1)).Data.(*model.Post) + time.Sleep(2 * time.Millisecond) + + o2 := &model.Post{} + o2.ChannelId = o1.ChannelId + o2.UserId = model.NewId() + o2.Message = "a" + model.NewId() + "b" + o2.ParentId = o1.Id + o2.RootId = o1.Id + o2 = (<-store.Post().Save(o2)).Data.(*model.Post) + time.Sleep(2 * time.Millisecond) + + o3 := &model.Post{} + o3.ChannelId = model.NewId() + o3.UserId = model.NewId() + o3.Message = "a" + model.NewId() + "b" + o3.CreateAt = createTime + o3 = (<-store.Post().Save(o3)).Data.(*model.Post) + time.Sleep(2 * time.Millisecond) + + r1 := (<-store.Post().GetPostsCreatedAt(o1.ChannelId, createTime)).Data.([]*model.Post) + + if len(r1) != 2 { + t.Fatalf("Got the wrong number of posts.") + } +} + +func TestPostStoreOverwrite(t *testing.T) { + Setup() + + o1 := &model.Post{} + o1.ChannelId = model.NewId() + o1.UserId = model.NewId() + o1.Message = "a" + model.NewId() + "AAAAAAAAAAA" + o1 = (<-store.Post().Save(o1)).Data.(*model.Post) + + o2 := &model.Post{} + o2.ChannelId = o1.ChannelId + o2.UserId = model.NewId() + o2.Message = "a" + model.NewId() + "CCCCCCCCC" + o2.ParentId = o1.Id + o2.RootId = o1.Id + o2 = (<-store.Post().Save(o2)).Data.(*model.Post) + + o3 := &model.Post{} + o3.ChannelId = o1.ChannelId + o3.UserId = model.NewId() + o3.Message = "a" + model.NewId() + "QQQQQQQQQQ" + o3 = (<-store.Post().Save(o3)).Data.(*model.Post) + + ro1 := (<-store.Post().Get(o1.Id)).Data.(*model.PostList).Posts[o1.Id] + ro2 := (<-store.Post().Get(o1.Id)).Data.(*model.PostList).Posts[o2.Id] + ro3 := (<-store.Post().Get(o3.Id)).Data.(*model.PostList).Posts[o3.Id] + + if ro1.Message != o1.Message { + t.Fatal("Failed to save/get") + } + + o1a := &model.Post{} + *o1a = *ro1 + o1a.Message = ro1.Message + "BBBBBBBBBB" + if result := <-store.Post().Overwrite(o1a); result.Err != nil { + t.Fatal(result.Err) + } + + ro1a := (<-store.Post().Get(o1.Id)).Data.(*model.PostList).Posts[o1.Id] + + if ro1a.Message != o1a.Message { + t.Fatal("Failed to overwrite/get") + } + + o2a := &model.Post{} + *o2a = *ro2 + o2a.Message = ro2.Message + "DDDDDDD" + if result := <-store.Post().Overwrite(o2a); result.Err != nil { + t.Fatal(result.Err) + } + + ro2a := (<-store.Post().Get(o1.Id)).Data.(*model.PostList).Posts[o2.Id] + + if ro2a.Message != o2a.Message { + t.Fatal("Failed to overwrite/get") + } + + o3a := &model.Post{} + *o3a = *ro3 + o3a.Message = ro3.Message + "WWWWWWW" + if result := <-store.Post().Overwrite(o3a); result.Err != nil { + t.Fatal(result.Err) + } + + ro3a := (<-store.Post().Get(o3.Id)).Data.(*model.PostList).Posts[o3.Id] + + if ro3a.Message != o3a.Message && ro3a.Hashtags != o3a.Hashtags { + t.Fatal("Failed to overwrite/get") + } + + o4 := Must(store.Post().Save(&model.Post{ + ChannelId: model.NewId(), + UserId: model.NewId(), + Message: model.NewId(), + Filenames: []string{"test"}, + })).(*model.Post) + + ro4 := (<-store.Post().Get(o4.Id)).Data.(*model.PostList).Posts[o4.Id] + + o4a := &model.Post{} + *o4a = *ro4 + o4a.Filenames = []string{} + o4a.FileIds = []string{model.NewId()} + if result := <-store.Post().Overwrite(o4a); result.Err != nil { + t.Fatal(result.Err) + } + + if ro4a := Must(store.Post().Get(o4.Id)).(*model.PostList).Posts[o4.Id]; len(ro4a.Filenames) != 0 { + t.Fatal("Failed to clear Filenames") + } else if len(ro4a.FileIds) != 1 { + t.Fatal("Failed to set FileIds") + } +} diff --git a/store/store.go b/store/store.go index 57741742c..a436f9ee7 100644 --- a/store/store.go +++ b/store/store.go @@ -152,6 +152,8 @@ type PostStore interface { AnalyticsPostCountsByDay(teamId string) StoreChannel AnalyticsPostCount(teamId string, mustHaveFile bool, mustHaveHashtag bool) StoreChannel InvalidateLastPostTimeCache(channelId string) + GetPostsCreatedAt(channelId string, time int64) StoreChannel + Overwrite(post *model.Post) StoreChannel } type UserStore interface { -- cgit v1.2.3-1-g7c22