diff options
author | George Goldberg <george@gberg.me> | 2018-09-17 15:51:26 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-09-17 15:51:26 +0100 |
commit | ab99f0656fabed8a62a8c6340be7d538cc7bf8d9 (patch) | |
tree | bb68ee1d0c743be23bba470f5d81ef11dc134182 /app | |
parent | 5786b0d6d57b90bbb0c262235dd9d19b497b5fae (diff) | |
download | chat-ab99f0656fabed8a62a8c6340be7d538cc7bf8d9.tar.gz chat-ab99f0656fabed8a62a8c6340be7d538cc7bf8d9.tar.bz2 chat-ab99f0656fabed8a62a8c6340be7d538cc7bf8d9.zip |
MM-11781: Basic Data Export Command Line. (#9296)
* MM-11781: Basic Data Export Command Line.
* ChannelStore new unit tests.
* TeamStore new unit tests.
* Unit test for new UserStore function.
* Unit tests for post store new methods.
* Review fixes.
* Fix duplicate command name.
Diffstat (limited to 'app')
-rw-r--r-- | app/export.go | 279 | ||||
-rw-r--r-- | app/export_converters.go | 113 | ||||
-rw-r--r-- | app/import_types.go | 80 |
3 files changed, 432 insertions, 40 deletions
diff --git a/app/export.go b/app/export.go new file mode 100644 index 000000000..a7a78cfee --- /dev/null +++ b/app/export.go @@ -0,0 +1,279 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "encoding/json" + "io" + "net/http" + "strings" + + "github.com/mattermost/mattermost-server/model" +) + +func (a *App) BulkExport(writer io.Writer) *model.AppError { + if err := a.ExportVersion(writer); err != nil { + return err + } + + if err := a.ExportAllTeams(writer); err != nil { + return err + } + + if err := a.ExportAllChannels(writer); err != nil { + return err + } + + if err := a.ExportAllUsers(writer); err != nil { + return err + } + + if err := a.ExportAllPosts(writer); err != nil { + return err + } + + return nil +} + +func (a *App) ExportWriteLine(writer io.Writer, line *LineImportData) *model.AppError { + b, err := json.Marshal(line) + if err != nil { + return model.NewAppError("BulkExport", "app.export.export_write_line.json_marshall.error", nil, "err="+err.Error(), http.StatusBadRequest) + } + + if _, err := writer.Write(append(b, '\n')); err != nil { + return model.NewAppError("BulkExport", "app.export.export_write_line.io_writer.error", nil, "err="+err.Error(), http.StatusBadRequest) + } + + return nil +} + +func (a *App) ExportVersion(writer io.Writer) *model.AppError { + version := 1 + versionLine := &LineImportData{ + Type: "version", + Version: &version, + } + + return a.ExportWriteLine(writer, versionLine) +} + +func (a *App) ExportAllTeams(writer io.Writer) *model.AppError { + afterId := strings.Repeat("0", 26) + for { + result := <-a.Srv.Store.Team().GetAllForExportAfter(1000, afterId) + + if result.Err != nil { + return result.Err + } + + teams := result.Data.([]*model.TeamForExport) + + if len(teams) == 0 { + break + } + + for _, team := range teams { + afterId = team.Id + + // Skip deleted. + if team.DeleteAt != 0 { + continue + } + + teamLine := ImportLineFromTeam(team) + if err := a.ExportWriteLine(writer, teamLine); err != nil { + return err + } + } + } + + return nil +} + +func (a *App) ExportAllChannels(writer io.Writer) *model.AppError { + afterId := strings.Repeat("0", 26) + for { + result := <-a.Srv.Store.Channel().GetAllChannelsForExportAfter(1000, afterId) + + if result.Err != nil { + return result.Err + } + + channels := result.Data.([]*model.ChannelForExport) + + if len(channels) == 0 { + break + } + + for _, channel := range channels { + afterId = channel.Id + + // Skip deleted. + if channel.DeleteAt != 0 { + continue + } + + channelLine := ImportLineFromChannel(channel) + if err := a.ExportWriteLine(writer, channelLine); err != nil { + return err + } + } + } + + return nil +} + +func (a *App) ExportAllUsers(writer io.Writer) *model.AppError { + afterId := strings.Repeat("0", 26) + for { + result := <-a.Srv.Store.User().GetAllAfter(1000, afterId) + + if result.Err != nil { + return result.Err + } + + users := result.Data.([]*model.User) + + if len(users) == 0 { + break + } + + for _, user := range users { + afterId = user.Id + + // Skip deleted. + if user.DeleteAt != 0 { + continue + } + + userLine := ImportLineFromUser(user) + + // Do the Team Memberships. + members, err := a.buildUserTeamAndChannelMemberships(user.Id) + if err != nil { + return err + } + + userLine.User.Teams = members + + if err := a.ExportWriteLine(writer, userLine); err != nil { + return err + } + } + } + + return nil +} + +func (a *App) buildUserTeamAndChannelMemberships(userId string) (*[]UserTeamImportData, *model.AppError) { + var memberships []UserTeamImportData + + result := <-a.Srv.Store.Team().GetTeamMembersForExport(userId) + + if result.Err != nil { + return nil, result.Err + } + + members := result.Data.([]*model.TeamMemberForExport) + + for _, member := range members { + // Skip deleted. + if member.DeleteAt != 0 { + continue + } + + memberData := ImportUserTeamDataFromTeamMember(member) + + // Do the Channel Memberships. + channelMembers, err := a.buildUserChannelMemberships(userId, member.TeamId) + if err != nil { + return nil, err + } + + memberData.Channels = channelMembers + + memberships = append(memberships, *memberData) + } + + return &memberships, nil +} + +func (a *App) buildUserChannelMemberships(userId string, teamId string) (*[]UserChannelImportData, *model.AppError) { + var memberships []UserChannelImportData + + result := <-a.Srv.Store.Channel().GetChannelMembersForExport(userId, teamId) + + if result.Err != nil { + return nil, result.Err + } + + members := result.Data.([]*model.ChannelMemberForExport) + + for _, member := range members { + memberships = append(memberships, *ImportUserChannelDataFromChannelMember(member)) + } + + return &memberships, nil +} + +func (a *App) ExportAllPosts(writer io.Writer) *model.AppError { + afterId := strings.Repeat("0", 26) + for { + result := <-a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId) + + if result.Err != nil { + return result.Err + } + + posts := result.Data.([]*model.PostForExport) + + if len(posts) == 0 { + break + } + + for _, post := range posts { + afterId = post.Id + + // Skip deleted. + if post.DeleteAt != 0 { + continue + } + + postLine := ImportLineForPost(post) + + // Do the Replies. + replies, err := a.buildPostReplies(post.Id) + if err != nil { + return err + } + + postLine.Post.Replies = replies + + if err := a.ExportWriteLine(writer, postLine); err != nil { + return err + } + } + } + + return nil +} + +func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) { + var replies []ReplyImportData + + result := <-a.Srv.Store.Post().GetRepliesForExport(postId) + + if result.Err != nil { + return nil, result.Err + } + + replyPosts := result.Data.([]*model.ReplyForExport) + + for _, reply := range replyPosts { + replies = append(replies, *ImportReplyFromPost(reply)) + } + + return &replies, nil +} diff --git a/app/export_converters.go b/app/export_converters.go new file mode 100644 index 000000000..cafe360cb --- /dev/null +++ b/app/export_converters.go @@ -0,0 +1,113 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "github.com/mattermost/mattermost-server/model" + "strings" +) + +func ImportLineFromTeam(team *model.TeamForExport) *LineImportData { + return &LineImportData{ + Type: "team", + Team: &TeamImportData{ + Name: &team.Name, + DisplayName: &team.DisplayName, + Type: &team.Type, + Description: &team.Description, + AllowOpenInvite: &team.AllowOpenInvite, + Scheme: team.SchemeName, + }, + } +} + +func ImportLineFromChannel(channel *model.ChannelForExport) *LineImportData { + return &LineImportData{ + Type: "channel", + Channel: &ChannelImportData{ + Team: &channel.TeamName, + Name: &channel.Name, + DisplayName: &channel.DisplayName, + Type: &channel.Type, + Header: &channel.Header, + Purpose: &channel.Purpose, + Scheme: channel.SchemeName, + }, + } +} + +func ImportLineFromUser(user *model.User) *LineImportData { + // Bulk Importer doesn't accept "empty string" for AuthService. + var authService *string + if user.AuthService != "" { + authService = &user.AuthService + } + + return &LineImportData{ + Type: "user", + User: &UserImportData{ + Username: &user.Username, + Email: &user.Email, + AuthService: authService, + AuthData: user.AuthData, + Nickname: &user.Nickname, + FirstName: &user.FirstName, + LastName: &user.LastName, + Position: &user.Position, + Roles: &user.Roles, + Locale: &user.Locale, + }, + } +} + +func ImportUserTeamDataFromTeamMember(member *model.TeamMemberForExport) *UserTeamImportData { + rolesList := strings.Fields(member.Roles) + if member.SchemeAdmin { + rolesList = append(rolesList, model.TEAM_ADMIN_ROLE_ID) + } + if member.SchemeUser { + rolesList = append(rolesList, model.TEAM_USER_ROLE_ID) + } + roles := strings.Join(rolesList, " ") + return &UserTeamImportData{ + Name: &member.TeamName, + Roles: &roles, + } +} + +func ImportUserChannelDataFromChannelMember(member *model.ChannelMemberForExport) *UserChannelImportData { + rolesList := strings.Fields(member.Roles) + if member.SchemeAdmin { + rolesList = append(rolesList, model.CHANNEL_ADMIN_ROLE_ID) + } + if member.SchemeUser { + rolesList = append(rolesList, model.CHANNEL_USER_ROLE_ID) + } + roles := strings.Join(rolesList, " ") + return &UserChannelImportData{ + Name: &member.ChannelName, + Roles: &roles, + } +} + +func ImportLineForPost(post *model.PostForExport) *LineImportData { + return &LineImportData{ + Type: "post", + Post: &PostImportData{ + Team: &post.TeamName, + Channel: &post.ChannelName, + User: &post.Username, + Message: &post.Message, + CreateAt: &post.CreateAt, + }, + } +} + +func ImportReplyFromPost(post *model.ReplyForExport) *ReplyImportData { + return &ReplyImportData{ + User: &post.Username, + Message: &post.Message, + CreateAt: &post.CreateAt, + } +} diff --git a/app/import_types.go b/app/import_types.go index 168d43bab..1a2e44018 100644 --- a/app/import_types.go +++ b/app/import_types.go @@ -9,24 +9,24 @@ import "github.com/mattermost/mattermost-server/model" type LineImportData struct { Type string `json:"type"` - Scheme *SchemeImportData `json:"scheme"` - Team *TeamImportData `json:"team"` - Channel *ChannelImportData `json:"channel"` - User *UserImportData `json:"user"` - Post *PostImportData `json:"post"` - DirectChannel *DirectChannelImportData `json:"direct_channel"` - DirectPost *DirectPostImportData `json:"direct_post"` - Emoji *EmojiImportData `json:"emoji"` - Version *int `json:"version"` + Scheme *SchemeImportData `json:"scheme,omitempty"` + Team *TeamImportData `json:"team,omitempty"` + Channel *ChannelImportData `json:"channel,omitempty"` + User *UserImportData `json:"user,omitempty"` + Post *PostImportData `json:"post,omitempty"` + DirectChannel *DirectChannelImportData `json:"direct_channel,omitempty"` + DirectPost *DirectPostImportData `json:"direct_post,omitempty"` + Emoji *EmojiImportData `json:"emoji,omitempty"` + Version *int `json:"version,omitempty"` } type TeamImportData struct { Name *string `json:"name"` DisplayName *string `json:"display_name"` Type *string `json:"type"` - Description *string `json:"description"` - AllowOpenInvite *bool `json:"allow_open_invite"` - Scheme *string `json:"scheme"` + Description *string `json:"description,omitempty"` + AllowOpenInvite *bool `json:"allow_open_invite,omitempty"` + Scheme *string `json:"scheme,omitempty"` } type ChannelImportData struct { @@ -34,38 +34,38 @@ type ChannelImportData struct { Name *string `json:"name"` DisplayName *string `json:"display_name"` Type *string `json:"type"` - Header *string `json:"header"` - Purpose *string `json:"purpose"` - Scheme *string `json:"scheme"` + Header *string `json:"header,omitempty"` + Purpose *string `json:"purpose,omitempty"` + Scheme *string `json:"scheme,omitempty"` } type UserImportData struct { - ProfileImage *string `json:"profile_image"` + ProfileImage *string `json:"profile_image,omitempty"` Username *string `json:"username"` Email *string `json:"email"` AuthService *string `json:"auth_service"` - AuthData *string `json:"auth_data"` - Password *string `json:"password"` + AuthData *string `json:"auth_data,omitempty"` + Password *string `json:"password,omitempty"` Nickname *string `json:"nickname"` FirstName *string `json:"first_name"` LastName *string `json:"last_name"` Position *string `json:"position"` Roles *string `json:"roles"` Locale *string `json:"locale"` - UseMarkdownPreview *string `json:"feature_enabled_markdown_preview"` - UseFormatting *string `json:"formatting"` - ShowUnreadSection *string `json:"show_unread_section"` + UseMarkdownPreview *string `json:"feature_enabled_markdown_preview,omitempty"` + UseFormatting *string `json:"formatting,omitempty"` + ShowUnreadSection *string `json:"show_unread_section,omitempty"` - Teams *[]UserTeamImportData `json:"teams"` + Teams *[]UserTeamImportData `json:"teams,omitempty"` - Theme *string `json:"theme"` - UseMilitaryTime *string `json:"military_time"` - CollapsePreviews *string `json:"link_previews"` - MessageDisplay *string `json:"message_display"` - ChannelDisplayMode *string `json:"channel_display_mode"` - TutorialStep *string `json:"tutorial_step"` + Theme *string `json:"theme,omitempty"` + UseMilitaryTime *string `json:"military_time,omitempty"` + CollapsePreviews *string `json:"link_previews,omitempty"` + MessageDisplay *string `json:"message_display,omitempty"` + ChannelDisplayMode *string `json:"channel_display_mode,omitempty"` + TutorialStep *string `json:"tutorial_step,omitempty"` - NotifyProps *UserNotifyPropsImportData `json:"notify_props"` + NotifyProps *UserNotifyPropsImportData `json:"notify_props,omitempty"` } type UserNotifyPropsImportData struct { @@ -85,15 +85,15 @@ type UserNotifyPropsImportData struct { type UserTeamImportData struct { Name *string `json:"name"` Roles *string `json:"roles"` - Theme *string `json:"theme"` - Channels *[]UserChannelImportData `json:"channels"` + Theme *string `json:"theme,omitempty"` + Channels *[]UserChannelImportData `json:"channels,omitempty"` } type UserChannelImportData struct { Name *string `json:"name"` Roles *string `json:"roles"` - NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props"` - Favorite *bool `json:"favorite"` + NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props,omitempty"` + Favorite *bool `json:"favorite,omitempty"` } type UserChannelNotifyPropsImportData struct { @@ -119,9 +119,9 @@ type ReplyImportData struct { Message *string `json:"message"` CreateAt *int64 `json:"create_at"` - FlaggedBy *[]string `json:"flagged_by"` - Reactions *[]ReactionImportData `json:"reactions"` - Attachments *[]AttachmentImportData `json:"attachments"` + FlaggedBy *[]string `json:"flagged_by,omitempty"` + Reactions *[]ReactionImportData `json:"reactions,omitempty"` + Attachments *[]AttachmentImportData `json:"attachments,omitempty"` } type PostImportData struct { @@ -132,10 +132,10 @@ type PostImportData struct { Message *string `json:"message"` CreateAt *int64 `json:"create_at"` - FlaggedBy *[]string `json:"flagged_by"` - Reactions *[]ReactionImportData `json:"reactions"` - Replies *[]ReplyImportData `json:"replies"` - Attachments *[]AttachmentImportData `json:"attachments"` + FlaggedBy *[]string `json:"flagged_by,omitempty"` + Reactions *[]ReactionImportData `json:"reactions,omitempty"` + Replies *[]ReplyImportData `json:"replies,omitempty"` + Attachments *[]AttachmentImportData `json:"attachments,omitempty"` } type DirectChannelImportData struct { |