From e07e9937e0e0ae4fcb2a51553238a7566d1e4c67 Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Fri, 27 Jan 2017 15:14:54 +0000 Subject: PLT-5366, PLT-5364, PLT-5363: Bulk Import Part 1. (#5204) This commit provides the first part of the bulk import system. The CLI command is provided, complete with validation & apply modes. All the basic properties of Teams and Channels can be imported. Users & Posts will follow separately in a future commit. --- app/import.go | 240 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 239 insertions(+), 1 deletion(-) (limited to 'app/import.go') diff --git a/app/import.go b/app/import.go index 8f2cf552e..882641799 100644 --- a/app/import.go +++ b/app/import.go @@ -4,9 +4,12 @@ package app import ( + "bufio" "bytes" + "encoding/json" "io" "regexp" + "strings" "unicode/utf8" l4g "github.com/alecthomas/log4go" @@ -14,7 +17,242 @@ import ( "github.com/mattermost/platform/utils" ) +// Import Data Models + +type LineImportData struct { + Type string `json:"type"` + Team *TeamImportData `json:"team"` + Channel *ChannelImportData `json:"channel"` +} + +type TeamImportData struct { + Name *string `json:"name"` + DisplayName *string `json:"display_name"` + Type *string `json:"type"` + Description *string `json:"description"` + AllowOpenInvite *bool `json:"allow_open_invite"` +} + +type ChannelImportData struct { + Team *string `json:"team"` + Name *string `json:"name"` + DisplayName *string `json:"display_name"` + Type *string `json:"type"` + Header *string `json:"header"` + Purpose *string `json:"purpose"` +} + +// +// -- Bulk Import Functions -- +// These functions import data directly into the database. Security and permission checks are bypassed but validity is +// still enforced. +// + +func BulkImport(fileReader io.Reader, dryRun bool) (*model.AppError, int) { + scanner := bufio.NewScanner(fileReader) + lineNumber := 0 + for scanner.Scan() { + decoder := json.NewDecoder(strings.NewReader(scanner.Text())) + lineNumber++ + + var line LineImportData + if err := decoder.Decode(&line); err != nil { + return model.NewLocAppError("BulkImport", "app.import.bulk_import.json_decode.error", nil, err.Error()), lineNumber + } else { + if err := ImportLine(line, dryRun); err != nil { + return err, lineNumber + } + } + } + + if err := scanner.Err(); err != nil { + return model.NewLocAppError("BulkImport", "app.import.bulk_import.file_scan.error", nil, err.Error()), 0 + } + + return nil, 0 +} + +func ImportLine(line LineImportData, dryRun bool) *model.AppError { + switch { + case line.Type == "team": + if line.Team == nil { + return model.NewLocAppError("BulkImport", "app.import.import_line.null_team.error", nil, "") + } else { + return ImportTeam(line.Team, dryRun) + } + case line.Type == "channel": + if line.Channel == nil { + return model.NewLocAppError("BulkImport", "app.import.import_line.null_channel.error", nil, "") + } else { + return ImportChannel(line.Channel, dryRun) + } + default: + return model.NewLocAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "") + } +} + +func ImportTeam(data *TeamImportData, dryRun bool) *model.AppError { + if err := validateTeamImportData(data); err != nil { + return err + } + + // If this is a Dry Run, do not continue any further. + if dryRun { + return nil + } + + var team *model.Team + if result := <-Srv.Store.Team().GetByName(*data.Name); result.Err == nil { + team = result.Data.(*model.Team) + } else { + team = &model.Team{} + } + + team.Name = *data.Name + team.DisplayName = *data.DisplayName + team.Type = *data.Type + + if data.Description != nil { + team.Description = *data.Description + } + + if data.AllowOpenInvite != nil { + team.AllowOpenInvite = *data.AllowOpenInvite + } + + if team.Id == "" { + if _, err := CreateTeam(team); err != nil { + return err + } + } else { + if _, err := UpdateTeam(team); err != nil { + return err + } + } + + return nil +} + +func validateTeamImportData(data *TeamImportData) *model.AppError { + + if data.Name == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_missing.error", nil, "") + } else if len(*data.Name) > model.TEAM_NAME_MAX_LENGTH { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_length.error", nil, "") + } else if model.IsReservedTeamName(*data.Name) { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_reserved.error", nil, "") + } else if !model.IsValidTeamName(*data.Name) { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_characters.error", nil, "") + } + + if data.DisplayName == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.display_name_missing.error", nil, "") + } else if utf8.RuneCountInString(*data.DisplayName) == 0 || utf8.RuneCountInString(*data.DisplayName) > model.TEAM_DISPLAY_NAME_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.display_name_length.error", nil, "") + } + + if data.Type == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.type_missing.error", nil, "") + } else if *data.Type != model.TEAM_OPEN && *data.Type != model.TEAM_INVITE { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.type_invalid.error", nil, "") + } + + if data.Description != nil && len(*data.Description) > model.TEAM_DESCRIPTION_MAX_LENGTH { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.description_length.error", nil, "") + } + + return nil +} + +func ImportChannel(data *ChannelImportData, dryRun bool) *model.AppError { + if err := validateChannelImportData(data); err != nil { + return err + } + + // If this is a Dry Run, do not continue any further. + if dryRun { + return nil + } + + var team *model.Team + if result := <-Srv.Store.Team().GetByName(*data.Team); result.Err != nil { + return model.NewLocAppError("BulkImport", "app.import.import_channel.team_not_found.error", map[string]interface{}{"TeamName": *data.Team}, "") + } else { + team = result.Data.(*model.Team) + } + + var channel *model.Channel + if result := <-Srv.Store.Channel().GetByNameIncludeDeleted(team.Id, *data.Name); result.Err == nil { + channel = result.Data.(*model.Channel) + } else { + channel = &model.Channel{} + } + + channel.TeamId = team.Id + channel.Name = *data.Name + channel.DisplayName = *data.DisplayName + channel.Type = *data.Type + + if data.Header != nil { + channel.Header = *data.Header + } + + if data.Purpose != nil { + channel.Purpose = *data.Purpose + } + + if channel.Id == "" { + if _, err := CreateChannel(channel, false); err != nil { + return err + } + } else { + if _, err := UpdateChannel(channel); err != nil { + return err + } + } + + return nil +} + +func validateChannelImportData(data *ChannelImportData) *model.AppError { + + if data.Team == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.team_missing.error", nil, "") + } + + if data.Name == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_missing.error", nil, "") + } else if len(*data.Name) > model.CHANNEL_NAME_MAX_LENGTH { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_length.error", nil, "") + } else if !model.IsValidChannelIdentifier(*data.Name) { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_characters.error", nil, "") + } + + if data.DisplayName == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.display_name_missing.error", nil, "") + } else if utf8.RuneCountInString(*data.DisplayName) == 0 || utf8.RuneCountInString(*data.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.display_name_length.error", nil, "") + } + + if data.Type == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.type_missing.error", nil, "") + } else if *data.Type != model.CHANNEL_OPEN && *data.Type != model.CHANNEL_PRIVATE { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.type_invalid.error", nil, "") + } + + if data.Header != nil && utf8.RuneCountInString(*data.Header) > model.CHANNEL_HEADER_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.header_length.error", nil, "") + } + + if data.Purpose != nil && utf8.RuneCountInString(*data.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.purpose_length.error", nil, "") + } + + return nil +} + // +// -- Old SlackImport Functions -- // Import functions are sutible for entering posts and users into the database without // some of the usual checks. (IsValid is still run) // @@ -73,7 +311,7 @@ func ImportUser(team *model.Team, user *model.User) *model.User { } } -func ImportChannel(channel *model.Channel) *model.Channel { +func OldImportChannel(channel *model.Channel) *model.Channel { if result := <-Srv.Store.Channel().Save(channel); result.Err != nil { return nil } else { -- cgit v1.2.3-1-g7c22