From e07e9937e0e0ae4fcb2a51553238a7566d1e4c67 Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Fri, 27 Jan 2017 15:14:54 +0000 Subject: PLT-5366, PLT-5364, PLT-5363: Bulk Import Part 1. (#5204) This commit provides the first part of the bulk import system. The CLI command is provided, complete with validation & apply modes. All the basic properties of Teams and Channels can be imported. Users & Posts will follow separately in a future commit. --- app/import.go | 240 ++++++++++++++++++++- app/import_test.go | 552 +++++++++++++++++++++++++++++++++++++++++++++++++ app/slackimport.go | 2 +- cmd/platform/import.go | 56 +++++ i18n/en.json | 112 ++++++++++ model/team.go | 22 +- 6 files changed, 974 insertions(+), 10 deletions(-) create mode 100644 app/import_test.go diff --git a/app/import.go b/app/import.go index 8f2cf552e..882641799 100644 --- a/app/import.go +++ b/app/import.go @@ -4,9 +4,12 @@ package app import ( + "bufio" "bytes" + "encoding/json" "io" "regexp" + "strings" "unicode/utf8" l4g "github.com/alecthomas/log4go" @@ -14,7 +17,242 @@ import ( "github.com/mattermost/platform/utils" ) +// Import Data Models + +type LineImportData struct { + Type string `json:"type"` + Team *TeamImportData `json:"team"` + Channel *ChannelImportData `json:"channel"` +} + +type TeamImportData struct { + Name *string `json:"name"` + DisplayName *string `json:"display_name"` + Type *string `json:"type"` + Description *string `json:"description"` + AllowOpenInvite *bool `json:"allow_open_invite"` +} + +type ChannelImportData struct { + Team *string `json:"team"` + Name *string `json:"name"` + DisplayName *string `json:"display_name"` + Type *string `json:"type"` + Header *string `json:"header"` + Purpose *string `json:"purpose"` +} + +// +// -- Bulk Import Functions -- +// These functions import data directly into the database. Security and permission checks are bypassed but validity is +// still enforced. +// + +func BulkImport(fileReader io.Reader, dryRun bool) (*model.AppError, int) { + scanner := bufio.NewScanner(fileReader) + lineNumber := 0 + for scanner.Scan() { + decoder := json.NewDecoder(strings.NewReader(scanner.Text())) + lineNumber++ + + var line LineImportData + if err := decoder.Decode(&line); err != nil { + return model.NewLocAppError("BulkImport", "app.import.bulk_import.json_decode.error", nil, err.Error()), lineNumber + } else { + if err := ImportLine(line, dryRun); err != nil { + return err, lineNumber + } + } + } + + if err := scanner.Err(); err != nil { + return model.NewLocAppError("BulkImport", "app.import.bulk_import.file_scan.error", nil, err.Error()), 0 + } + + return nil, 0 +} + +func ImportLine(line LineImportData, dryRun bool) *model.AppError { + switch { + case line.Type == "team": + if line.Team == nil { + return model.NewLocAppError("BulkImport", "app.import.import_line.null_team.error", nil, "") + } else { + return ImportTeam(line.Team, dryRun) + } + case line.Type == "channel": + if line.Channel == nil { + return model.NewLocAppError("BulkImport", "app.import.import_line.null_channel.error", nil, "") + } else { + return ImportChannel(line.Channel, dryRun) + } + default: + return model.NewLocAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "") + } +} + +func ImportTeam(data *TeamImportData, dryRun bool) *model.AppError { + if err := validateTeamImportData(data); err != nil { + return err + } + + // If this is a Dry Run, do not continue any further. + if dryRun { + return nil + } + + var team *model.Team + if result := <-Srv.Store.Team().GetByName(*data.Name); result.Err == nil { + team = result.Data.(*model.Team) + } else { + team = &model.Team{} + } + + team.Name = *data.Name + team.DisplayName = *data.DisplayName + team.Type = *data.Type + + if data.Description != nil { + team.Description = *data.Description + } + + if data.AllowOpenInvite != nil { + team.AllowOpenInvite = *data.AllowOpenInvite + } + + if team.Id == "" { + if _, err := CreateTeam(team); err != nil { + return err + } + } else { + if _, err := UpdateTeam(team); err != nil { + return err + } + } + + return nil +} + +func validateTeamImportData(data *TeamImportData) *model.AppError { + + if data.Name == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_missing.error", nil, "") + } else if len(*data.Name) > model.TEAM_NAME_MAX_LENGTH { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_length.error", nil, "") + } else if model.IsReservedTeamName(*data.Name) { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_reserved.error", nil, "") + } else if !model.IsValidTeamName(*data.Name) { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_characters.error", nil, "") + } + + if data.DisplayName == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.display_name_missing.error", nil, "") + } else if utf8.RuneCountInString(*data.DisplayName) == 0 || utf8.RuneCountInString(*data.DisplayName) > model.TEAM_DISPLAY_NAME_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.display_name_length.error", nil, "") + } + + if data.Type == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.type_missing.error", nil, "") + } else if *data.Type != model.TEAM_OPEN && *data.Type != model.TEAM_INVITE { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.type_invalid.error", nil, "") + } + + if data.Description != nil && len(*data.Description) > model.TEAM_DESCRIPTION_MAX_LENGTH { + return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.description_length.error", nil, "") + } + + return nil +} + +func ImportChannel(data *ChannelImportData, dryRun bool) *model.AppError { + if err := validateChannelImportData(data); err != nil { + return err + } + + // If this is a Dry Run, do not continue any further. + if dryRun { + return nil + } + + var team *model.Team + if result := <-Srv.Store.Team().GetByName(*data.Team); result.Err != nil { + return model.NewLocAppError("BulkImport", "app.import.import_channel.team_not_found.error", map[string]interface{}{"TeamName": *data.Team}, "") + } else { + team = result.Data.(*model.Team) + } + + var channel *model.Channel + if result := <-Srv.Store.Channel().GetByNameIncludeDeleted(team.Id, *data.Name); result.Err == nil { + channel = result.Data.(*model.Channel) + } else { + channel = &model.Channel{} + } + + channel.TeamId = team.Id + channel.Name = *data.Name + channel.DisplayName = *data.DisplayName + channel.Type = *data.Type + + if data.Header != nil { + channel.Header = *data.Header + } + + if data.Purpose != nil { + channel.Purpose = *data.Purpose + } + + if channel.Id == "" { + if _, err := CreateChannel(channel, false); err != nil { + return err + } + } else { + if _, err := UpdateChannel(channel); err != nil { + return err + } + } + + return nil +} + +func validateChannelImportData(data *ChannelImportData) *model.AppError { + + if data.Team == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.team_missing.error", nil, "") + } + + if data.Name == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_missing.error", nil, "") + } else if len(*data.Name) > model.CHANNEL_NAME_MAX_LENGTH { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_length.error", nil, "") + } else if !model.IsValidChannelIdentifier(*data.Name) { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_characters.error", nil, "") + } + + if data.DisplayName == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.display_name_missing.error", nil, "") + } else if utf8.RuneCountInString(*data.DisplayName) == 0 || utf8.RuneCountInString(*data.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.display_name_length.error", nil, "") + } + + if data.Type == nil { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.type_missing.error", nil, "") + } else if *data.Type != model.CHANNEL_OPEN && *data.Type != model.CHANNEL_PRIVATE { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.type_invalid.error", nil, "") + } + + if data.Header != nil && utf8.RuneCountInString(*data.Header) > model.CHANNEL_HEADER_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.header_length.error", nil, "") + } + + if data.Purpose != nil && utf8.RuneCountInString(*data.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES { + return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.purpose_length.error", nil, "") + } + + return nil +} + // +// -- Old SlackImport Functions -- // Import functions are sutible for entering posts and users into the database without // some of the usual checks. (IsValid is still run) // @@ -73,7 +311,7 @@ func ImportUser(team *model.Team, user *model.User) *model.User { } } -func ImportChannel(channel *model.Channel) *model.Channel { +func OldImportChannel(channel *model.Channel) *model.Channel { if result := <-Srv.Store.Channel().Save(channel); result.Err != nil { return nil } else { diff --git a/app/import_test.go b/app/import_test.go new file mode 100644 index 000000000..8f08ff34a --- /dev/null +++ b/app/import_test.go @@ -0,0 +1,552 @@ +// Copyright (c) 2017 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "github.com/mattermost/platform/model" + "strings" + "testing" +) + +func ptrStr(s string) *string { + return &s +} + +func ptrInt64(i int64) *int64 { + return &i +} + +func ptrBool(b bool) *bool { + return &b +} + +func TestImportValidateTeamImportData(t *testing.T) { + + // Test with minimum required valid properties. + data := TeamImportData{ + Name: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Validation failed but should have been valid.") + } + + // Test with various invalid names. + data = TeamImportData{ + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to missing name.") + } + + data.Name = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to too long name.") + } + + data.Name = ptrStr("login") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to reserved word in name.") + } + + data.Name = ptrStr("Test::''ASD") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to non alphanum characters in name.") + } + + data.Name = ptrStr("A") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to short name.") + } + + // Test team various invalid display names. + data = TeamImportData{ + Name: ptrStr("teamname"), + Type: ptrStr("O"), + } + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to missing display_name.") + } + + data.DisplayName = ptrStr("") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to empty display_name.") + } + + data.DisplayName = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to too long display_name.") + } + + // Test with various valid and invalid types. + data = TeamImportData{ + Name: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + } + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to missing type.") + } + + data.Type = ptrStr("A") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to invalid type.") + } + + data.Type = ptrStr("I") + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid type.") + } + + // Test with all the combinations of optional parameters. + data = TeamImportData{ + Name: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + Description: ptrStr("The team description."), + AllowOpenInvite: ptrBool(true), + } + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid optional properties.") + } + + data.AllowOpenInvite = ptrBool(false) + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Should have succeeded with allow open invites false.") + } + + data.Description = ptrStr(strings.Repeat("abcdefghij ", 26)) + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to too long description.") + } +} + +func TestImportValidateChannelImportData(t *testing.T) { + + // Test with minimum required valid properties. + data := ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err != nil { + t.Fatal("Validation failed but should have been valid.") + } + + // Test with missing team. + data = ChannelImportData{ + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing team.") + } + + // Test with various invalid names. + data = ChannelImportData{ + Team: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing name.") + } + + data.Name = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long name.") + } + + data.Name = ptrStr("Test::''ASD") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to non alphanum characters in name.") + } + + data.Name = ptrStr("A") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to short name.") + } + + // Test team various invalid display names. + data = ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing display_name.") + } + + data.DisplayName = ptrStr("") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to empty display_name.") + } + + data.DisplayName = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long display_name.") + } + + // Test with various valid and invalid types. + data = ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing type.") + } + + data.Type = ptrStr("A") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to invalid type.") + } + + data.Type = ptrStr("P") + if err := validateChannelImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid type.") + } + + // Test with all the combinations of optional parameters. + data = ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + Header: ptrStr("Channel Header Here"), + Purpose: ptrStr("Channel Purpose Here"), + } + if err := validateChannelImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid optional properties.") + } + + data.Header = ptrStr(strings.Repeat("abcdefghij ", 103)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long header.") + } + + data.Header = ptrStr("Channel Header Here") + data.Purpose = ptrStr(strings.Repeat("abcdefghij ", 26)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long purpose.") + } +} + +func TestImportImportTeam(t *testing.T) { + _ = Setup() + + // Check how many teams are in the database. + var teamsCount int64 + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + teamsCount = r.Data.(int64) + } else { + t.Fatalf("Failed to get team count.") + } + + data := TeamImportData{ + Name: ptrStr(model.NewId()), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("XXX"), + Description: ptrStr("The team description."), + AllowOpenInvite: ptrBool(true), + } + + // Try importing an invalid team in dryRun mode. + if err := ImportTeam(&data, true); err == nil { + t.Fatalf("Should have received an error importing an invalid team.") + } + + // Do a valid team in dry-run mode. + data.Type = ptrStr("O") + if err := ImportTeam(&data, true); err != nil { + t.Fatalf("Received an error validating valid team.") + } + + // Check that no more teams are in the DB. + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64) != teamsCount { + t.Fatalf("Teams got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Do an invalid team in apply mode, check db changes. + data.Type = ptrStr("XXX") + if err := ImportTeam(&data, false); err == nil { + t.Fatalf("Import should have failed on invalid team.") + } + + // Check that no more teams are in the DB. + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64) != teamsCount { + t.Fatalf("Invalid team got persisted.") + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Do a valid team in apply mode, check db changes. + data.Type = ptrStr("O") + if err := ImportTeam(&data, false); err != nil { + t.Fatalf("Received an error importing valid team.") + } + + // Check that one more team is in the DB. + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64)-1 != teamsCount { + t.Fatalf("Team did not get saved in apply run mode.", r.Data.(int64), teamsCount) + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Get the team and check that all the fields are correct. + if team, err := GetTeamByName(*data.Name); err != nil { + t.Fatalf("Failed to get team from database.") + } else { + if team.DisplayName != *data.DisplayName || team.Type != *data.Type || team.Description != *data.Description || team.AllowOpenInvite != *data.AllowOpenInvite { + t.Fatalf("Imported team properties do not match import data.") + } + } + + // Alter all the fields of that team (apart from unique identifier) and import again. + data.DisplayName = ptrStr("Display Name 2") + data.Type = ptrStr("P") + data.Description = ptrStr("The new description") + data.AllowOpenInvite = ptrBool(false) + + // Check that the original number of teams are again in the DB (because this query doesn't include deleted). + data.Type = ptrStr("O") + if err := ImportTeam(&data, false); err != nil { + t.Fatalf("Received an error importing updated valid team.") + } + + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64)-1 != teamsCount { + t.Fatalf("Team alterations did not get saved in apply run mode.", r.Data.(int64), teamsCount) + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Get the team and check that all fields are correct. + if team, err := GetTeamByName(*data.Name); err != nil { + t.Fatalf("Failed to get team from database.") + } else { + if team.DisplayName != *data.DisplayName || team.Type != *data.Type || team.Description != *data.Description || team.AllowOpenInvite != *data.AllowOpenInvite { + t.Fatalf("Updated team properties do not match import data.") + } + } +} + +func TestImportImportChannel(t *testing.T) { + _ = Setup() + + // Import a Team. + teamName := model.NewId() + ImportTeam(&TeamImportData{ + Name: &teamName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + var team *model.Team + if te, err := GetTeamByName(teamName); err != nil { + t.Fatalf("Failed to get team from database.") + } else { + team = te + } + + // Check how many channels are in the database. + var channelCount int64 + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + channelCount = r.Data.(int64) + } else { + t.Fatalf("Failed to get team count.") + } + + // Do an invalid channel in dry-run mode. + data := ChannelImportData{ + Team: &teamName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + Header: ptrStr("Channe Header"), + Purpose: ptrStr("Channel Purpose"), + } + if err := ImportChannel(&data, true); err == nil { + t.Fatalf("Expected error due to invalid name.") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Channels got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel with a nonexistent team in dry-run mode. + data.Name = ptrStr("channelname") + data.Team = ptrStr(model.NewId()) + if err := ImportChannel(&data, true); err != nil { + t.Fatalf("Expected success as cannot validate channel name in dry run mode.") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Channels got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel in dry-run mode. + data.Team = &teamName + if err := ImportChannel(&data, true); err != nil { + t.Fatalf("Expected success as valid team.") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Channels got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do an invalid channel in apply mode. + data.Name = nil + if err := ImportChannel(&data, false); err == nil { + t.Fatalf("Expected error due to invalid name (apply mode).") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Invalid channel got persisted in apply mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel in apply mode with a nonexistant team. + data.Name = ptrStr("channelname") + data.Team = ptrStr(model.NewId()) + if err := ImportChannel(&data, false); err == nil { + t.Fatalf("Expected error due to non-existant team (apply mode).") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Invalid team channel got persisted in apply mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel in apply mode. + data.Team = &teamName + if err := ImportChannel(&data, false); err != nil { + t.Fatalf("Expected success in apply mode: %v", err.Error()) + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount+1 { + t.Fatalf("Channels did not get persisted in apply mode: found %v expected %v + 1", r.Data.(int64), channelCount) + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Get the Channel and check all the fields are correct. + if channel, err := GetChannelByName(*data.Name, team.Id); err != nil { + t.Fatalf("Failed to get channel from database.") + } else { + if channel.Name != *data.Name || channel.DisplayName != *data.DisplayName || channel.Type != *data.Type || channel.Header != *data.Header || channel.Purpose != *data.Purpose { + t.Fatalf("Imported team properties do not match Import Data.") + } + } + + // Alter all the fields of that channel. + data.DisplayName = ptrStr("Chaned Disp Name") + data.Type = ptrStr(model.CHANNEL_PRIVATE) + data.Header = ptrStr("New Header") + data.Purpose = ptrStr("New Purpose") + if err := ImportChannel(&data, false); err != nil { + t.Fatalf("Expected success in apply mode: %v", err.Error()) + } + + // Check channel count the same. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Updated channel did not get correctly persisted in apply mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Get the Channel and check all the fields are correct. + if channel, err := GetChannelByName(*data.Name, team.Id); err != nil { + t.Fatalf("Failed to get channel from database.") + } else { + if channel.Name != *data.Name || channel.DisplayName != *data.DisplayName || channel.Type != *data.Type || channel.Header != *data.Header || channel.Purpose != *data.Purpose { + t.Fatalf("Updated team properties do not match Import Data.") + } + } + +} + +func TestImportImportLine(t *testing.T) { + _ = Setup() + + // Try import line with an invalid type. + line := LineImportData{ + Type: "gibberish", + } + + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line with invalid type.") + } + + // Try import line with team type but nil team. + line.Type = "team" + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line of type team with a nil team.") + } + + // Try import line with channel type but nil channel. + line.Type = "channel" + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line with type channel with a nil channel.") + } +} + +func TestImportBulkImport(t *testing.T) { + _ = Setup() + + teamName := model.NewId() + channelName := model.NewId() + + // Run bulk import with a valid 1 of everything. + data1 := `{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} +{"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}}` + + if err, line := BulkImport(strings.NewReader(data1), false); err != nil || line != 0 { + t.Fatalf("BulkImport should have succeeded: %v, %v", err.Error(), line) + } + + // Run bulk import using a string that contains a line with invalid json. + data2 := `{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "vinewy665jam3n6oxzhsdgajly"}` + if err, line := BulkImport(strings.NewReader(data2), false); err == nil || line != 1 { + t.Fatalf("Should have failed due to invalid JSON on line 1.") + } +} diff --git a/app/slackimport.go b/app/slackimport.go index 508803126..edeb601e2 100644 --- a/app/slackimport.go +++ b/app/slackimport.go @@ -484,7 +484,7 @@ func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[str if mChannel == nil { // Haven't found an existing channel to merge with. Try importing it as a new one. - mChannel = ImportChannel(&newChannel) + mChannel = OldImportChannel(&newChannel) if mChannel == nil { l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName) log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName})) diff --git a/cmd/platform/import.go b/cmd/platform/import.go index 09b135354..76ed1fb81 100644 --- a/cmd/platform/import.go +++ b/cmd/platform/import.go @@ -6,6 +6,7 @@ import ( "errors" "os" + "fmt" "github.com/mattermost/platform/app" "github.com/spf13/cobra" ) @@ -23,8 +24,19 @@ var slackImportCmd = &cobra.Command{ RunE: slackImportCmdF, } +var bulkImportCmd = &cobra.Command{ + Use: "bulk [file]", + Short: "Import bulk data.", + Long: "Import data from a Mattermost Bulk Import File.", + Example: " import bulk bulk_data.json", + RunE: bulkImportCmdF, +} + func init() { + bulkImportCmd.Flags().Bool("apply", false, "Save the import data to the database. Use with caution - this cannot be reverted.") + importCmd.AddCommand( + bulkImportCmd, slackImportCmd, ) } @@ -60,3 +72,47 @@ func slackImportCmdF(cmd *cobra.Command, args []string) error { return nil } + +func bulkImportCmdF(cmd *cobra.Command, args []string) error { + initDBCommandContextCobra(cmd) + + apply, err := cmd.Flags().GetBool("apply") + if err != nil { + return errors.New("Apply flag error") + } + + if len(args) != 1 { + return errors.New("Incorrect number of arguments.") + } + + fileReader, err := os.Open(args[0]) + if err != nil { + return err + } + defer fileReader.Close() + + if apply { + CommandPrettyPrintln("Running Bulk Import. This may take a long time.") + } else { + CommandPrettyPrintln("Running Bulk Import Data Validation.") + CommandPrettyPrintln("** This checks the validity of the entities in the data file, but does not persist any changes **") + CommandPrettyPrintln("Use the --apply flag to perform the actual data import.") + } + + CommandPrettyPrintln("") + + if err, lineNumber := app.BulkImport(fileReader, !apply); err != nil { + CommandPrettyPrintln(err.Error()) + if lineNumber != 0 { + CommandPrettyPrintln(fmt.Sprintf("Error occurred on data file line %v", lineNumber)) + } + } + + if apply { + CommandPrettyPrintln("Finished Bulk Import.") + } else { + CommandPrettyPrintln("Validation complete. You can now perform the import by rerunning this command with the --apply flag.") + } + + return nil +} diff --git a/i18n/en.json b/i18n/en.json index 6cc82bf37..355fc9945 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -2779,6 +2779,118 @@ "id": "app.channel.post_update_channel_purpose_message.updated_to", "translation": "%s updated the channel purpose to: %s" }, + { + "id": "app.import.bulk_import.json_decode.error", + "translation": "JSON decode of line failed." + }, + { + "id": "app.import.bulk_import.file_scan.error", + "translation": "Error reading import data file." + }, + { + "id": "app.import.import_line.null_team.error", + "translation": "Import data line has type \"team\" but the team object is null." + }, + { + "id": "app.import.import_line.null_channel.error", + "translation": "Import data line has type \"channel\" but the channel object is null." + }, + { + "id": "app.import.import_line.unknown_line_type.error", + "translation": "Import data line has unknown type \"{{.Type}}\"." + }, + { + "id": "app.import.validate_team_import_data.name_missing.error", + "translation": "Missing required team property: name." + }, + { + "id": "app.import.validate_team_import_data.name_length.error", + "translation": "Team name is too long." + }, + { + "id": "app.import.validate_team_import_data.name_reserved.error", + "translation": "Team name contains reserved word." + }, + { + "id": "app.import.validate_team_import_data.name_characters.error", + "translation": "Team name contains invalid characters." + }, + { + "id": "app.import.validate_team_import_data.display_name_missing.error", + "translation": "Missing required team property: display_name" + }, + { + "id": "app.import.validate_team_import_data.display_name_length.error", + "translation": "Team display_name is not within permitted length constraints." + }, + { + "id": "app.import.validate_team_import_data.type_missing.error", + "translation": "Missing required team property: type" + }, + { + "id": "app.import.validate_team_import_data.type_invalid.error", + "translation": "Team type is not valid." + }, + { + "id": "app.import.validate_team_import_data.create_at_zero.error", + "translation": "Team create_at must not be 0 if provided." + }, + { + "id": "app.import.validate_team_import_data.description_length.error", + "translation": "Team description is too long." + }, + { + "id": "app.import.validate_team_import_data.allowed_domains_length.error", + "translation": "Team allowed_domains is too long." + }, + { + "id": "app.import.import_channel.team_not_found.error", + "translation": "Error importing channel. Team with name \"{{.TeamName}}\" could not be found." + }, + { + "id": "app.import.validate_channel_import_data.team_missing.error", + "translation": "Missing required channel property: team" + }, + { + "id": "app.import.validate_channel_import_data.name_missing.error", + "translation": "Missing required channel property: name" + }, + { + "id": "app.import.validate_channel_import_data.name_length.error", + "translation": "Channel name is too long." + }, + { + "id": "app.import.validate_channel_import_data.name_characters.error", + "translation": "Channel name contains invalid characters." + }, + { + "id": "app.import.validate_channel_import_data.display_name_missing.error", + "translation": "Missing required channel property: display_name." + }, + { + "id": "app.import.validate_channel_import_data.display_name_length.error", + "translation": "Channel display_name is not within permitted length constraints." + }, + { + "id": "app.import.validate_channel_import_data.type_missing.error", + "translation": "Missing required channel property: type." + }, + { + "id": "app.import.validate_channel_import_data.type_invalid.error", + "translation": "Channel type is invalid." + }, + { + "id": "app.import.validate_channel_import_data.create_at_zero.error", + "translation": "Channel create_at must not be 0 if provided." + }, + { + "id": "app.import.validate_channel_import_data.header_length.error", + "translation": "Channel header is too long." + }, + { + "id": "app.import.validate_channel_import_data.purpose_length.error", + "translation": "Channel purpose is too long." + }, { "id": "authentication.permissions.create_team_roles.description", "translation": "Ability to create new teams" diff --git a/model/team.go b/model/team.go index 195bac571..873775ef8 100644 --- a/model/team.go +++ b/model/team.go @@ -13,8 +13,14 @@ import ( ) const ( - TEAM_OPEN = "O" - TEAM_INVITE = "I" + TEAM_OPEN = "O" + TEAM_INVITE = "I" + TEAM_ALLOWED_DOMAINS_MAX_LENGTH = 500 + TEAM_COMPANY_NAME_MAX_LENGTH = 64 + TEAM_DESCRIPTION_MAX_LENGTH = 255 + TEAM_DISPLAY_NAME_MAX_RUNES = 64 + TEAM_EMAIL_MAX_LENGTH = 128 + TEAM_NAME_MAX_LENGTH = 64 ) type Team struct { @@ -123,7 +129,7 @@ func (o *Team) IsValid() *AppError { return NewLocAppError("Team.IsValid", "model.team.is_valid.update_at.app_error", nil, "id="+o.Id) } - if len(o.Email) > 128 { + if len(o.Email) > TEAM_EMAIL_MAX_LENGTH { return NewLocAppError("Team.IsValid", "model.team.is_valid.email.app_error", nil, "id="+o.Id) } @@ -131,15 +137,15 @@ func (o *Team) IsValid() *AppError { return NewLocAppError("Team.IsValid", "model.team.is_valid.email.app_error", nil, "id="+o.Id) } - if utf8.RuneCountInString(o.DisplayName) == 0 || utf8.RuneCountInString(o.DisplayName) > 64 { + if utf8.RuneCountInString(o.DisplayName) == 0 || utf8.RuneCountInString(o.DisplayName) > TEAM_DISPLAY_NAME_MAX_RUNES { return NewLocAppError("Team.IsValid", "model.team.is_valid.name.app_error", nil, "id="+o.Id) } - if len(o.Name) > 64 { + if len(o.Name) > TEAM_NAME_MAX_LENGTH { return NewLocAppError("Team.IsValid", "model.team.is_valid.url.app_error", nil, "id="+o.Id) } - if len(o.Description) > 255 { + if len(o.Description) > TEAM_DESCRIPTION_MAX_LENGTH { return NewLocAppError("Team.IsValid", "model.team.is_valid.description.app_error", nil, "id="+o.Id) } @@ -155,11 +161,11 @@ func (o *Team) IsValid() *AppError { return NewLocAppError("Team.IsValid", "model.team.is_valid.type.app_error", nil, "id="+o.Id) } - if len(o.CompanyName) > 64 { + if len(o.CompanyName) > TEAM_COMPANY_NAME_MAX_LENGTH { return NewLocAppError("Team.IsValid", "model.team.is_valid.company.app_error", nil, "id="+o.Id) } - if len(o.AllowedDomains) > 500 { + if len(o.AllowedDomains) > TEAM_ALLOWED_DOMAINS_MAX_LENGTH { return NewLocAppError("Team.IsValid", "model.team.is_valid.domains.app_error", nil, "id="+o.Id) } -- cgit v1.2.3-1-g7c22