From e07e9937e0e0ae4fcb2a51553238a7566d1e4c67 Mon Sep 17 00:00:00 2001 From: George Goldberg Date: Fri, 27 Jan 2017 15:14:54 +0000 Subject: PLT-5366, PLT-5364, PLT-5363: Bulk Import Part 1. (#5204) This commit provides the first part of the bulk import system. The CLI command is provided, complete with validation & apply modes. All the basic properties of Teams and Channels can be imported. Users & Posts will follow separately in a future commit. --- app/import_test.go | 552 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 552 insertions(+) create mode 100644 app/import_test.go (limited to 'app/import_test.go') diff --git a/app/import_test.go b/app/import_test.go new file mode 100644 index 000000000..8f08ff34a --- /dev/null +++ b/app/import_test.go @@ -0,0 +1,552 @@ +// Copyright (c) 2017 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "github.com/mattermost/platform/model" + "strings" + "testing" +) + +func ptrStr(s string) *string { + return &s +} + +func ptrInt64(i int64) *int64 { + return &i +} + +func ptrBool(b bool) *bool { + return &b +} + +func TestImportValidateTeamImportData(t *testing.T) { + + // Test with minimum required valid properties. + data := TeamImportData{ + Name: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Validation failed but should have been valid.") + } + + // Test with various invalid names. + data = TeamImportData{ + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to missing name.") + } + + data.Name = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to too long name.") + } + + data.Name = ptrStr("login") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to reserved word in name.") + } + + data.Name = ptrStr("Test::''ASD") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to non alphanum characters in name.") + } + + data.Name = ptrStr("A") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to short name.") + } + + // Test team various invalid display names. + data = TeamImportData{ + Name: ptrStr("teamname"), + Type: ptrStr("O"), + } + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to missing display_name.") + } + + data.DisplayName = ptrStr("") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to empty display_name.") + } + + data.DisplayName = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to too long display_name.") + } + + // Test with various valid and invalid types. + data = TeamImportData{ + Name: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + } + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to missing type.") + } + + data.Type = ptrStr("A") + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to invalid type.") + } + + data.Type = ptrStr("I") + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid type.") + } + + // Test with all the combinations of optional parameters. + data = TeamImportData{ + Name: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + Description: ptrStr("The team description."), + AllowOpenInvite: ptrBool(true), + } + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid optional properties.") + } + + data.AllowOpenInvite = ptrBool(false) + if err := validateTeamImportData(&data); err != nil { + t.Fatal("Should have succeeded with allow open invites false.") + } + + data.Description = ptrStr(strings.Repeat("abcdefghij ", 26)) + if err := validateTeamImportData(&data); err == nil { + t.Fatal("Should have failed due to too long description.") + } +} + +func TestImportValidateChannelImportData(t *testing.T) { + + // Test with minimum required valid properties. + data := ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err != nil { + t.Fatal("Validation failed but should have been valid.") + } + + // Test with missing team. + data = ChannelImportData{ + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing team.") + } + + // Test with various invalid names. + data = ChannelImportData{ + Team: ptrStr("teamname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing name.") + } + + data.Name = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long name.") + } + + data.Name = ptrStr("Test::''ASD") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to non alphanum characters in name.") + } + + data.Name = ptrStr("A") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to short name.") + } + + // Test team various invalid display names. + data = ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + Type: ptrStr("O"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing display_name.") + } + + data.DisplayName = ptrStr("") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to empty display_name.") + } + + data.DisplayName = ptrStr(strings.Repeat("abcdefghij", 7)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long display_name.") + } + + // Test with various valid and invalid types. + data = ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + } + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to missing type.") + } + + data.Type = ptrStr("A") + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to invalid type.") + } + + data.Type = ptrStr("P") + if err := validateChannelImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid type.") + } + + // Test with all the combinations of optional parameters. + data = ChannelImportData{ + Team: ptrStr("teamname"), + Name: ptrStr("channelname"), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + Header: ptrStr("Channel Header Here"), + Purpose: ptrStr("Channel Purpose Here"), + } + if err := validateChannelImportData(&data); err != nil { + t.Fatal("Should have succeeded with valid optional properties.") + } + + data.Header = ptrStr(strings.Repeat("abcdefghij ", 103)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long header.") + } + + data.Header = ptrStr("Channel Header Here") + data.Purpose = ptrStr(strings.Repeat("abcdefghij ", 26)) + if err := validateChannelImportData(&data); err == nil { + t.Fatal("Should have failed due to too long purpose.") + } +} + +func TestImportImportTeam(t *testing.T) { + _ = Setup() + + // Check how many teams are in the database. + var teamsCount int64 + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + teamsCount = r.Data.(int64) + } else { + t.Fatalf("Failed to get team count.") + } + + data := TeamImportData{ + Name: ptrStr(model.NewId()), + DisplayName: ptrStr("Display Name"), + Type: ptrStr("XXX"), + Description: ptrStr("The team description."), + AllowOpenInvite: ptrBool(true), + } + + // Try importing an invalid team in dryRun mode. + if err := ImportTeam(&data, true); err == nil { + t.Fatalf("Should have received an error importing an invalid team.") + } + + // Do a valid team in dry-run mode. + data.Type = ptrStr("O") + if err := ImportTeam(&data, true); err != nil { + t.Fatalf("Received an error validating valid team.") + } + + // Check that no more teams are in the DB. + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64) != teamsCount { + t.Fatalf("Teams got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Do an invalid team in apply mode, check db changes. + data.Type = ptrStr("XXX") + if err := ImportTeam(&data, false); err == nil { + t.Fatalf("Import should have failed on invalid team.") + } + + // Check that no more teams are in the DB. + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64) != teamsCount { + t.Fatalf("Invalid team got persisted.") + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Do a valid team in apply mode, check db changes. + data.Type = ptrStr("O") + if err := ImportTeam(&data, false); err != nil { + t.Fatalf("Received an error importing valid team.") + } + + // Check that one more team is in the DB. + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64)-1 != teamsCount { + t.Fatalf("Team did not get saved in apply run mode.", r.Data.(int64), teamsCount) + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Get the team and check that all the fields are correct. + if team, err := GetTeamByName(*data.Name); err != nil { + t.Fatalf("Failed to get team from database.") + } else { + if team.DisplayName != *data.DisplayName || team.Type != *data.Type || team.Description != *data.Description || team.AllowOpenInvite != *data.AllowOpenInvite { + t.Fatalf("Imported team properties do not match import data.") + } + } + + // Alter all the fields of that team (apart from unique identifier) and import again. + data.DisplayName = ptrStr("Display Name 2") + data.Type = ptrStr("P") + data.Description = ptrStr("The new description") + data.AllowOpenInvite = ptrBool(false) + + // Check that the original number of teams are again in the DB (because this query doesn't include deleted). + data.Type = ptrStr("O") + if err := ImportTeam(&data, false); err != nil { + t.Fatalf("Received an error importing updated valid team.") + } + + if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil { + if r.Data.(int64)-1 != teamsCount { + t.Fatalf("Team alterations did not get saved in apply run mode.", r.Data.(int64), teamsCount) + } + } else { + t.Fatalf("Failed to get team count.") + } + + // Get the team and check that all fields are correct. + if team, err := GetTeamByName(*data.Name); err != nil { + t.Fatalf("Failed to get team from database.") + } else { + if team.DisplayName != *data.DisplayName || team.Type != *data.Type || team.Description != *data.Description || team.AllowOpenInvite != *data.AllowOpenInvite { + t.Fatalf("Updated team properties do not match import data.") + } + } +} + +func TestImportImportChannel(t *testing.T) { + _ = Setup() + + // Import a Team. + teamName := model.NewId() + ImportTeam(&TeamImportData{ + Name: &teamName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + }, false) + var team *model.Team + if te, err := GetTeamByName(teamName); err != nil { + t.Fatalf("Failed to get team from database.") + } else { + team = te + } + + // Check how many channels are in the database. + var channelCount int64 + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + channelCount = r.Data.(int64) + } else { + t.Fatalf("Failed to get team count.") + } + + // Do an invalid channel in dry-run mode. + data := ChannelImportData{ + Team: &teamName, + DisplayName: ptrStr("Display Name"), + Type: ptrStr("O"), + Header: ptrStr("Channe Header"), + Purpose: ptrStr("Channel Purpose"), + } + if err := ImportChannel(&data, true); err == nil { + t.Fatalf("Expected error due to invalid name.") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Channels got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel with a nonexistent team in dry-run mode. + data.Name = ptrStr("channelname") + data.Team = ptrStr(model.NewId()) + if err := ImportChannel(&data, true); err != nil { + t.Fatalf("Expected success as cannot validate channel name in dry run mode.") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Channels got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel in dry-run mode. + data.Team = &teamName + if err := ImportChannel(&data, true); err != nil { + t.Fatalf("Expected success as valid team.") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Channels got persisted in dry run mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do an invalid channel in apply mode. + data.Name = nil + if err := ImportChannel(&data, false); err == nil { + t.Fatalf("Expected error due to invalid name (apply mode).") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Invalid channel got persisted in apply mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel in apply mode with a nonexistant team. + data.Name = ptrStr("channelname") + data.Team = ptrStr(model.NewId()) + if err := ImportChannel(&data, false); err == nil { + t.Fatalf("Expected error due to non-existant team (apply mode).") + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Invalid team channel got persisted in apply mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Do a valid channel in apply mode. + data.Team = &teamName + if err := ImportChannel(&data, false); err != nil { + t.Fatalf("Expected success in apply mode: %v", err.Error()) + } + + // Check that no more channels are in the DB. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount+1 { + t.Fatalf("Channels did not get persisted in apply mode: found %v expected %v + 1", r.Data.(int64), channelCount) + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Get the Channel and check all the fields are correct. + if channel, err := GetChannelByName(*data.Name, team.Id); err != nil { + t.Fatalf("Failed to get channel from database.") + } else { + if channel.Name != *data.Name || channel.DisplayName != *data.DisplayName || channel.Type != *data.Type || channel.Header != *data.Header || channel.Purpose != *data.Purpose { + t.Fatalf("Imported team properties do not match Import Data.") + } + } + + // Alter all the fields of that channel. + data.DisplayName = ptrStr("Chaned Disp Name") + data.Type = ptrStr(model.CHANNEL_PRIVATE) + data.Header = ptrStr("New Header") + data.Purpose = ptrStr("New Purpose") + if err := ImportChannel(&data, false); err != nil { + t.Fatalf("Expected success in apply mode: %v", err.Error()) + } + + // Check channel count the same. + if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil { + if r.Data.(int64) != channelCount { + t.Fatalf("Updated channel did not get correctly persisted in apply mode.") + } + } else { + t.Fatalf("Failed to get channel count.") + } + + // Get the Channel and check all the fields are correct. + if channel, err := GetChannelByName(*data.Name, team.Id); err != nil { + t.Fatalf("Failed to get channel from database.") + } else { + if channel.Name != *data.Name || channel.DisplayName != *data.DisplayName || channel.Type != *data.Type || channel.Header != *data.Header || channel.Purpose != *data.Purpose { + t.Fatalf("Updated team properties do not match Import Data.") + } + } + +} + +func TestImportImportLine(t *testing.T) { + _ = Setup() + + // Try import line with an invalid type. + line := LineImportData{ + Type: "gibberish", + } + + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line with invalid type.") + } + + // Try import line with team type but nil team. + line.Type = "team" + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line of type team with a nil team.") + } + + // Try import line with channel type but nil channel. + line.Type = "channel" + if err := ImportLine(line, false); err == nil { + t.Fatalf("Expected an error when importing a line with type channel with a nil channel.") + } +} + +func TestImportBulkImport(t *testing.T) { + _ = Setup() + + teamName := model.NewId() + channelName := model.NewId() + + // Run bulk import with a valid 1 of everything. + data1 := `{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} +{"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}}` + + if err, line := BulkImport(strings.NewReader(data1), false); err != nil || line != 0 { + t.Fatalf("BulkImport should have succeeded: %v, %v", err.Error(), line) + } + + // Run bulk import using a string that contains a line with invalid json. + data2 := `{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "vinewy665jam3n6oxzhsdgajly"}` + if err, line := BulkImport(strings.NewReader(data2), false); err == nil || line != 1 { + t.Fatalf("Should have failed due to invalid JSON on line 1.") + } +} -- cgit v1.2.3-1-g7c22