Improve test coverage reporting / accuracy (#7819)

* improve test coverage reporting / accuracy

* handle absolute coverpaths

* move tests into multiple files

* rename codecov.yml (https://github.com/codecov/support/issues/426)

7 files changed, 341 insertions, 287 deletions

diff --git a/cmd/platform/channel_test.go b/cmd/platform/channel_test.go
new file mode 100644
index 000000000..1e6915679
--- /dev/null
+++ b/cmd/platform/channel_test.go
@@ -0,0 +1,89 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/api"
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestJoinChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+
+	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// Joining twice should succeed
+	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// should fail because channel does not exist
+	require.Error(t, runCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name+"asdf", th.BasicUser2.Email))
+}
+
+func TestRemoveChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+
+	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// should fail because channel does not exist
+	require.Error(t, runCommand(t, "channel", "remove", th.BasicTeam.Name+":doesnotexist", th.BasicUser2.Email))
+
+	checkCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// Leaving twice should succeed
+	checkCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+}
+
+func TestListChannels(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
+
+	output := checkCommand(t, "channel", "list", th.BasicTeam.Name)
+
+	if !strings.Contains(string(output), "town-square") {
+		t.Fatal("should have channels")
+	}
+
+	if !strings.Contains(string(output), channel.Name+" (archived)") {
+		t.Fatal("should have archived channel")
+	}
+}
+
+func TestRestoreChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
+
+	checkCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
+
+	// restoring twice should succeed
+	checkCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
+}
+
+func TestCreateChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	id := model.NewId()
+	name := "name" + id
+
+	checkCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--name", name)
+
+	name = name + "-private"
+	checkCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--private", "--name", name)
+}
diff --git a/cmd/platform/cli_test.go b/cmd/platform/cli_test.go
deleted file mode 100644
index 7d6717660..000000000
--- a/cmd/platform/cli_test.go
+++ /dev/null
@@ -1,287 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package main
-
-import (
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/mattermost/mattermost-server/api"
-	"github.com/mattermost/mattermost-server/model"
-)
-
-var testExePath string
-
-func checkCommand(t *testing.T, args ...string) string {
-	output, err := exec.Command(testExePath, args...).CombinedOutput()
-	require.NoError(t, err, string(output))
-	return string(output)
-}
-
-func TestCliVersion(t *testing.T) {
-	checkCommand(t, "version")
-}
-
-func TestCliCreateTeam(t *testing.T) {
-	th := api.Setup().InitSystemAdmin()
-	defer th.TearDown()
-
-	id := model.NewId()
-	name := "name" + id
-	displayName := "Name " + id
-
-	checkCommand(t, "team", "create", "--name", name, "--display_name", displayName)
-
-	found := th.SystemAdminClient.Must(th.SystemAdminClient.FindTeamByName(name)).Data.(bool)
-
-	if !found {
-		t.Fatal("Failed to create Team")
-	}
-}
-
-func TestCliCreateUserWithTeam(t *testing.T) {
-	th := api.Setup().InitSystemAdmin()
-	defer th.TearDown()
-
-	id := model.NewId()
-	email := "success+" + id + "@simulator.amazonses.com"
-	username := "name" + id
-
-	checkCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
-
-	checkCommand(t, "team", "add", th.SystemAdminTeam.Id, email)
-
-	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == email {
-			found = true
-		}
-
-	}
-
-	if !found {
-		t.Fatal("Failed to create User")
-	}
-}
-
-func TestCliCreateUserWithoutTeam(t *testing.T) {
-	th := api.Setup()
-	defer th.TearDown()
-
-	id := model.NewId()
-	email := "success+" + id + "@simulator.amazonses.com"
-	username := "name" + id
-
-	checkCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
-
-	if result := <-th.App.Srv.Store.User().GetByEmail(email); result.Err != nil {
-		t.Fatal()
-	} else {
-		user := result.Data.(*model.User)
-		if user.Email != email {
-			t.Fatal()
-		}
-	}
-}
-
-func TestCliAssignRole(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	checkCommand(t, "roles", "system_admin", th.BasicUser.Email)
-
-	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err != nil {
-		t.Fatal()
-	} else {
-		user := result.Data.(*model.User)
-		if user.Roles != "system_admin system_user" {
-			t.Fatal()
-		}
-	}
-}
-
-func TestCliJoinChannel(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-
-	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
-
-	// Joining twice should succeed
-	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
-
-	// should fail because channel does not exist
-	require.Error(t, exec.Command(testExePath, "channel", "add", th.BasicTeam.Name+":"+channel.Name+"asdf", th.BasicUser2.Email).Run())
-}
-
-func TestCliRemoveChannel(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-
-	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
-
-	// should fail because channel does not exist
-	require.Error(t, exec.Command(testExePath, "channel", "remove", th.BasicTeam.Name+":doesnotexist", th.BasicUser2.Email).Run())
-
-	checkCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
-
-	// Leaving twice should succeed
-	checkCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
-}
-
-func TestCliListChannels(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
-
-	output := checkCommand(t, "channel", "list", th.BasicTeam.Name)
-
-	if !strings.Contains(string(output), "town-square") {
-		t.Fatal("should have channels")
-	}
-
-	if !strings.Contains(string(output), channel.Name+" (archived)") {
-		t.Fatal("should have archived channel")
-	}
-}
-
-func TestCliRestoreChannel(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
-
-	checkCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
-
-	// restoring twice should succeed
-	checkCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
-}
-
-func TestCliJoinTeam(t *testing.T) {
-	th := api.Setup().InitSystemAdmin().InitBasic()
-	defer th.TearDown()
-
-	checkCommand(t, "team", "add", th.SystemAdminTeam.Name, th.BasicUser.Email)
-
-	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == th.BasicUser.Email {
-			found = true
-		}
-
-	}
-
-	if !found {
-		t.Fatal("Failed to create User")
-	}
-}
-
-func TestCliLeaveTeam(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	checkCommand(t, "team", "remove", th.BasicTeam.Name, th.BasicUser.Email)
-
-	profiles := th.BasicClient.Must(th.BasicClient.GetProfilesInTeam(th.BasicTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == th.BasicUser.Email {
-			found = true
-		}
-
-	}
-
-	if found {
-		t.Fatal("profile should not be on team")
-	}
-
-	if result := <-th.App.Srv.Store.Team().GetTeamsByUserId(th.BasicUser.Id); result.Err != nil {
-		teamMembers := result.Data.([]*model.TeamMember)
-		if len(teamMembers) > 0 {
-			t.Fatal("Shouldn't be in team")
-		}
-	}
-}
-
-func TestCliResetPassword(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	checkCommand(t, "user", "password", th.BasicUser.Email, "password2")
-
-	th.BasicClient.Logout()
-	th.BasicUser.Password = "password2"
-	th.LoginBasic()
-}
-
-func TestCliCreateChannel(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	id := model.NewId()
-	name := "name" + id
-
-	checkCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--name", name)
-
-	name = name + "-private"
-	checkCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--private", "--name", name)
-}
-
-func TestCliMakeUserActiveAndInactive(t *testing.T) {
-	th := api.Setup().InitBasic()
-	defer th.TearDown()
-
-	// first inactivate the user
-	checkCommand(t, "user", "deactivate", th.BasicUser.Email)
-
-	// activate the inactive user
-	checkCommand(t, "user", "activate", th.BasicUser.Email)
-}
-
-func TestMain(m *testing.M) {
-	dir, err := ioutil.TempDir("", "cli_test")
-	if err != nil {
-		panic(err)
-	}
-	defer os.RemoveAll(dir)
-
-	testExePath = filepath.Join(dir, "cli")
-	files, err := filepath.Glob("./*.go")
-	if err != nil {
-		panic(err)
-	}
-
-	cmd := exec.Command("go", append([]string{"build", "-o", testExePath}, files...)...)
-	cmd.Stderr = os.Stderr
-	cmd.Stdout = os.Stdout
-	if err := cmd.Run(); err != nil {
-		panic(err)
-	}
-
-	status := 0
-	defer func() {
-		os.Exit(status)
-	}()
-	status = m.Run()
-}
diff --git a/cmd/platform/platform_test.go b/cmd/platform/platform_test.go
new file mode 100644
index 000000000..8a2839c7e
--- /dev/null
+++ b/cmd/platform/platform_test.go
@@ -0,0 +1,53 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+var coverprofileCounters map[string]int = make(map[string]int)
+
+func execArgs(t *testing.T, args []string) []string {
+	ret := []string{"-test.run", "ExecCommand"}
+	if coverprofile := flag.Lookup("test.coverprofile").Value.String(); coverprofile != "" {
+		dir := filepath.Dir(coverprofile)
+		base := filepath.Base(coverprofile)
+		baseParts := strings.SplitN(base, ".", 2)
+		coverprofileCounters[t.Name()] = coverprofileCounters[t.Name()] + 1
+		baseParts[0] = fmt.Sprintf("%v-%v-%v", baseParts[0], t.Name(), coverprofileCounters[t.Name()])
+		ret = append(ret, "-test.coverprofile", filepath.Join(dir, strings.Join(baseParts, ".")))
+	}
+	return append(append(ret, "--"), args...)
+}
+
+func checkCommand(t *testing.T, args ...string) string {
+	path, err := os.Executable()
+	require.NoError(t, err)
+	output, err := exec.Command(path, execArgs(t, args)...).CombinedOutput()
+	require.NoError(t, err, string(output))
+	return string(output)
+}
+
+func runCommand(t *testing.T, args ...string) error {
+	path, err := os.Executable()
+	require.NoError(t, err)
+	return exec.Command(path, execArgs(t, args)...).Run()
+}
+
+func TestExecCommand(t *testing.T) {
+	if filter := flag.Lookup("test.run").Value.String(); filter != "ExecCommand" {
+		t.Skip("use -run ExecCommand to execute a command via the test executable")
+	}
+	rootCmd.SetArgs(flag.Args())
+	require.NoError(t, rootCmd.Execute())
+}
diff --git a/cmd/platform/roles_test.go b/cmd/platform/roles_test.go
new file mode 100644
index 000000000..1a5ae5173
--- /dev/null
+++ b/cmd/platform/roles_test.go
@@ -0,0 +1,27 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/api"
+	"github.com/mattermost/mattermost-server/model"
+)
+
+func TestAssignRole(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "roles", "system_admin", th.BasicUser.Email)
+
+	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err != nil {
+		t.Fatal()
+	} else {
+		user := result.Data.(*model.User)
+		if user.Roles != "system_admin system_user" {
+			t.Fatal()
+		}
+	}
+}
diff --git a/cmd/platform/team_test.go b/cmd/platform/team_test.go
new file mode 100644
index 000000000..1e13d6cfa
--- /dev/null
+++ b/cmd/platform/team_test.go
@@ -0,0 +1,79 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/api"
+	"github.com/mattermost/mattermost-server/model"
+)
+
+func TestCreateTeam(t *testing.T) {
+	th := api.Setup().InitSystemAdmin()
+	defer th.TearDown()
+
+	id := model.NewId()
+	name := "name" + id
+	displayName := "Name " + id
+
+	checkCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+
+	found := th.SystemAdminClient.Must(th.SystemAdminClient.FindTeamByName(name)).Data.(bool)
+
+	if !found {
+		t.Fatal("Failed to create Team")
+	}
+}
+
+func TestJoinTeam(t *testing.T) {
+	th := api.Setup().InitSystemAdmin().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "team", "add", th.SystemAdminTeam.Name, th.BasicUser.Email)
+
+	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
+
+	found := false
+
+	for _, user := range profiles {
+		if user.Email == th.BasicUser.Email {
+			found = true
+		}
+
+	}
+
+	if !found {
+		t.Fatal("Failed to create User")
+	}
+}
+
+func TestLeaveTeam(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "team", "remove", th.BasicTeam.Name, th.BasicUser.Email)
+
+	profiles := th.BasicClient.Must(th.BasicClient.GetProfilesInTeam(th.BasicTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
+
+	found := false
+
+	for _, user := range profiles {
+		if user.Email == th.BasicUser.Email {
+			found = true
+		}
+
+	}
+
+	if found {
+		t.Fatal("profile should not be on team")
+	}
+
+	if result := <-th.App.Srv.Store.Team().GetTeamsByUserId(th.BasicUser.Id); result.Err != nil {
+		teamMembers := result.Data.([]*model.TeamMember)
+		if len(teamMembers) > 0 {
+			t.Fatal("Shouldn't be in team")
+		}
+	}
+}
diff --git a/cmd/platform/user_test.go b/cmd/platform/user_test.go
new file mode 100644
index 000000000..5383ad914
--- /dev/null
+++ b/cmd/platform/user_test.go
@@ -0,0 +1,81 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/api"
+	"github.com/mattermost/mattermost-server/model"
+)
+
+func TestCreateUserWithTeam(t *testing.T) {
+	th := api.Setup().InitSystemAdmin()
+	defer th.TearDown()
+
+	id := model.NewId()
+	email := "success+" + id + "@simulator.amazonses.com"
+	username := "name" + id
+
+	checkCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
+
+	checkCommand(t, "team", "add", th.SystemAdminTeam.Id, email)
+
+	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
+
+	found := false
+
+	for _, user := range profiles {
+		if user.Email == email {
+			found = true
+		}
+
+	}
+
+	if !found {
+		t.Fatal("Failed to create User")
+	}
+}
+
+func TestCreateUserWithoutTeam(t *testing.T) {
+	th := api.Setup()
+	defer th.TearDown()
+
+	id := model.NewId()
+	email := "success+" + id + "@simulator.amazonses.com"
+	username := "name" + id
+
+	checkCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
+
+	if result := <-th.App.Srv.Store.User().GetByEmail(email); result.Err != nil {
+		t.Fatal()
+	} else {
+		user := result.Data.(*model.User)
+		if user.Email != email {
+			t.Fatal()
+		}
+	}
+}
+
+func TestResetPassword(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "user", "password", th.BasicUser.Email, "password2")
+
+	th.BasicClient.Logout()
+	th.BasicUser.Password = "password2"
+	th.LoginBasic()
+}
+
+func TestMakeUserActiveAndInactive(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	// first inactivate the user
+	checkCommand(t, "user", "deactivate", th.BasicUser.Email)
+
+	// activate the inactive user
+	checkCommand(t, "user", "activate", th.BasicUser.Email)
+}
diff --git a/cmd/platform/version_test.go b/cmd/platform/version_test.go
new file mode 100644
index 000000000..eea2549ee
--- /dev/null
+++ b/cmd/platform/version_test.go
@@ -0,0 +1,12 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"testing"
+)
+
+func TestVersion(t *testing.T) {
+	checkCommand(t, "version")
+}