Use tmpfs containers for api/api4 tests, move and speed up CLI tests (#7606)

* use tmpfs containers for api/api4, move and speed up cli tests

* minor optimizations

* add missing files, fix pre-existing race condition

* add . to TestMain check

* add requested log message

3 files changed, 103 insertions, 419 deletions

diff --git a/api/api_test.go b/api/api_test.go
new file mode 100644
index 000000000..3d4f5f156
--- /dev/null
+++ b/api/api_test.go
@@ -0,0 +1,48 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package api
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	l4g "github.com/alecthomas/log4go"
+
+	"github.com/mattermost/mattermost-server/store/storetest"
+	"github.com/mattermost/mattermost-server/utils"
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// In the case where a dev just wants to run a single test, it's faster to just use the default
+	// store.
+	if filter := flag.Lookup("test.run").Value.String(); filter != "" && filter != "." {
+		utils.TranslationsPreInit()
+		utils.LoadConfig("config.json")
+		l4g.Info("-test.run used, not creating temporary containers")
+		os.Exit(m.Run())
+	}
+
+	utils.TranslationsPreInit()
+	utils.LoadConfig("config.json")
+	utils.InitTranslations(utils.Cfg.LocalizationSettings)
+
+	status := 0
+
+	container, settings, err := storetest.NewMySQLContainer()
+	if err != nil {
+		panic(err)
+	}
+
+	UseTestStore(container, settings)
+
+	defer func() {
+		StopTestStore()
+		os.Exit(status)
+	}()
+
+	status = m.Run()
+}
diff --git a/api/apitestlib.go b/api/apitestlib.go
index a8e70fe42..48637ee59 100644
--- a/api/apitestlib.go
+++ b/api/apitestlib.go
@@ -4,12 +4,15 @@
 package api
 
 import (
+	"net"
 	"time"
 
 	"github.com/mattermost/mattermost-server/api4"
 	"github.com/mattermost/mattermost-server/app"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
+	"github.com/mattermost/mattermost-server/store/sqlstore"
+	"github.com/mattermost/mattermost-server/store/storetest"
 	"github.com/mattermost/mattermost-server/utils"
 	"github.com/mattermost/mattermost-server/wsapi"
 
@@ -33,13 +36,43 @@ type TestHelper struct {
 	SystemAdminChannel *model.Channel
 }
 
+type persistentTestStore struct {
+	store.Store
+}
+
+func (*persistentTestStore) Close() {}
+
+var testStoreContainer *storetest.RunningContainer
+var testStore *persistentTestStore
+
+// UseTestStore sets the container and corresponding settings to use for tests. Once the tests are
+// complete (e.g. at the end of your TestMain implementation), you should call StopTestStore.
+func UseTestStore(container *storetest.RunningContainer, settings *model.SqlSettings) {
+	testStoreContainer = container
+	testStore = &persistentTestStore{store.NewLayeredStore(sqlstore.NewSqlSupplier(*settings, nil), nil, nil)}
+}
+
+func StopTestStore() {
+	if testStoreContainer != nil {
+		testStoreContainer.Stop()
+		testStoreContainer = nil
+	}
+}
+
 func setupTestHelper(enterprise bool) *TestHelper {
-	utils.TranslationsPreInit()
+	if utils.T == nil {
+		utils.TranslationsPreInit()
+	}
 	utils.LoadConfig("config.json")
 	utils.InitTranslations(utils.Cfg.LocalizationSettings)
 
+	var options []app.Option
+	if testStore != nil {
+		options = append(options, app.StoreOverride(testStore))
+	}
+
 	th := &TestHelper{
-		App: app.New(),
+		App: app.New(options...),
 	}
 
 	*utils.Cfg.TeamSettings.MaxUsersPerTeam = 50
@@ -81,8 +114,11 @@ func ReloadConfigForSetup() {
 }
 
 func (me *TestHelper) InitBasic() *TestHelper {
+	me.waitForConnectivity()
+
 	me.BasicClient = me.CreateClient()
 	me.BasicUser = me.CreateUser(me.BasicClient)
+	me.App.UpdateUserRoles(me.BasicUser.Id, model.ROLE_SYSTEM_USER.Id)
 	me.LoginBasic()
 	me.BasicTeam = me.CreateTeam(me.BasicClient)
 	me.LinkUserToTeam(me.BasicUser, me.BasicTeam)
@@ -100,6 +136,8 @@ func (me *TestHelper) InitBasic() *TestHelper {
 }
 
 func (me *TestHelper) InitSystemAdmin() *TestHelper {
+	me.waitForConnectivity()
+
 	me.SystemAdminClient = me.CreateClient()
 	me.SystemAdminUser = me.CreateUser(me.SystemAdminClient)
 	me.SystemAdminUser.Password = "Password1"
@@ -113,6 +151,17 @@ func (me *TestHelper) InitSystemAdmin() *TestHelper {
 	return me
 }
 
+func (me *TestHelper) waitForConnectivity() {
+	for i := 0; i < 1000; i++ {
+		_, err := net.Dial("tcp", "localhost"+*utils.Cfg.ServiceSettings.ListenAddress)
+		if err == nil {
+			return
+		}
+		time.Sleep(time.Millisecond * 20)
+	}
+	panic("unable to connect")
+}
+
 func (me *TestHelper) CreateClient() *model.Client {
 	return model.NewClient("http://localhost" + *utils.Cfg.ServiceSettings.ListenAddress)
 }
@@ -305,4 +354,8 @@ func (me *TestHelper) LoginSystemAdmin() {
 
 func (me *TestHelper) TearDown() {
 	me.App.Shutdown()
+	if err := recover(); err != nil {
+		StopTestStore()
+		panic(err)
+	}
 }
diff --git a/api/cli_test.go b/api/cli_test.go
deleted file mode 100644
index c1e30635d..000000000
--- a/api/cli_test.go
+++ /dev/null
@@ -1,417 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package api
-
-import (
-	"os/exec"
-	"strings"
-	"testing"
-
-	"github.com/mattermost/mattermost-server/model"
-)
-
-func TestCliVersion(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go version`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-}
-
-func TestCliCreateTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitSystemAdmin()
-	defer th.TearDown()
-
-	id := model.NewId()
-	name := "name" + id
-	displayName := "Name " + id
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go team create --name "`+name+`" --display_name "`+displayName+`"`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	found := th.SystemAdminClient.Must(th.SystemAdminClient.FindTeamByName(name)).Data.(bool)
-
-	if !found {
-		t.Fatal("Failed to create Team")
-	}
-}
-
-func TestCliCreateUserWithTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitSystemAdmin()
-	defer th.TearDown()
-
-	id := model.NewId()
-	email := "success+" + id + "@simulator.amazonses.com"
-	username := "name" + id
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go user create --email "`+email+`" --password "mypassword1" --username "`+username+`"`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	cmd2 := exec.Command("bash", "-c", `go run ../cmd/platform/*.go team add `+th.SystemAdminTeam.Id+" "+email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 != nil {
-		t.Log(string(output2))
-		t.Fatal(err2)
-	}
-
-	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == email {
-			found = true
-		}
-
-	}
-
-	if !found {
-		t.Fatal("Failed to create User")
-	}
-}
-
-func TestCliCreateUserWithoutTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup()
-	defer th.TearDown()
-
-	id := model.NewId()
-	email := "success+" + id + "@simulator.amazonses.com"
-	username := "name" + id
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go user create --email "`+email+`" --password "mypassword1" --username "`+username+`"`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	if result := <-th.App.Srv.Store.User().GetByEmail(email); result.Err != nil {
-		t.Fatal()
-	} else {
-		user := result.Data.(*model.User)
-		if user.Email != email {
-			t.Fatal()
-		}
-	}
-}
-
-func TestCliAssignRole(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go roles system_admin "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err != nil {
-		t.Fatal()
-	} else {
-		user := result.Data.(*model.User)
-		if user.Roles != "system_admin system_user" {
-			t.Fatal()
-		}
-	}
-}
-
-func TestCliJoinChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	// Joining twice should succeed
-	cmd1 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output1, err1 := cmd1.CombinedOutput()
-	if err1 != nil {
-		t.Log(string(output1))
-		t.Fatal(err1)
-	}
-
-	// should fail because channel does not exist
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+"asdf "+th.BasicUser2.Email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 == nil {
-		t.Log(string(output2))
-		t.Fatal()
-	}
-}
-
-func TestCliRemoveChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	// should fail because channel does not exist
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel remove "+th.BasicTeam.Name+":doesnotexist "+th.BasicUser2.Email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 == nil {
-		t.Log(string(output2))
-		t.Fatal()
-	}
-
-	cmd3 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel remove "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output3, err3 := cmd3.CombinedOutput()
-	if err3 != nil {
-		t.Log(string(output3))
-		t.Fatal(err3)
-	}
-
-	// Leaving twice should succeed
-	cmd4 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel remove "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output4, err4 := cmd4.CombinedOutput()
-	if err4 != nil {
-		t.Log(string(output4))
-		t.Fatal(err4)
-	}
-}
-
-func TestCliListChannels(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel list "+th.BasicTeam.Name)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	if !strings.Contains(string(output), "town-square") {
-		t.Fatal("should have channels")
-	}
-
-	if !strings.Contains(string(output), channel.Name+" (archived)") {
-		t.Fatal("should have archived channel")
-	}
-}
-
-func TestCliRestoreChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel restore "+th.BasicTeam.Name+":"+channel.Name)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	// restoring twice should succeed
-	cmd1 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel restore "+th.BasicTeam.Name+":"+channel.Name)
-	output1, err1 := cmd1.CombinedOutput()
-	if err1 != nil {
-		t.Log(string(output1))
-		t.Fatal(err1)
-	}
-}
-
-func TestCliJoinTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitSystemAdmin().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go team add "+th.SystemAdminTeam.Name+" "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == th.BasicUser.Email {
-			found = true
-		}
-
-	}
-
-	if !found {
-		t.Fatal("Failed to create User")
-	}
-}
-
-func TestCliLeaveTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go team remove "+th.BasicTeam.Name+" "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	profiles := th.BasicClient.Must(th.BasicClient.GetProfilesInTeam(th.BasicTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == th.BasicUser.Email {
-			found = true
-		}
-
-	}
-
-	if found {
-		t.Fatal("profile should not be on team")
-	}
-
-	if result := <-th.App.Srv.Store.Team().GetTeamsByUserId(th.BasicUser.Id); result.Err != nil {
-		teamMembers := result.Data.([]*model.TeamMember)
-		if len(teamMembers) > 0 {
-			t.Fatal("Shouldn't be in team")
-		}
-	}
-}
-
-func TestCliResetPassword(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go user password "+th.BasicUser.Email+" password2")
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	th.BasicClient.Logout()
-	th.BasicUser.Password = "password2"
-	th.LoginBasic()
-}
-
-func TestCliCreateChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	id := model.NewId()
-	name := "name" + id
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel create --display_name "+name+" --team "+th.BasicTeam.Name+" --name "+name)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	name = name + "-private"
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel create --display_name="+name+" --team "+th.BasicTeam.Name+" --private --name "+name)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 != nil {
-		t.Log(string(output2))
-		t.Fatal(err2)
-	}
-}
-
-func TestCliMakeUserActiveAndInactive(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	// first inactivate the user
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go user deactivate "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal()
-	}
-
-	// activate the inactive user
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go user activate "+th.BasicUser.Email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 != nil {
-		t.Log(string(output2))
-		t.Fatal()
-	}
-}