Use tmpfs containers for api/api4 tests, move and speed up CLI tests (#7606)

* use tmpfs containers for api/api4, move and speed up cli tests

* minor optimizations

* add missing files, fix pre-existing race condition

* add . to TestMain check

* add requested log message

10 files changed, 508 insertions, 434 deletions

diff --git a/Makefile b/Makefile
index 4b70b09b6..2643b8aa2 100644
--- a/Makefile
+++ b/Makefile
@@ -56,6 +56,8 @@ GO_LINKER_FLAGS ?= -ldflags \
 # GOOS/GOARCH of the build host, used to determine whether we're cross-compiling or not
 BUILDER_GOOS_GOARCH="$(shell $(GO) env GOOS)_$(shell $(GO) env GOARCH)"
 
+PLATFORM_FILES=$(shell ls -1 ./cmd/platform/*.go | grep -v _test.go)
+
 # Output paths
 DIST_ROOT=dist
 DIST_PATH=$(DIST_ROOT)/mattermost
@@ -328,10 +330,10 @@ endif
 test-server: test-te test-ee
 
 internal-test-web-client:
-	$(GO) run $(GOFLAGS) ./cmd/platform/*go test web_client_tests
+	$(GO) run $(GOFLAGS) $(PLATFORM_FILES) test web_client_tests
 
 run-server-for-web-client-tests:
-	$(GO) run $(GOFLAGS) ./cmd/platform/*go test web_client_tests_server
+	$(GO) run $(GOFLAGS) $(PLATFORM_FILES) test web_client_tests_server
 
 test-client:
 	@echo Running client tests
@@ -350,13 +352,13 @@ run-server: start-docker
 	@echo Running mattermost for development
 
 	mkdir -p $(BUILD_WEBAPP_DIR)/dist/files
-	$(GO) run $(GOFLAGS) $(GO_LINKER_FLAGS) ./cmd/platform/*.go --disableconfigwatch &
+	$(GO) run $(GOFLAGS) $(GO_LINKER_FLAGS) $(PLATFORM_FILES) --disableconfigwatch &
 
 run-cli: start-docker
 	@echo Running mattermost for development
 	@echo Example should be like 'make ARGS="-version" run-cli'
 
-	$(GO) run $(GOFLAGS) $(GO_LINKER_FLAGS) ./cmd/platform/*.go ${ARGS}
+	$(GO) run $(GOFLAGS) $(GO_LINKER_FLAGS) $(PLATFORM_FILES) ${ARGS}
 
 run-client:
 	@echo Running mattermost client for development
@@ -407,7 +409,7 @@ restart-client: | stop-client run-client
 
 run-job-server:
 	@echo Running job server for development
-	$(GO) run $(GOFLAGS) $(GO_LINKER_FLAGS) ./cmd/platform/*.go jobserver --disableconfigwatch &
+	$(GO) run $(GOFLAGS) $(GO_LINKER_FLAGS) $(PLATFORM_FILES) jobserver --disableconfigwatch &
 
 clean: stop-docker
 	@echo Cleaning
diff --git a/api/api_test.go b/api/api_test.go
new file mode 100644
index 000000000..3d4f5f156
--- /dev/null
+++ b/api/api_test.go
@@ -0,0 +1,48 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package api
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	l4g "github.com/alecthomas/log4go"
+
+	"github.com/mattermost/mattermost-server/store/storetest"
+	"github.com/mattermost/mattermost-server/utils"
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// In the case where a dev just wants to run a single test, it's faster to just use the default
+	// store.
+	if filter := flag.Lookup("test.run").Value.String(); filter != "" && filter != "." {
+		utils.TranslationsPreInit()
+		utils.LoadConfig("config.json")
+		l4g.Info("-test.run used, not creating temporary containers")
+		os.Exit(m.Run())
+	}
+
+	utils.TranslationsPreInit()
+	utils.LoadConfig("config.json")
+	utils.InitTranslations(utils.Cfg.LocalizationSettings)
+
+	status := 0
+
+	container, settings, err := storetest.NewMySQLContainer()
+	if err != nil {
+		panic(err)
+	}
+
+	UseTestStore(container, settings)
+
+	defer func() {
+		StopTestStore()
+		os.Exit(status)
+	}()
+
+	status = m.Run()
+}
diff --git a/api/apitestlib.go b/api/apitestlib.go
index a8e70fe42..48637ee59 100644
--- a/api/apitestlib.go
+++ b/api/apitestlib.go
@@ -4,12 +4,15 @@
 package api
 
 import (
+	"net"
 	"time"
 
 	"github.com/mattermost/mattermost-server/api4"
 	"github.com/mattermost/mattermost-server/app"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
+	"github.com/mattermost/mattermost-server/store/sqlstore"
+	"github.com/mattermost/mattermost-server/store/storetest"
 	"github.com/mattermost/mattermost-server/utils"
 	"github.com/mattermost/mattermost-server/wsapi"
 
@@ -33,13 +36,43 @@ type TestHelper struct {
 	SystemAdminChannel *model.Channel
 }
 
+type persistentTestStore struct {
+	store.Store
+}
+
+func (*persistentTestStore) Close() {}
+
+var testStoreContainer *storetest.RunningContainer
+var testStore *persistentTestStore
+
+// UseTestStore sets the container and corresponding settings to use for tests. Once the tests are
+// complete (e.g. at the end of your TestMain implementation), you should call StopTestStore.
+func UseTestStore(container *storetest.RunningContainer, settings *model.SqlSettings) {
+	testStoreContainer = container
+	testStore = &persistentTestStore{store.NewLayeredStore(sqlstore.NewSqlSupplier(*settings, nil), nil, nil)}
+}
+
+func StopTestStore() {
+	if testStoreContainer != nil {
+		testStoreContainer.Stop()
+		testStoreContainer = nil
+	}
+}
+
 func setupTestHelper(enterprise bool) *TestHelper {
-	utils.TranslationsPreInit()
+	if utils.T == nil {
+		utils.TranslationsPreInit()
+	}
 	utils.LoadConfig("config.json")
 	utils.InitTranslations(utils.Cfg.LocalizationSettings)
 
+	var options []app.Option
+	if testStore != nil {
+		options = append(options, app.StoreOverride(testStore))
+	}
+
 	th := &TestHelper{
-		App: app.New(),
+		App: app.New(options...),
 	}
 
 	*utils.Cfg.TeamSettings.MaxUsersPerTeam = 50
@@ -81,8 +114,11 @@ func ReloadConfigForSetup() {
 }
 
 func (me *TestHelper) InitBasic() *TestHelper {
+	me.waitForConnectivity()
+
 	me.BasicClient = me.CreateClient()
 	me.BasicUser = me.CreateUser(me.BasicClient)
+	me.App.UpdateUserRoles(me.BasicUser.Id, model.ROLE_SYSTEM_USER.Id)
 	me.LoginBasic()
 	me.BasicTeam = me.CreateTeam(me.BasicClient)
 	me.LinkUserToTeam(me.BasicUser, me.BasicTeam)
@@ -100,6 +136,8 @@ func (me *TestHelper) InitBasic() *TestHelper {
 }
 
 func (me *TestHelper) InitSystemAdmin() *TestHelper {
+	me.waitForConnectivity()
+
 	me.SystemAdminClient = me.CreateClient()
 	me.SystemAdminUser = me.CreateUser(me.SystemAdminClient)
 	me.SystemAdminUser.Password = "Password1"
@@ -113,6 +151,17 @@ func (me *TestHelper) InitSystemAdmin() *TestHelper {
 	return me
 }
 
+func (me *TestHelper) waitForConnectivity() {
+	for i := 0; i < 1000; i++ {
+		_, err := net.Dial("tcp", "localhost"+*utils.Cfg.ServiceSettings.ListenAddress)
+		if err == nil {
+			return
+		}
+		time.Sleep(time.Millisecond * 20)
+	}
+	panic("unable to connect")
+}
+
 func (me *TestHelper) CreateClient() *model.Client {
 	return model.NewClient("http://localhost" + *utils.Cfg.ServiceSettings.ListenAddress)
 }
@@ -305,4 +354,8 @@ func (me *TestHelper) LoginSystemAdmin() {
 
 func (me *TestHelper) TearDown() {
 	me.App.Shutdown()
+	if err := recover(); err != nil {
+		StopTestStore()
+		panic(err)
+	}
 }
diff --git a/api/cli_test.go b/api/cli_test.go
deleted file mode 100644
index c1e30635d..000000000
--- a/api/cli_test.go
+++ /dev/null
@@ -1,417 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package api
-
-import (
-	"os/exec"
-	"strings"
-	"testing"
-
-	"github.com/mattermost/mattermost-server/model"
-)
-
-func TestCliVersion(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go version`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-}
-
-func TestCliCreateTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitSystemAdmin()
-	defer th.TearDown()
-
-	id := model.NewId()
-	name := "name" + id
-	displayName := "Name " + id
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go team create --name "`+name+`" --display_name "`+displayName+`"`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	found := th.SystemAdminClient.Must(th.SystemAdminClient.FindTeamByName(name)).Data.(bool)
-
-	if !found {
-		t.Fatal("Failed to create Team")
-	}
-}
-
-func TestCliCreateUserWithTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitSystemAdmin()
-	defer th.TearDown()
-
-	id := model.NewId()
-	email := "success+" + id + "@simulator.amazonses.com"
-	username := "name" + id
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go user create --email "`+email+`" --password "mypassword1" --username "`+username+`"`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	cmd2 := exec.Command("bash", "-c", `go run ../cmd/platform/*.go team add `+th.SystemAdminTeam.Id+" "+email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 != nil {
-		t.Log(string(output2))
-		t.Fatal(err2)
-	}
-
-	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == email {
-			found = true
-		}
-
-	}
-
-	if !found {
-		t.Fatal("Failed to create User")
-	}
-}
-
-func TestCliCreateUserWithoutTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup()
-	defer th.TearDown()
-
-	id := model.NewId()
-	email := "success+" + id + "@simulator.amazonses.com"
-	username := "name" + id
-
-	cmd := exec.Command("bash", "-c", `go run ../cmd/platform/*.go user create --email "`+email+`" --password "mypassword1" --username "`+username+`"`)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	if result := <-th.App.Srv.Store.User().GetByEmail(email); result.Err != nil {
-		t.Fatal()
-	} else {
-		user := result.Data.(*model.User)
-		if user.Email != email {
-			t.Fatal()
-		}
-	}
-}
-
-func TestCliAssignRole(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go roles system_admin "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err != nil {
-		t.Fatal()
-	} else {
-		user := result.Data.(*model.User)
-		if user.Roles != "system_admin system_user" {
-			t.Fatal()
-		}
-	}
-}
-
-func TestCliJoinChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	// Joining twice should succeed
-	cmd1 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output1, err1 := cmd1.CombinedOutput()
-	if err1 != nil {
-		t.Log(string(output1))
-		t.Fatal(err1)
-	}
-
-	// should fail because channel does not exist
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+"asdf "+th.BasicUser2.Email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 == nil {
-		t.Log(string(output2))
-		t.Fatal()
-	}
-}
-
-func TestCliRemoveChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel add "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	// should fail because channel does not exist
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel remove "+th.BasicTeam.Name+":doesnotexist "+th.BasicUser2.Email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 == nil {
-		t.Log(string(output2))
-		t.Fatal()
-	}
-
-	cmd3 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel remove "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output3, err3 := cmd3.CombinedOutput()
-	if err3 != nil {
-		t.Log(string(output3))
-		t.Fatal(err3)
-	}
-
-	// Leaving twice should succeed
-	cmd4 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel remove "+th.BasicTeam.Name+":"+channel.Name+" "+th.BasicUser2.Email)
-	output4, err4 := cmd4.CombinedOutput()
-	if err4 != nil {
-		t.Log(string(output4))
-		t.Fatal(err4)
-	}
-}
-
-func TestCliListChannels(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel list "+th.BasicTeam.Name)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	if !strings.Contains(string(output), "town-square") {
-		t.Fatal("should have channels")
-	}
-
-	if !strings.Contains(string(output), channel.Name+" (archived)") {
-		t.Fatal("should have archived channel")
-	}
-}
-
-func TestCliRestoreChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
-	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel restore "+th.BasicTeam.Name+":"+channel.Name)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	// restoring twice should succeed
-	cmd1 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel restore "+th.BasicTeam.Name+":"+channel.Name)
-	output1, err1 := cmd1.CombinedOutput()
-	if err1 != nil {
-		t.Log(string(output1))
-		t.Fatal(err1)
-	}
-}
-
-func TestCliJoinTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitSystemAdmin().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go team add "+th.SystemAdminTeam.Name+" "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == th.BasicUser.Email {
-			found = true
-		}
-
-	}
-
-	if !found {
-		t.Fatal("Failed to create User")
-	}
-}
-
-func TestCliLeaveTeam(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go team remove "+th.BasicTeam.Name+" "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	profiles := th.BasicClient.Must(th.BasicClient.GetProfilesInTeam(th.BasicTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
-
-	found := false
-
-	for _, user := range profiles {
-		if user.Email == th.BasicUser.Email {
-			found = true
-		}
-
-	}
-
-	if found {
-		t.Fatal("profile should not be on team")
-	}
-
-	if result := <-th.App.Srv.Store.Team().GetTeamsByUserId(th.BasicUser.Id); result.Err != nil {
-		teamMembers := result.Data.([]*model.TeamMember)
-		if len(teamMembers) > 0 {
-			t.Fatal("Shouldn't be in team")
-		}
-	}
-}
-
-func TestCliResetPassword(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go user password "+th.BasicUser.Email+" password2")
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	th.BasicClient.Logout()
-	th.BasicUser.Password = "password2"
-	th.LoginBasic()
-}
-
-func TestCliCreateChannel(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	id := model.NewId()
-	name := "name" + id
-
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel create --display_name "+name+" --team "+th.BasicTeam.Name+" --name "+name)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal(err)
-	}
-
-	name = name + "-private"
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go channel create --display_name="+name+" --team "+th.BasicTeam.Name+" --private --name "+name)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 != nil {
-		t.Log(string(output2))
-		t.Fatal(err2)
-	}
-}
-
-func TestCliMakeUserActiveAndInactive(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := Setup().InitBasic()
-	defer th.TearDown()
-
-	// first inactivate the user
-	cmd := exec.Command("bash", "-c", "go run ../cmd/platform/*.go user deactivate "+th.BasicUser.Email)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Log(string(output))
-		t.Fatal()
-	}
-
-	// activate the inactive user
-	cmd2 := exec.Command("bash", "-c", "go run ../cmd/platform/*.go user activate "+th.BasicUser.Email)
-	output2, err2 := cmd2.CombinedOutput()
-	if err2 != nil {
-		t.Log(string(output2))
-		t.Fatal()
-	}
-}
diff --git a/api4/api_test.go b/api4/api_test.go
new file mode 100644
index 000000000..8a8753a4f
--- /dev/null
+++ b/api4/api_test.go
@@ -0,0 +1,48 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package api4
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	l4g "github.com/alecthomas/log4go"
+
+	"github.com/mattermost/mattermost-server/store/storetest"
+	"github.com/mattermost/mattermost-server/utils"
+)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// In the case where a dev just wants to run a single test, it's faster to just use the default
+	// store.
+	if filter := flag.Lookup("test.run").Value.String(); filter != "" && filter != "." {
+		utils.TranslationsPreInit()
+		utils.LoadConfig("config.json")
+		l4g.Info("-test.run used, not creating temporary containers")
+		os.Exit(m.Run())
+	}
+
+	utils.TranslationsPreInit()
+	utils.LoadConfig("config.json")
+	utils.InitTranslations(utils.Cfg.LocalizationSettings)
+
+	status := 0
+
+	container, settings, err := storetest.NewMySQLContainer()
+	if err != nil {
+		panic(err)
+	}
+
+	UseTestStore(container, settings)
+
+	defer func() {
+		StopTestStore()
+		os.Exit(status)
+	}()
+
+	status = m.Run()
+}
diff --git a/api4/apitestlib.go b/api4/apitestlib.go
index d1da7bfd8..80a44c13f 100644
--- a/api4/apitestlib.go
+++ b/api4/apitestlib.go
@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"reflect"
@@ -21,6 +22,8 @@ import (
 	"github.com/mattermost/mattermost-server/app"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
+	"github.com/mattermost/mattermost-server/store/sqlstore"
+	"github.com/mattermost/mattermost-server/store/storetest"
 	"github.com/mattermost/mattermost-server/utils"
 	"github.com/mattermost/mattermost-server/wsapi"
 
@@ -45,13 +48,43 @@ type TestHelper struct {
 	SystemAdminUser   *model.User
 }
 
+type persistentTestStore struct {
+	store.Store
+}
+
+func (*persistentTestStore) Close() {}
+
+var testStoreContainer *storetest.RunningContainer
+var testStore *persistentTestStore
+
+// UseTestStore sets the container and corresponding settings to use for tests. Once the tests are
+// complete (e.g. at the end of your TestMain implementation), you should call StopTestStore.
+func UseTestStore(container *storetest.RunningContainer, settings *model.SqlSettings) {
+	testStoreContainer = container
+	testStore = &persistentTestStore{store.NewLayeredStore(sqlstore.NewSqlSupplier(*settings, nil), nil, nil)}
+}
+
+func StopTestStore() {
+	if testStoreContainer != nil {
+		testStoreContainer.Stop()
+		testStoreContainer = nil
+	}
+}
+
 func setupTestHelper(enterprise bool) *TestHelper {
-	utils.TranslationsPreInit()
+	if utils.T == nil {
+		utils.TranslationsPreInit()
+	}
 	utils.LoadConfig("config.json")
 	utils.InitTranslations(utils.Cfg.LocalizationSettings)
 
+	var options []app.Option
+	if testStore != nil {
+		options = append(options, app.StoreOverride(testStore))
+	}
+
 	th := &TestHelper{
-		App: app.New(),
+		App: app.New(options...),
 	}
 
 	*utils.Cfg.TeamSettings.MaxUsersPerTeam = 50
@@ -142,10 +175,18 @@ func (me *TestHelper) TearDown() {
 	me.App.Shutdown()
 
 	utils.EnableDebugLogForTest()
+
+	if err := recover(); err != nil {
+		StopTestStore()
+		panic(err)
+	}
 }
 
 func (me *TestHelper) InitBasic() *TestHelper {
+	me.waitForConnectivity()
+
 	me.TeamAdminUser = me.CreateUser()
+	me.App.UpdateUserRoles(me.TeamAdminUser.Id, model.ROLE_SYSTEM_USER.Id)
 	me.LoginTeamAdmin()
 	me.BasicTeam = me.CreateTeam()
 	me.BasicChannel = me.CreatePublicChannel()
@@ -169,6 +210,8 @@ func (me *TestHelper) InitBasic() *TestHelper {
 }
 
 func (me *TestHelper) InitSystemAdmin() *TestHelper {
+	me.waitForConnectivity()
+
 	me.SystemAdminUser = me.CreateUser()
 	me.App.UpdateUserRoles(me.SystemAdminUser.Id, model.ROLE_SYSTEM_USER.Id+" "+model.ROLE_SYSTEM_ADMIN.Id)
 	me.LoginSystemAdmin()
@@ -176,6 +219,17 @@ func (me *TestHelper) InitSystemAdmin() *TestHelper {
 	return me
 }
 
+func (me *TestHelper) waitForConnectivity() {
+	for i := 0; i < 1000; i++ {
+		_, err := net.Dial("tcp", "localhost"+*utils.Cfg.ServiceSettings.ListenAddress)
+		if err == nil {
+			return
+		}
+		time.Sleep(time.Millisecond * 20)
+	}
+	panic("unable to connect")
+}
+
 func (me *TestHelper) CreateClient() *model.Client4 {
 	return model.NewAPIv4Client("http://localhost" + *utils.Cfg.ServiceSettings.ListenAddress)
 }
diff --git a/app/options.go b/app/options.go
index 3058769d6..e5ac85706 100644
--- a/app/options.go
+++ b/app/options.go
@@ -12,7 +12,7 @@ type Option func(a *App)
 // By default, the app will use the store specified by the configuration. This allows you to
 // construct an app with a different store.
 //
-// The storeOrFactory parameter must be either a store.Store or func() store.Store.
+// The storeOrFactory parameter must be either a store.Store or func(App) store.Store.
 func StoreOverride(storeOrFactory interface{}) Option {
 	return func(a *App) {
 		switch s := storeOrFactory.(type) {
@@ -20,8 +20,10 @@ func StoreOverride(storeOrFactory interface{}) Option {
 			a.newStore = func() store.Store {
 				return s
 			}
-		case func() store.Store:
-			a.newStore = s
+		case func(*App) store.Store:
+			a.newStore = func() store.Store {
+				return s(a)
+			}
 		default:
 			panic("invalid StoreOverride")
 		}
diff --git a/cmd/platform/cli_test.go b/cmd/platform/cli_test.go
new file mode 100644
index 000000000..7d6717660
--- /dev/null
+++ b/cmd/platform/cli_test.go
@@ -0,0 +1,287 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/api"
+	"github.com/mattermost/mattermost-server/model"
+)
+
+var testExePath string
+
+func checkCommand(t *testing.T, args ...string) string {
+	output, err := exec.Command(testExePath, args...).CombinedOutput()
+	require.NoError(t, err, string(output))
+	return string(output)
+}
+
+func TestCliVersion(t *testing.T) {
+	checkCommand(t, "version")
+}
+
+func TestCliCreateTeam(t *testing.T) {
+	th := api.Setup().InitSystemAdmin()
+	defer th.TearDown()
+
+	id := model.NewId()
+	name := "name" + id
+	displayName := "Name " + id
+
+	checkCommand(t, "team", "create", "--name", name, "--display_name", displayName)
+
+	found := th.SystemAdminClient.Must(th.SystemAdminClient.FindTeamByName(name)).Data.(bool)
+
+	if !found {
+		t.Fatal("Failed to create Team")
+	}
+}
+
+func TestCliCreateUserWithTeam(t *testing.T) {
+	th := api.Setup().InitSystemAdmin()
+	defer th.TearDown()
+
+	id := model.NewId()
+	email := "success+" + id + "@simulator.amazonses.com"
+	username := "name" + id
+
+	checkCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
+
+	checkCommand(t, "team", "add", th.SystemAdminTeam.Id, email)
+
+	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
+
+	found := false
+
+	for _, user := range profiles {
+		if user.Email == email {
+			found = true
+		}
+
+	}
+
+	if !found {
+		t.Fatal("Failed to create User")
+	}
+}
+
+func TestCliCreateUserWithoutTeam(t *testing.T) {
+	th := api.Setup()
+	defer th.TearDown()
+
+	id := model.NewId()
+	email := "success+" + id + "@simulator.amazonses.com"
+	username := "name" + id
+
+	checkCommand(t, "user", "create", "--email", email, "--password", "mypassword1", "--username", username)
+
+	if result := <-th.App.Srv.Store.User().GetByEmail(email); result.Err != nil {
+		t.Fatal()
+	} else {
+		user := result.Data.(*model.User)
+		if user.Email != email {
+			t.Fatal()
+		}
+	}
+}
+
+func TestCliAssignRole(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "roles", "system_admin", th.BasicUser.Email)
+
+	if result := <-th.App.Srv.Store.User().GetByEmail(th.BasicUser.Email); result.Err != nil {
+		t.Fatal()
+	} else {
+		user := result.Data.(*model.User)
+		if user.Roles != "system_admin system_user" {
+			t.Fatal()
+		}
+	}
+}
+
+func TestCliJoinChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+
+	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// Joining twice should succeed
+	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// should fail because channel does not exist
+	require.Error(t, exec.Command(testExePath, "channel", "add", th.BasicTeam.Name+":"+channel.Name+"asdf", th.BasicUser2.Email).Run())
+}
+
+func TestCliRemoveChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+
+	checkCommand(t, "channel", "add", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// should fail because channel does not exist
+	require.Error(t, exec.Command(testExePath, "channel", "remove", th.BasicTeam.Name+":doesnotexist", th.BasicUser2.Email).Run())
+
+	checkCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+
+	// Leaving twice should succeed
+	checkCommand(t, "channel", "remove", th.BasicTeam.Name+":"+channel.Name, th.BasicUser2.Email)
+}
+
+func TestCliListChannels(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
+
+	output := checkCommand(t, "channel", "list", th.BasicTeam.Name)
+
+	if !strings.Contains(string(output), "town-square") {
+		t.Fatal("should have channels")
+	}
+
+	if !strings.Contains(string(output), channel.Name+" (archived)") {
+		t.Fatal("should have archived channel")
+	}
+}
+
+func TestCliRestoreChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	channel := th.CreateChannel(th.BasicClient, th.BasicTeam)
+	th.BasicClient.Must(th.BasicClient.DeleteChannel(channel.Id))
+
+	checkCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
+
+	// restoring twice should succeed
+	checkCommand(t, "channel", "restore", th.BasicTeam.Name+":"+channel.Name)
+}
+
+func TestCliJoinTeam(t *testing.T) {
+	th := api.Setup().InitSystemAdmin().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "team", "add", th.SystemAdminTeam.Name, th.BasicUser.Email)
+
+	profiles := th.SystemAdminClient.Must(th.SystemAdminClient.GetProfilesInTeam(th.SystemAdminTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
+
+	found := false
+
+	for _, user := range profiles {
+		if user.Email == th.BasicUser.Email {
+			found = true
+		}
+
+	}
+
+	if !found {
+		t.Fatal("Failed to create User")
+	}
+}
+
+func TestCliLeaveTeam(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "team", "remove", th.BasicTeam.Name, th.BasicUser.Email)
+
+	profiles := th.BasicClient.Must(th.BasicClient.GetProfilesInTeam(th.BasicTeam.Id, 0, 1000, "")).Data.(map[string]*model.User)
+
+	found := false
+
+	for _, user := range profiles {
+		if user.Email == th.BasicUser.Email {
+			found = true
+		}
+
+	}
+
+	if found {
+		t.Fatal("profile should not be on team")
+	}
+
+	if result := <-th.App.Srv.Store.Team().GetTeamsByUserId(th.BasicUser.Id); result.Err != nil {
+		teamMembers := result.Data.([]*model.TeamMember)
+		if len(teamMembers) > 0 {
+			t.Fatal("Shouldn't be in team")
+		}
+	}
+}
+
+func TestCliResetPassword(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	checkCommand(t, "user", "password", th.BasicUser.Email, "password2")
+
+	th.BasicClient.Logout()
+	th.BasicUser.Password = "password2"
+	th.LoginBasic()
+}
+
+func TestCliCreateChannel(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	id := model.NewId()
+	name := "name" + id
+
+	checkCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--name", name)
+
+	name = name + "-private"
+	checkCommand(t, "channel", "create", "--display_name", name, "--team", th.BasicTeam.Name, "--private", "--name", name)
+}
+
+func TestCliMakeUserActiveAndInactive(t *testing.T) {
+	th := api.Setup().InitBasic()
+	defer th.TearDown()
+
+	// first inactivate the user
+	checkCommand(t, "user", "deactivate", th.BasicUser.Email)
+
+	// activate the inactive user
+	checkCommand(t, "user", "activate", th.BasicUser.Email)
+}
+
+func TestMain(m *testing.M) {
+	dir, err := ioutil.TempDir("", "cli_test")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir)
+
+	testExePath = filepath.Join(dir, "cli")
+	files, err := filepath.Glob("./*.go")
+	if err != nil {
+		panic(err)
+	}
+
+	cmd := exec.Command("go", append([]string{"build", "-o", testExePath}, files...)...)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+	if err := cmd.Run(); err != nil {
+		panic(err)
+	}
+
+	status := 0
+	defer func() {
+		os.Exit(status)
+	}()
+	status = m.Run()
+}
diff --git a/store/sqlstore/store_test.go b/store/sqlstore/store_test.go
index d627ad0eb..d1a5714d6 100644
--- a/store/sqlstore/store_test.go
+++ b/store/sqlstore/store_test.go
@@ -4,7 +4,6 @@
 package sqlstore
 
 import (
-	"flag"
 	"os"
 	"sync"
 	"testing"
@@ -99,8 +98,6 @@ func tearDownStores() {
 }
 
 func TestMain(m *testing.M) {
-	flag.Parse()
-
 	utils.TranslationsPreInit()
 	utils.LoadConfig("config.json")
 	utils.InitTranslations(utils.Cfg.LocalizationSettings)
diff --git a/store/storetest/docker.go b/store/storetest/docker.go
index e62190ef2..cd2a3075a 100644
--- a/store/storetest/docker.go
+++ b/store/storetest/docker.go
@@ -31,7 +31,7 @@ type RunningContainer struct {
 }
 
 func (c *RunningContainer) Stop() error {
-	l4g.Info("removing container: %v", c.Id)
+	l4g.Info("Removing container: %v", c.Id)
 	return exec.Command("docker", "rm", "-f", c.Id).Run()
 }
 
@@ -111,7 +111,7 @@ func runContainer(args []string) (*RunningContainer, error) {
 		exec.Command("docker", "rm", "-f", id).Run()
 		return nil, err
 	}
-	l4g.Info("running container: %v", id)
+	l4g.Info("Running container: %v", id)
 	return &RunningContainer{containers[0]}, nil
 }