XYZ-37: Advanced Permissions Phase 1 Backend. (#8159)

* XYZ-13: Update Permission and Role structs to new design.

* XYZ-10: Role store.

* XYZ-9/XYZ-44: Roles API endpoints and WebSocket message.

* XYZ-8: Switch server permissions checks to store backed roles.

* XYZ-58: Proper validation of roles where required.

* XYZ-11/XYZ-55: Migration to store backed roles from policy config.

* XYZ-37: Update unit tests to work with database roles.

* XYZ-56: Remove the "guest" role.

* Changes to SetDefaultRolesFromConfig.

* Short-circuit the store if nothing has changed.

* Address first round of review comments.

* Address second round of review comments.

11 files changed, 567 insertions, 53 deletions

diff --git a/app/app.go b/app/app.go
index 1e46d29d0..4cc9ff7df 100644
--- a/app/app.go
+++ b/app/app.go
@@ -7,6 +7,7 @@ import (
 	"html/template"
 	"net"
 	"net/http"
+	"reflect"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -25,6 +26,8 @@ import (
 	"github.com/mattermost/mattermost-server/utils"
 )
 
+const ADVANCED_PERMISSIONS_MIGRATION_KEY = "AdvancedPermissionsMigrationComplete"
+
 type App struct {
 	goroutineCount      int32
 	goroutineExitSignal chan struct{}
@@ -62,7 +65,6 @@ type App struct {
 
 	htmlTemplateWatcher *utils.HTMLTemplateWatcher
 	sessionCache        *utils.Cache
-	roles               map[string]*model.Role
 	configListenerId    string
 	licenseListenerId   string
 	disableConfigWatch  bool
@@ -120,7 +122,6 @@ func New(options ...Option) (*App, error) {
 	})
 	app.licenseListenerId = utils.AddLicenseListener(app.configOrLicenseListener)
 	app.regenerateClientConfig()
-	app.SetDefaultRolesBasedOnConfig()
 
 	l4g.Info(utils.T("api.server.new_server.init.info"))
 
@@ -157,7 +158,6 @@ func New(options ...Option) (*App, error) {
 
 func (a *App) configOrLicenseListener() {
 	a.regenerateClientConfig()
-	a.SetDefaultRolesBasedOnConfig()
 }
 
 func (a *App) Shutdown() {
@@ -450,3 +450,57 @@ func (a *App) Handle404(w http.ResponseWriter, r *http.Request) {
 
 	utils.RenderWebError(err, w, r)
 }
+
+// This function migrates the default built in roles from code/config to the database.
+func (a *App) DoAdvancedPermissionsMigration() {
+	// If the migration is already marked as completed, don't do it again.
+	if result := <-a.Srv.Store.System().GetByName(ADVANCED_PERMISSIONS_MIGRATION_KEY); result.Err == nil {
+		return
+	}
+
+	l4g.Info("Migrating roles to database.")
+	roles := model.MakeDefaultRoles()
+	roles = utils.SetRolePermissionsFromConfig(roles, a.Config())
+
+	allSucceeded := true
+
+	for _, role := range roles {
+		if result := <-a.Srv.Store.Role().Save(role); result.Err != nil {
+			// If this failed for reasons other than the role already existing, don't mark the migration as done.
+			if result2 := <-a.Srv.Store.Role().GetByName(role.Name); result2.Err != nil {
+				l4g.Critical("Failed to migrate role to database.")
+				l4g.Critical(result.Err)
+				allSucceeded = false
+			} else {
+				// If the role already existed, check it is the same and update if not.
+				fetchedRole := result.Data.(*model.Role)
+				if !reflect.DeepEqual(fetchedRole.Permissions, role.Permissions) ||
+					fetchedRole.DisplayName != role.DisplayName ||
+					fetchedRole.Description != role.Description ||
+					fetchedRole.SchemeManaged != role.SchemeManaged {
+					role.Id = fetchedRole.Id
+					if result := <-a.Srv.Store.Role().Save(role); result.Err != nil {
+						// Role is not the same, but failed to update.
+						l4g.Critical("Failed to migrate role to database.")
+						l4g.Critical(result.Err)
+						allSucceeded = false
+					}
+				}
+			}
+		}
+	}
+
+	if !allSucceeded {
+		return
+	}
+
+	system := model.System{
+		Name:  ADVANCED_PERMISSIONS_MIGRATION_KEY,
+		Value: "true",
+	}
+
+	if result := <-a.Srv.Store.System().Save(&system); result.Err != nil {
+		l4g.Critical("Failed to mark advanced permissions migration as completed.")
+		l4g.Critical(result.Err)
+	}
+}
diff --git a/app/app_test.go b/app/app_test.go
index 25b19ead8..6d62bf249 100644
--- a/app/app_test.go
+++ b/app/app_test.go
@@ -9,7 +9,6 @@ import (
 	"testing"
 
 	l4g "github.com/alecthomas/log4go"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -71,3 +70,342 @@ func TestUpdateConfig(t *testing.T) {
 		*cfg.ServiceSettings.SiteURL = "foo"
 	})
 }
+
+func TestDoAdvancedPermissionsMigration(t *testing.T) {
+	th := Setup()
+	defer th.TearDown()
+
+	if testStoreSqlSupplier == nil {
+		t.Skip("This test requires a TestStore to be run.")
+	}
+
+	th.ResetRoleMigration()
+
+	th.App.DoAdvancedPermissionsMigration()
+
+	roleNames := []string{
+		"system_user",
+		"system_admin",
+		"team_user",
+		"team_admin",
+		"channel_user",
+		"channel_admin",
+		"system_post_all",
+		"system_post_all_public",
+		"system_user_access_token",
+		"team_post_all",
+		"team_post_all_public",
+	}
+
+	roles1, err1 := th.App.GetRolesByNames(roleNames)
+	assert.Nil(t, err1)
+	assert.Equal(t, len(roles1), len(roleNames))
+
+	expected1 := map[string][]string{
+		"channel_user": []string{
+			model.PERMISSION_READ_CHANNEL.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_UPLOAD_FILE.Id,
+			model.PERMISSION_GET_PUBLIC_LINK.Id,
+			model.PERMISSION_CREATE_POST.Id,
+			model.PERMISSION_EDIT_POST.Id,
+			model.PERMISSION_USE_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_DELETE_POST.Id,
+		},
+		"channel_admin": []string{
+			model.PERMISSION_MANAGE_CHANNEL_ROLES.Id,
+		},
+		"team_user": []string{
+			model.PERMISSION_LIST_TEAM_CHANNELS.Id,
+			model.PERMISSION_JOIN_PUBLIC_CHANNELS.Id,
+			model.PERMISSION_READ_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_VIEW_TEAM.Id,
+			model.PERMISSION_CREATE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_DELETE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_CREATE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_DELETE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_INVITE_USER.Id,
+			model.PERMISSION_ADD_USER_TO_TEAM.Id,
+		},
+		"team_post_all": []string{
+			model.PERMISSION_CREATE_POST.Id,
+		},
+		"team_post_all_public": []string{
+			model.PERMISSION_CREATE_POST_PUBLIC.Id,
+		},
+		"team_admin": []string{
+			model.PERMISSION_EDIT_OTHERS_POSTS.Id,
+			model.PERMISSION_REMOVE_USER_FROM_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM.Id,
+			model.PERMISSION_IMPORT_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM_ROLES.Id,
+			model.PERMISSION_MANAGE_CHANNEL_ROLES.Id,
+			model.PERMISSION_MANAGE_OTHERS_WEBHOOKS.Id,
+			model.PERMISSION_MANAGE_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_OTHERS_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_WEBHOOKS.Id,
+			model.PERMISSION_DELETE_POST.Id,
+			model.PERMISSION_DELETE_OTHERS_POSTS.Id,
+		},
+		"system_user": []string{
+			model.PERMISSION_CREATE_DIRECT_CHANNEL.Id,
+			model.PERMISSION_CREATE_GROUP_CHANNEL.Id,
+			model.PERMISSION_PERMANENT_DELETE_USER.Id,
+			model.PERMISSION_CREATE_TEAM.Id,
+		},
+		"system_post_all": []string{
+			model.PERMISSION_CREATE_POST.Id,
+		},
+		"system_post_all_public": []string{
+			model.PERMISSION_CREATE_POST_PUBLIC.Id,
+		},
+		"system_user_access_token": []string{
+			model.PERMISSION_CREATE_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_READ_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_REVOKE_USER_ACCESS_TOKEN.Id,
+		},
+		"system_admin": []string{
+			model.PERMISSION_ASSIGN_SYSTEM_ADMIN_ROLE.Id,
+			model.PERMISSION_MANAGE_SYSTEM.Id,
+			model.PERMISSION_MANAGE_ROLES.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_DELETE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_CREATE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_DELETE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_CREATE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_MANAGE_SYSTEM_WIDE_OAUTH.Id,
+			model.PERMISSION_MANAGE_OTHERS_WEBHOOKS.Id,
+			model.PERMISSION_EDIT_OTHER_USERS.Id,
+			model.PERMISSION_MANAGE_OAUTH.Id,
+			model.PERMISSION_INVITE_USER.Id,
+			model.PERMISSION_DELETE_POST.Id,
+			model.PERMISSION_DELETE_OTHERS_POSTS.Id,
+			model.PERMISSION_CREATE_TEAM.Id,
+			model.PERMISSION_ADD_USER_TO_TEAM.Id,
+			model.PERMISSION_LIST_USERS_WITHOUT_TEAM.Id,
+			model.PERMISSION_MANAGE_JOBS.Id,
+			model.PERMISSION_CREATE_POST_PUBLIC.Id,
+			model.PERMISSION_CREATE_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_READ_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_REVOKE_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_LIST_TEAM_CHANNELS.Id,
+			model.PERMISSION_JOIN_PUBLIC_CHANNELS.Id,
+			model.PERMISSION_READ_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_VIEW_TEAM.Id,
+			model.PERMISSION_READ_CHANNEL.Id,
+			model.PERMISSION_UPLOAD_FILE.Id,
+			model.PERMISSION_GET_PUBLIC_LINK.Id,
+			model.PERMISSION_CREATE_POST.Id,
+			model.PERMISSION_EDIT_POST.Id,
+			model.PERMISSION_USE_SLASH_COMMANDS.Id,
+			model.PERMISSION_EDIT_OTHERS_POSTS.Id,
+			model.PERMISSION_REMOVE_USER_FROM_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM.Id,
+			model.PERMISSION_IMPORT_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM_ROLES.Id,
+			model.PERMISSION_MANAGE_CHANNEL_ROLES.Id,
+			model.PERMISSION_MANAGE_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_OTHERS_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_WEBHOOKS.Id,
+		},
+	}
+
+	// Check the migration matches what's expected.
+	for name, permissions := range expected1 {
+		role, err := th.App.GetRoleByName(name)
+		assert.Nil(t, err)
+		assert.Equal(t, role.Permissions, permissions)
+	}
+
+	// Add a license and change the policy config.
+	isLicensed := utils.IsLicensed()
+	license := utils.License()
+	restrictPublicChannel := *th.App.Config().TeamSettings.RestrictPublicChannelManagement
+	restrictPrivateChannel := *th.App.Config().TeamSettings.RestrictPrivateChannelManagement
+
+	defer func() {
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.RestrictPublicChannelManagement = restrictPublicChannel })
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.RestrictPrivateChannelManagement = restrictPrivateChannel })
+		utils.SetIsLicensed(isLicensed)
+		utils.SetLicense(license)
+	}()
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.TeamSettings.RestrictPublicChannelManagement = model.PERMISSIONS_TEAM_ADMIN
+	})
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.TeamSettings.RestrictPrivateChannelManagement = model.PERMISSIONS_TEAM_ADMIN
+	})
+	utils.SetIsLicensed(true)
+	utils.SetLicense(&model.License{Features: &model.Features{}})
+	utils.License().Features.SetDefaults()
+
+	// Check the migration doesn't change anything if run again.
+	th.App.DoAdvancedPermissionsMigration()
+
+	roles2, err2 := th.App.GetRolesByNames(roleNames)
+	assert.Nil(t, err2)
+	assert.Equal(t, len(roles2), len(roleNames))
+
+	for name, permissions := range expected1 {
+		role, err := th.App.GetRoleByName(name)
+		assert.Nil(t, err)
+		assert.Equal(t, permissions, role.Permissions)
+	}
+
+	// Reset the database
+	th.ResetRoleMigration()
+
+	// Do the migration again with different policy config settings and a license.
+	th.App.DoAdvancedPermissionsMigration()
+
+	// Check the role permissions.
+	expected2 := map[string][]string{
+		"channel_user": []string{
+			model.PERMISSION_READ_CHANNEL.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_UPLOAD_FILE.Id,
+			model.PERMISSION_GET_PUBLIC_LINK.Id,
+			model.PERMISSION_CREATE_POST.Id,
+			model.PERMISSION_EDIT_POST.Id,
+			model.PERMISSION_USE_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_DELETE_POST.Id,
+		},
+		"channel_admin": []string{
+			model.PERMISSION_MANAGE_CHANNEL_ROLES.Id,
+		},
+		"team_user": []string{
+			model.PERMISSION_LIST_TEAM_CHANNELS.Id,
+			model.PERMISSION_JOIN_PUBLIC_CHANNELS.Id,
+			model.PERMISSION_READ_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_VIEW_TEAM.Id,
+			model.PERMISSION_CREATE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_DELETE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_CREATE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_DELETE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_INVITE_USER.Id,
+			model.PERMISSION_ADD_USER_TO_TEAM.Id,
+		},
+		"team_post_all": []string{
+			model.PERMISSION_CREATE_POST.Id,
+		},
+		"team_post_all_public": []string{
+			model.PERMISSION_CREATE_POST_PUBLIC.Id,
+		},
+		"team_admin": []string{
+			model.PERMISSION_EDIT_OTHERS_POSTS.Id,
+			model.PERMISSION_REMOVE_USER_FROM_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM.Id,
+			model.PERMISSION_IMPORT_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM_ROLES.Id,
+			model.PERMISSION_MANAGE_CHANNEL_ROLES.Id,
+			model.PERMISSION_MANAGE_OTHERS_WEBHOOKS.Id,
+			model.PERMISSION_MANAGE_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_OTHERS_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_WEBHOOKS.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_DELETE_POST.Id,
+			model.PERMISSION_DELETE_OTHERS_POSTS.Id,
+		},
+		"system_user": []string{
+			model.PERMISSION_CREATE_DIRECT_CHANNEL.Id,
+			model.PERMISSION_CREATE_GROUP_CHANNEL.Id,
+			model.PERMISSION_PERMANENT_DELETE_USER.Id,
+			model.PERMISSION_CREATE_TEAM.Id,
+		},
+		"system_post_all": []string{
+			model.PERMISSION_CREATE_POST.Id,
+		},
+		"system_post_all_public": []string{
+			model.PERMISSION_CREATE_POST_PUBLIC.Id,
+		},
+		"system_user_access_token": []string{
+			model.PERMISSION_CREATE_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_READ_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_REVOKE_USER_ACCESS_TOKEN.Id,
+		},
+		"system_admin": []string{
+			model.PERMISSION_ASSIGN_SYSTEM_ADMIN_ROLE.Id,
+			model.PERMISSION_MANAGE_SYSTEM.Id,
+			model.PERMISSION_MANAGE_ROLES.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_MANAGE_PUBLIC_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_MEMBERS.Id,
+			model.PERMISSION_DELETE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_CREATE_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_MANAGE_PRIVATE_CHANNEL_PROPERTIES.Id,
+			model.PERMISSION_DELETE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_CREATE_PRIVATE_CHANNEL.Id,
+			model.PERMISSION_MANAGE_SYSTEM_WIDE_OAUTH.Id,
+			model.PERMISSION_MANAGE_OTHERS_WEBHOOKS.Id,
+			model.PERMISSION_EDIT_OTHER_USERS.Id,
+			model.PERMISSION_MANAGE_OAUTH.Id,
+			model.PERMISSION_INVITE_USER.Id,
+			model.PERMISSION_DELETE_POST.Id,
+			model.PERMISSION_DELETE_OTHERS_POSTS.Id,
+			model.PERMISSION_CREATE_TEAM.Id,
+			model.PERMISSION_ADD_USER_TO_TEAM.Id,
+			model.PERMISSION_LIST_USERS_WITHOUT_TEAM.Id,
+			model.PERMISSION_MANAGE_JOBS.Id,
+			model.PERMISSION_CREATE_POST_PUBLIC.Id,
+			model.PERMISSION_CREATE_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_READ_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_REVOKE_USER_ACCESS_TOKEN.Id,
+			model.PERMISSION_LIST_TEAM_CHANNELS.Id,
+			model.PERMISSION_JOIN_PUBLIC_CHANNELS.Id,
+			model.PERMISSION_READ_PUBLIC_CHANNEL.Id,
+			model.PERMISSION_VIEW_TEAM.Id,
+			model.PERMISSION_READ_CHANNEL.Id,
+			model.PERMISSION_UPLOAD_FILE.Id,
+			model.PERMISSION_GET_PUBLIC_LINK.Id,
+			model.PERMISSION_CREATE_POST.Id,
+			model.PERMISSION_EDIT_POST.Id,
+			model.PERMISSION_USE_SLASH_COMMANDS.Id,
+			model.PERMISSION_EDIT_OTHERS_POSTS.Id,
+			model.PERMISSION_REMOVE_USER_FROM_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM.Id,
+			model.PERMISSION_IMPORT_TEAM.Id,
+			model.PERMISSION_MANAGE_TEAM_ROLES.Id,
+			model.PERMISSION_MANAGE_CHANNEL_ROLES.Id,
+			model.PERMISSION_MANAGE_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_OTHERS_SLASH_COMMANDS.Id,
+			model.PERMISSION_MANAGE_WEBHOOKS.Id,
+		},
+	}
+
+	roles3, err3 := th.App.GetRolesByNames(roleNames)
+	assert.Nil(t, err3)
+	assert.Equal(t, len(roles3), len(roleNames))
+
+	for name, permissions := range expected2 {
+		role, err := th.App.GetRoleByName(name)
+		assert.Nil(t, err)
+		assert.Equal(t, permissions, role.Permissions)
+	}
+
+	// Remove the license.
+	utils.SetIsLicensed(false)
+
+	// Do the migration again.
+	th.ResetRoleMigration()
+	th.App.DoAdvancedPermissionsMigration()
+
+	// Check the role permissions.
+	roles4, err4 := th.App.GetRolesByNames(roleNames)
+	assert.Nil(t, err4)
+	assert.Equal(t, len(roles4), len(roleNames))
+
+	for name, permissions := range expected1 {
+		role, err := th.App.GetRoleByName(name)
+		assert.Nil(t, err)
+		assert.Equal(t, permissions, role.Permissions)
+	}
+}
diff --git a/app/apptestlib.go b/app/apptestlib.go
index 09afc8f76..2b56c7ded 100644
--- a/app/apptestlib.go
+++ b/app/apptestlib.go
@@ -13,6 +13,7 @@ import (
 
 	l4g "github.com/alecthomas/log4go"
 
+	"github.com/mattermost/mattermost-server/einterfaces"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/plugin"
 	"github.com/mattermost/mattermost-server/plugin/pluginenv"
@@ -43,12 +44,16 @@ func (*persistentTestStore) Close() {}
 
 var testStoreContainer *storetest.RunningContainer
 var testStore *persistentTestStore
+var testStoreSqlSupplier *sqlstore.SqlSupplier
+var testClusterInterface *FakeClusterInterface
 
 // UseTestStore sets the container and corresponding settings to use for tests. Once the tests are
 // complete (e.g. at the end of your TestMain implementation), you should call StopTestStore.
 func UseTestStore(container *storetest.RunningContainer, settings *model.SqlSettings) {
+	testClusterInterface = &FakeClusterInterface{}
 	testStoreContainer = container
-	testStore = &persistentTestStore{store.NewLayeredStore(sqlstore.NewSqlSupplier(*settings, nil), nil, nil)}
+	testStoreSqlSupplier = sqlstore.NewSqlSupplier(*settings, nil)
+	testStore = &persistentTestStore{store.NewLayeredStore(testStoreSqlSupplier, nil, testClusterInterface)}
 }
 
 func StopTestStore() {
@@ -98,6 +103,9 @@ func setupTestHelper(enterprise bool) *TestHelper {
 	}
 	th.App.StartServer()
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress })
+
+	th.App.DoAdvancedPermissionsMigration()
+
 	th.App.Srv.Store.MarkSystemRanUnitTests()
 
 	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true })
@@ -313,3 +321,45 @@ func (me *TestHelper) InstallPlugin(manifest *model.Manifest, hooks plugin.Hooks
 		panic(err)
 	}
 }
+
+func (me *TestHelper) ResetRoleMigration() {
+	if _, err := testStoreSqlSupplier.GetMaster().Exec("DELETE from Roles"); err != nil {
+		panic(err)
+	}
+
+	testClusterInterface.sendClearRoleCacheMessage()
+
+	if _, err := testStoreSqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil {
+		panic(err)
+	}
+}
+
+type FakeClusterInterface struct {
+	clusterMessageHandler einterfaces.ClusterMessageHandler
+}
+
+func (me *FakeClusterInterface) StartInterNodeCommunication() {}
+func (me *FakeClusterInterface) StopInterNodeCommunication()  {}
+func (me *FakeClusterInterface) RegisterClusterMessageHandler(event string, crm einterfaces.ClusterMessageHandler) {
+	me.clusterMessageHandler = crm
+}
+func (me *FakeClusterInterface) GetClusterId() string                             { return "" }
+func (me *FakeClusterInterface) IsLeader() bool                                   { return false }
+func (me *FakeClusterInterface) GetMyClusterInfo() *model.ClusterInfo             { return nil }
+func (me *FakeClusterInterface) GetClusterInfos() []*model.ClusterInfo            { return nil }
+func (me *FakeClusterInterface) SendClusterMessage(cluster *model.ClusterMessage) {}
+func (me *FakeClusterInterface) NotifyMsg(buf []byte)                             {}
+func (me *FakeClusterInterface) GetClusterStats() ([]*model.ClusterStats, *model.AppError) {
+	return nil, nil
+}
+func (me *FakeClusterInterface) GetLogs(page, perPage int) ([]string, *model.AppError) {
+	return []string{}, nil
+}
+func (me *FakeClusterInterface) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
+	return nil
+}
+func (me *FakeClusterInterface) sendClearRoleCacheMessage() {
+	me.clusterMessageHandler(&model.ClusterMessage{
+		Event: model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES,
+	})
+}
diff --git a/app/authorization.go b/app/authorization.go
index 3a64bb717..632dd7566 100644
--- a/app/authorization.go
+++ b/app/authorization.go
@@ -12,7 +12,7 @@ import (
 )
 
 func (a *App) SessionHasPermissionTo(session model.Session, permission *model.Permission) bool {
-	if !a.CheckIfRolesGrantPermission(session.GetUserRoles(), permission.Id) {
+	if !a.RolesGrantPermission(session.GetUserRoles(), permission.Id) {
 		a.ClearSessionCacheForUser(session.UserId)
 		return false
 	}
@@ -28,12 +28,12 @@ func (a *App) SessionHasPermissionToTeam(session model.Session, teamId string, p
 
 	teamMember := session.GetTeamByTeamId(teamId)
 	if teamMember != nil {
-		if a.CheckIfRolesGrantPermission(teamMember.GetRoles(), permission.Id) {
+		if a.RolesGrantPermission(teamMember.GetRoles(), permission.Id) {
 			return true
 		}
 	}
 
-	return a.CheckIfRolesGrantPermission(session.GetUserRoles(), permission.Id)
+	return a.RolesGrantPermission(session.GetUserRoles(), permission.Id)
 }
 
 func (a *App) SessionHasPermissionToChannel(session model.Session, channelId string, permission *model.Permission) bool {
@@ -48,7 +48,7 @@ func (a *App) SessionHasPermissionToChannel(session model.Session, channelId str
 		ids := cmcresult.Data.(map[string]string)
 		if roles, ok := ids[channelId]; ok {
 			channelRoles = strings.Fields(roles)
-			if a.CheckIfRolesGrantPermission(channelRoles, permission.Id) {
+			if a.RolesGrantPermission(channelRoles, permission.Id) {
 				return true
 			}
 		}
@@ -69,7 +69,7 @@ func (a *App) SessionHasPermissionToChannelByPost(session model.Session, postId
 	if result := <-a.Srv.Store.Channel().GetMemberForPost(postId, session.UserId); result.Err == nil {
 		channelMember = result.Data.(*model.ChannelMember)
 
-		if a.CheckIfRolesGrantPermission(channelMember.GetRoles(), permission.Id) {
+		if a.RolesGrantPermission(channelMember.GetRoles(), permission.Id) {
 			return true
 		}
 	}
@@ -119,7 +119,7 @@ func (a *App) HasPermissionTo(askingUserId string, permission *model.Permission)
 
 	roles := user.GetRoles()
 
-	return a.CheckIfRolesGrantPermission(roles, permission.Id)
+	return a.RolesGrantPermission(roles, permission.Id)
 }
 
 func (a *App) HasPermissionToTeam(askingUserId string, teamId string, permission *model.Permission) bool {
@@ -134,7 +134,7 @@ func (a *App) HasPermissionToTeam(askingUserId string, teamId string, permission
 
 	roles := teamMember.GetRoles()
 
-	if a.CheckIfRolesGrantPermission(roles, permission.Id) {
+	if a.RolesGrantPermission(roles, permission.Id) {
 		return true
 	}
 
@@ -149,7 +149,7 @@ func (a *App) HasPermissionToChannel(askingUserId string, channelId string, perm
 	channelMember, err := a.GetChannelMember(channelId, askingUserId)
 	if err == nil {
 		roles := channelMember.GetRoles()
-		if a.CheckIfRolesGrantPermission(roles, permission.Id) {
+		if a.RolesGrantPermission(roles, permission.Id) {
 			return true
 		}
 	}
@@ -168,7 +168,7 @@ func (a *App) HasPermissionToChannelByPost(askingUserId string, postId string, p
 	if result := <-a.Srv.Store.Channel().GetMemberForPost(postId, askingUserId); result.Err == nil {
 		channelMember = result.Data.(*model.ChannelMember)
 
-		if a.CheckIfRolesGrantPermission(channelMember.GetRoles(), permission.Id) {
+		if a.RolesGrantPermission(channelMember.GetRoles(), permission.Id) {
 			return true
 		}
 	}
@@ -193,17 +193,21 @@ func (a *App) HasPermissionToUser(askingUserId string, userId string) bool {
 	return false
 }
 
-func (a *App) CheckIfRolesGrantPermission(roles []string, permissionId string) bool {
-	for _, roleId := range roles {
-		if role := a.Role(roleId); role == nil {
-			l4g.Debug("Bad role in system " + roleId)
-			return false
-		} else {
-			permissions := role.Permissions
-			for _, permission := range permissions {
-				if permission == permissionId {
-					return true
-				}
+func (a *App) RolesGrantPermission(roleNames []string, permissionId string) bool {
+	roles, err := a.GetRolesByNames(roleNames)
+	if err != nil {
+		// This should only happen if something is very broken. We can't realistically
+		// recover the situation, so deny permission and log an error.
+		l4g.Error("Failed to get roles from database with role names: " + strings.Join(roleNames, ","))
+		l4g.Error(err)
+		return false
+	}
+
+	for _, role := range roles {
+		permissions := role.Permissions
+		for _, permission := range permissions {
+			if permission == permissionId {
+				return true
 			}
 		}
 	}
diff --git a/app/authorization_test.go b/app/authorization_test.go
index a65fe8333..2127a682e 100644
--- a/app/authorization_test.go
+++ b/app/authorization_test.go
@@ -18,9 +18,9 @@ func TestCheckIfRolesGrantPermission(t *testing.T) {
 		permissionId string
 		shouldGrant  bool
 	}{
-		{[]string{model.SYSTEM_ADMIN_ROLE_ID}, th.App.Role(model.SYSTEM_ADMIN_ROLE_ID).Permissions[0], true},
+		{[]string{model.SYSTEM_ADMIN_ROLE_ID}, model.PERMISSION_MANAGE_SYSTEM.Id, true},
 		{[]string{model.SYSTEM_ADMIN_ROLE_ID}, "non-existant-permission", false},
-		{[]string{model.CHANNEL_USER_ROLE_ID}, th.App.Role(model.CHANNEL_USER_ROLE_ID).Permissions[0], true},
+		{[]string{model.CHANNEL_USER_ROLE_ID}, model.PERMISSION_READ_CHANNEL.Id, true},
 		{[]string{model.CHANNEL_USER_ROLE_ID}, model.PERMISSION_MANAGE_SYSTEM.Id, false},
 		{[]string{model.SYSTEM_ADMIN_ROLE_ID, model.CHANNEL_USER_ROLE_ID}, model.PERMISSION_MANAGE_SYSTEM.Id, true},
 		{[]string{model.CHANNEL_USER_ROLE_ID, model.SYSTEM_ADMIN_ROLE_ID}, model.PERMISSION_MANAGE_SYSTEM.Id, true},
@@ -29,7 +29,7 @@ func TestCheckIfRolesGrantPermission(t *testing.T) {
 	}
 
 	for testnum, testcase := range cases {
-		if th.App.CheckIfRolesGrantPermission(testcase.roles, testcase.permissionId) != testcase.shouldGrant {
+		if th.App.RolesGrantPermission(testcase.roles, testcase.permissionId) != testcase.shouldGrant {
 			t.Fatal("Failed test case ", testnum)
 		}
 	}
diff --git a/app/channel.go b/app/channel.go
index 0054fe14b..c588ebda9 100644
--- a/app/channel.go
+++ b/app/channel.go
@@ -431,6 +431,10 @@ func (a *App) UpdateChannelMemberRoles(channelId string, userId string, newRoles
 		return nil, err
 	}
 
+	if err := a.CheckRolesExist(strings.Fields(newRoles)); err != nil {
+		return nil, err
+	}
+
 	member.Roles = newRoles
 
 	if result := <-a.Srv.Store.Channel().UpdateMember(member); result.Err != nil {
diff --git a/app/import_test.go b/app/import_test.go
index 6a284f63d..be2befd82 100644
--- a/app/import_test.go
+++ b/app/import_test.go
@@ -411,10 +411,6 @@ func TestImportValidateUserImportData(t *testing.T) {
 	}
 	data.Position = ptrStr("The Boss")
 
-	data.Roles = ptrStr("system_user wat")
-	if err := validateUserImportData(&data); err == nil {
-		t.Fatal("Validation should have failed due to too unrecognised role.")
-	}
 	data.Roles = nil
 	if err := validateUserImportData(&data); err != nil {
 		t.Fatal("Validation failed but should have been valid.")
@@ -478,12 +474,6 @@ func TestImportValidateUserTeamsImportData(t *testing.T) {
 	}
 	data[0].Name = ptrStr("teamname")
 
-	// Invalid Roles
-	data[0].Roles = ptrStr("wtf")
-	if err := validateUserTeamsImportData(&data); err == nil {
-		t.Fatal("Should have failed due to invalid roles.")
-	}
-
 	// Valid (nil roles)
 	data[0].Roles = nil
 	if err := validateUserTeamsImportData(&data); err != nil {
@@ -516,12 +506,6 @@ func TestImportValidateUserChannelsImportData(t *testing.T) {
 	}
 	data[0].Name = ptrStr("channelname")
 
-	// Invalid Roles
-	data[0].Roles = ptrStr("wtf")
-	if err := validateUserChannelsImportData(&data); err == nil {
-		t.Fatal("Should have failed due to invalid roles.")
-	}
-
 	// Valid (nil roles)
 	data[0].Roles = nil
 	if err := validateUserChannelsImportData(&data); err != nil {
diff --git a/app/post.go b/app/post.go
index bf4725e77..fc42a6b6b 100644
--- a/app/post.go
+++ b/app/post.go
@@ -127,7 +127,7 @@ func (a *App) CreatePost(post *model.Post, channel *model.Channel, triggerWebhoo
 	if utils.IsLicensed() && *a.Config().TeamSettings.ExperimentalTownSquareIsReadOnly &&
 		!post.IsSystemMessage() &&
 		channel.Name == model.DEFAULT_CHANNEL &&
-		!a.CheckIfRolesGrantPermission(user.GetRoles(), model.PERMISSION_MANAGE_SYSTEM.Id) {
+		!a.RolesGrantPermission(user.GetRoles(), model.PERMISSION_MANAGE_SYSTEM.Id) {
 		return nil, model.NewAppError("createPost", "api.post.create_post.town_square_read_only", nil, "", http.StatusForbidden)
 	}
 
diff --git a/app/role.go b/app/role.go
index 5f39dd623..c99d8365b 100644
--- a/app/role.go
+++ b/app/role.go
@@ -1,19 +1,91 @@
-// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// Copyright (c) 2018-present Mattermost, Inc. All Rights Reserved.
 // See License.txt for license information.
 
 package app
 
 import (
+	"reflect"
+
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/utils"
+	"net/http"
 )
 
-func (a *App) Role(id string) *model.Role {
-	return a.roles[id]
+func (a *App) GetRole(id string) (*model.Role, *model.AppError) {
+	if result := <-a.Srv.Store.Role().Get(id); result.Err != nil {
+		return nil, result.Err
+	} else {
+		return result.Data.(*model.Role), nil
+	}
+}
+
+func (a *App) GetRoleByName(name string) (*model.Role, *model.AppError) {
+	if result := <-a.Srv.Store.Role().GetByName(name); result.Err != nil {
+		return nil, result.Err
+	} else {
+		return result.Data.(*model.Role), nil
+	}
+}
+
+func (a *App) GetRolesByNames(names []string) ([]*model.Role, *model.AppError) {
+	if result := <-a.Srv.Store.Role().GetByNames(names); result.Err != nil {
+		return nil, result.Err
+	} else {
+		return result.Data.([]*model.Role), nil
+	}
+}
+
+func (a *App) PatchRole(role *model.Role, patch *model.RolePatch) (*model.Role, *model.AppError) {
+	// If patch is a no-op then short-circuit the store.
+	if patch.Permissions != nil && reflect.DeepEqual(*patch.Permissions, role.Permissions) {
+		return role, nil
+	}
+
+	role.Patch(patch)
+	role, err := a.UpdateRole(role)
+	if err != nil {
+		return nil, err
+	}
+
+	return role, err
 }
 
-// Updates the roles based on the app config and the global license check. You may need to invoke
-// this when license changes are made.
-func (a *App) SetDefaultRolesBasedOnConfig() {
-	a.roles = utils.DefaultRolesBasedOnConfig(a.Config())
+func (a *App) UpdateRole(role *model.Role) (*model.Role, *model.AppError) {
+	if result := <-a.Srv.Store.Role().Save(role); result.Err != nil {
+		return nil, result.Err
+	} else {
+		a.sendUpdatedRoleEvent(role)
+
+		return role, nil
+	}
+}
+
+func (a *App) CheckRolesExist(roleNames []string) *model.AppError {
+	roles, err := a.GetRolesByNames(roleNames)
+	if err != nil {
+		return err
+	}
+
+	for _, name := range roleNames {
+		nameFound := false
+		for _, role := range roles {
+			if name == role.Name {
+				nameFound = true
+				break
+			}
+		}
+		if !nameFound {
+			return model.NewAppError("CheckRolesExist", "app.role.check_roles_exist.role_not_found", nil, "role="+name, http.StatusBadRequest)
+		}
+	}
+
+	return nil
+}
+
+func (a *App) sendUpdatedRoleEvent(role *model.Role) {
+	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_ROLE_UPDATED, "", "", "", nil)
+	message.Add("role", role.ToJson())
+
+	a.Go(func() {
+		a.Publish(message)
+	})
 }
diff --git a/app/team.go b/app/team.go
index 71eb00569..95d9895ec 100644
--- a/app/team.go
+++ b/app/team.go
@@ -157,6 +157,10 @@ func (a *App) UpdateTeamMemberRoles(teamId string, userId string, newRoles strin
 		return nil, err
 	}
 
+	if err := a.CheckRolesExist(strings.Fields(newRoles)); err != nil {
+		return nil, err
+	}
+
 	member.Roles = newRoles
 
 	if result := <-a.Srv.Store.Team().UpdateMember(member); result.Err != nil {
diff --git a/app/user.go b/app/user.go
index 64e49e293..156503fb0 100644
--- a/app/user.go
+++ b/app/user.go
@@ -1230,6 +1230,10 @@ func (a *App) UpdateUserRoles(userId string, newRoles string, sendWebSocketEvent
 		return nil, err
 	}
 
+	if err := a.CheckRolesExist(strings.Fields(newRoles)); err != nil {
+		return nil, err
+	}
+
 	user.Roles = newRoles
 	uchan := a.Srv.Store.User().Update(user, true)
 	schan := a.Srv.Store.Session().UpdateRoles(user.Id, newRoles)