MM-8796: Full implementation of "Schemes" in Store/Model/App layers. (#8357)

* Add Scheme model and stub store.

* Port ChannelStore to be Scheme aware.

* Make almost all the API/APP layer work with ChannelSchemes.

Only thing still hacky is UpdateChannelMemberRoles().

* Add basic SchemeStore implementation.

* Migrate UpdateChannelMemberRoles properly and fix tests.

* Update store tests and mocks so they work.

* Include creating default roles in Scheme create store function.

* Implement role deletion and start scheme deletion.

* Only use non-deleted roles for authorization.

* Add GetByScheme method to Team store.

* Add GetChannelsByScheme.

* Update store mocks.

* Implement scheme deletion in the store.

* Rename is valid function.

* Add offset and limit to queries to fetch teams and channels by scheme.

* Fix queries.

* Implement scheme awareness in Team store and add a migration.

* Tidy up ChannelStore mapping functions and add exhaustive unit tests.

* Add all missing i18n.

* Proper tests for TeamStore internal functions and fix them.

* Make additional TeamMember fields nullable.

* Make new ChannelMember fields nullable.

* Create new nullable columns without defaults.

* Make new fields in large tables nullalble.

* Fix empty list of TeamMembers.

* Deduplicate SQL queries.

* Fix spelling.

* Fix review comment.

* More review fixes.

* More review fixes.

5 files changed, 149 insertions, 29 deletions

diff --git a/app/authorization.go b/app/authorization.go
index 2187472f7..6bce9e6e7 100644
--- a/app/authorization.go
+++ b/app/authorization.go
@@ -199,6 +199,10 @@ func (a *App) RolesGrantPermission(roleNames []string, permissionId string) bool
 	}
 
 	for _, role := range roles {
+		if role.DeleteAt != 0 {
+			continue
+		}
+
 		permissions := role.Permissions
 		for _, permission := range permissions {
 			if permission == permissionId {
diff --git a/app/channel.go b/app/channel.go
index 76eb4d337..c63023fb3 100644
--- a/app/channel.go
+++ b/app/channel.go
@@ -32,7 +32,7 @@ func (a *App) CreateDefaultChannels(teamId string) ([]*model.Channel, *model.App
 	return channels, nil
 }
 
-func (a *App) JoinDefaultChannels(teamId string, user *model.User, channelRole string, userRequestorId string) *model.AppError {
+func (a *App) JoinDefaultChannels(teamId string, user *model.User, shouldBeAdmin bool, userRequestorId string) *model.AppError {
 	var err *model.AppError = nil
 
 	var requestor *model.User
@@ -52,7 +52,8 @@ func (a *App) JoinDefaultChannels(teamId string, user *model.User, channelRole s
 		cm := &model.ChannelMember{
 			ChannelId:   townSquare.Id,
 			UserId:      user.Id,
-			Roles:       channelRole,
+			SchemeUser:  true,
+			SchemeAdmin: shouldBeAdmin,
 			NotifyProps: model.GetDefaultChannelNotifyProps(),
 		}
 
@@ -85,7 +86,8 @@ func (a *App) JoinDefaultChannels(teamId string, user *model.User, channelRole s
 		cm := &model.ChannelMember{
 			ChannelId:   offTopic.Id,
 			UserId:      user.Id,
-			Roles:       channelRole,
+			SchemeUser:  true,
+			SchemeAdmin: shouldBeAdmin,
 			NotifyProps: model.GetDefaultChannelNotifyProps(),
 		}
 
@@ -166,7 +168,8 @@ func (a *App) CreateChannel(channel *model.Channel, addMember bool) (*model.Chan
 			cm := &model.ChannelMember{
 				ChannelId:   sc.Id,
 				UserId:      channel.CreatorId,
-				Roles:       model.CHANNEL_USER_ROLE_ID + " " + model.CHANNEL_ADMIN_ROLE_ID,
+				SchemeUser:  true,
+				SchemeAdmin: true,
 				NotifyProps: model.GetDefaultChannelNotifyProps(),
 			}
 
@@ -322,7 +325,7 @@ func (a *App) createGroupChannel(userIds []string, creatorId string) (*model.Cha
 				UserId:      user.Id,
 				ChannelId:   group.Id,
 				NotifyProps: model.GetDefaultChannelNotifyProps(),
-				Roles:       model.CHANNEL_USER_ROLE_ID,
+				SchemeUser:  true,
 			}
 
 			if result := <-a.Srv.Store.Channel().SaveMember(cm); result.Err != nil {
@@ -432,6 +435,39 @@ func (a *App) PatchChannel(channel *model.Channel, patch *model.ChannelPatch, us
 	return channel, err
 }
 
+func (a *App) GetSchemeRolesForChannel(channelId string) (string, string, *model.AppError) {
+	var channel *model.Channel
+	var err *model.AppError
+
+	if channel, err = a.GetChannel(channelId); err != nil {
+		return "", "", err
+	}
+
+	if channel.SchemeId != nil && len(*channel.SchemeId) != 0 {
+		if scheme, err := a.GetScheme(*channel.SchemeId); err != nil {
+			return "", "", err
+		} else {
+			return scheme.DefaultChannelUserRole, scheme.DefaultChannelAdminRole, nil
+		}
+	}
+
+	var team *model.Team
+
+	if team, err = a.GetTeam(channel.TeamId); err != nil {
+		return "", "", err
+	}
+
+	if team.SchemeId != nil && len(*team.SchemeId) != 0 {
+		if scheme, err := a.GetScheme(*team.SchemeId); err != nil {
+			return "", "", err
+		} else {
+			return scheme.DefaultChannelUserRole, scheme.DefaultChannelAdminRole, nil
+		}
+	}
+
+	return model.CHANNEL_USER_ROLE_ID, model.CHANNEL_ADMIN_ROLE_ID, nil
+}
+
 func (a *App) UpdateChannelMemberRoles(channelId string, userId string, newRoles string) (*model.ChannelMember, *model.AppError) {
 	var member *model.ChannelMember
 	var err *model.AppError
@@ -439,14 +475,42 @@ func (a *App) UpdateChannelMemberRoles(channelId string, userId string, newRoles
 		return nil, err
 	}
 
-	if err := a.CheckRolesExist(strings.Fields(newRoles)); err != nil {
+	schemeUserRole, schemeAdminRole, err := a.GetSchemeRolesForChannel(channelId)
+	if err != nil {
 		return nil, err
 	}
 
-	member.Roles = newRoles
+	var newExplicitRoles []string
+	member.SchemeUser = false
+	member.SchemeAdmin = false
+
+	for _, roleName := range strings.Fields(newRoles) {
+		if role, err := a.GetRoleByName(roleName); err != nil {
+			err.StatusCode = http.StatusBadRequest
+			return nil, err
+		} else if !role.SchemeManaged {
+			// The role is not scheme-managed, so it's OK to apply it to the explicit roles field.
+			newExplicitRoles = append(newExplicitRoles, roleName)
+		} else {
+			// The role is scheme-managed, so need to check if it is part of the scheme for this channel or not.
+			switch roleName {
+			case schemeAdminRole:
+				member.SchemeAdmin = true
+			case schemeUserRole:
+				member.SchemeUser = true
+			default:
+				// If not part of the scheme for this channel, then it is not allowed to apply it as an explicit role.
+				return nil, model.NewAppError("UpdateChannelMemberRoles", "api.channel.update_channel_member_roles.scheme_role.app_error", nil, "role_name="+roleName, http.StatusBadRequest)
+			}
+		}
+	}
+
+	member.ExplicitRoles = strings.Join(newExplicitRoles, " ")
 
 	if result := <-a.Srv.Store.Channel().UpdateMember(member); result.Err != nil {
 		return nil, result.Err
+	} else {
+		member = result.Data.(*model.ChannelMember)
 	}
 
 	a.InvalidateCacheForUser(userId)
@@ -591,7 +655,7 @@ func (a *App) addUserToChannel(user *model.User, channel *model.Channel, teamMem
 		ChannelId:   channel.Id,
 		UserId:      user.Id,
 		NotifyProps: model.GetDefaultChannelNotifyProps(),
-		Roles:       model.CHANNEL_USER_ROLE_ID,
+		SchemeUser:  true,
 	}
 	if result := <-a.Srv.Store.Channel().SaveMember(newMember); result.Err != nil {
 		l4g.Error("Failed to add member user_id=%v channel_id=%v err=%v", user.Id, channel.Id, result.Err)
diff --git a/app/channel_test.go b/app/channel_test.go
index a4e0806a6..de8a6a6a0 100644
--- a/app/channel_test.go
+++ b/app/channel_test.go
@@ -120,7 +120,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordTownSquare(t *testi
 
 	// create a new user that joins the default channels
 	user := th.CreateUser()
-	th.App.JoinDefaultChannels(th.BasicTeam.Id, user, model.CHANNEL_USER_ROLE_ID, "")
+	th.App.JoinDefaultChannels(th.BasicTeam.Id, user, false, "")
 
 	// there should be a ChannelMemberHistory record for the user
 	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, townSquareChannelId)).([]*model.ChannelMemberHistoryResult)
@@ -146,7 +146,7 @@ func TestJoinDefaultChannelsCreatesChannelMemberHistoryRecordOffTopic(t *testing
 
 	// create a new user that joins the default channels
 	user := th.CreateUser()
-	th.App.JoinDefaultChannels(th.BasicTeam.Id, user, model.CHANNEL_USER_ROLE_ID, "")
+	th.App.JoinDefaultChannels(th.BasicTeam.Id, user, false, "")
 
 	// there should be a ChannelMemberHistory record for the user
 	histories := store.Must(th.App.Srv.Store.ChannelMemberHistory().GetUsersInChannelDuring(model.GetMillis()-100, model.GetMillis()+100, offTopicChannelId)).([]*model.ChannelMemberHistoryResult)
diff --git a/app/scheme.go b/app/scheme.go
new file mode 100644
index 000000000..26ec6cd2a
--- /dev/null
+++ b/app/scheme.go
@@ -0,0 +1,14 @@
+// Copyright (c) 2018-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import "github.com/mattermost/mattermost-server/model"
+
+func (a *App) GetScheme(id string) (*model.Scheme, *model.AppError) {
+	if result := <-a.Srv.Store.Scheme().Get(id); result.Err != nil {
+		return nil, result.Err
+	} else {
+		return result.Data.(*model.Scheme), nil
+	}
+}
diff --git a/app/team.go b/app/team.go
index 47e28f2ed..3b30816be 100644
--- a/app/team.go
+++ b/app/team.go
@@ -141,17 +141,31 @@ func (a *App) sendTeamEvent(team *model.Team, event string) {
 	a.Publish(message)
 }
 
+func (a *App) GetSchemeRolesForTeam(teamId string) (string, string, *model.AppError) {
+	var team *model.Team
+	var err *model.AppError
+
+	if team, err = a.GetTeam(teamId); err != nil {
+		return "", "", err
+	}
+
+	if team.SchemeId != nil && len(*team.SchemeId) != 0 {
+		if scheme, err := a.GetScheme(*team.SchemeId); err != nil {
+			return "", "", err
+		} else {
+			return scheme.DefaultTeamUserRole, scheme.DefaultTeamAdminRole, nil
+		}
+	}
+
+	return model.TEAM_USER_ROLE_ID, model.TEAM_ADMIN_ROLE_ID, nil
+}
+
 func (a *App) UpdateTeamMemberRoles(teamId string, userId string, newRoles string) (*model.TeamMember, *model.AppError) {
 	var member *model.TeamMember
-	if result := <-a.Srv.Store.Team().GetTeamsForUser(userId); result.Err != nil {
+	if result := <-a.Srv.Store.Team().GetMember(teamId, userId); result.Err != nil {
 		return nil, result.Err
 	} else {
-		members := result.Data.([]*model.TeamMember)
-		for _, m := range members {
-			if m.TeamId == teamId {
-				member = m
-			}
-		}
+		member = result.Data.(*model.TeamMember)
 	}
 
 	if member == nil {
@@ -159,14 +173,42 @@ func (a *App) UpdateTeamMemberRoles(teamId string, userId string, newRoles strin
 		return nil, err
 	}
 
-	if err := a.CheckRolesExist(strings.Fields(newRoles)); err != nil {
+	schemeUserRole, schemeAdminRole, err := a.GetSchemeRolesForTeam(teamId)
+	if err != nil {
 		return nil, err
 	}
 
-	member.Roles = newRoles
+	var newExplicitRoles []string
+	member.SchemeUser = false
+	member.SchemeAdmin = false
+
+	for _, roleName := range strings.Fields(newRoles) {
+		if role, err := a.GetRoleByName(roleName); err != nil {
+			err.StatusCode = http.StatusBadRequest
+			return nil, err
+		} else if !role.SchemeManaged {
+			// The role is not scheme-managed, so it's OK to apply it to the explicit roles field.
+			newExplicitRoles = append(newExplicitRoles, roleName)
+		} else {
+			// The role is scheme-managed, so need to check if it is part of the scheme for this channel or not.
+			switch roleName {
+			case schemeAdminRole:
+				member.SchemeAdmin = true
+			case schemeUserRole:
+				member.SchemeUser = true
+			default:
+				// If not part of the scheme for this channel, then it is not allowed to apply it as an explicit role.
+				return nil, model.NewAppError("UpdateTeamMemberRoles", "api.channel.update_team_member_roles.scheme_role.app_error", nil, "role_name="+roleName, http.StatusBadRequest)
+			}
+		}
+	}
+
+	member.ExplicitRoles = strings.Join(newExplicitRoles, " ")
 
 	if result := <-a.Srv.Store.Team().UpdateMember(member); result.Err != nil {
 		return nil, result.Err
+	} else {
+		member = result.Data.(*model.TeamMember)
 	}
 
 	a.ClearSessionCacheForUser(userId)
@@ -292,13 +334,13 @@ func (a *App) AddUserToTeamByInviteId(inviteId string, userId string) (*model.Te
 // 3. a pointer to an AppError if something went wrong.
 func (a *App) joinUserToTeam(team *model.Team, user *model.User) (*model.TeamMember, bool, *model.AppError) {
 	tm := &model.TeamMember{
-		TeamId: team.Id,
-		UserId: user.Id,
-		Roles:  model.TEAM_USER_ROLE_ID,
+		TeamId:     team.Id,
+		UserId:     user.Id,
+		SchemeUser: true,
 	}
 
 	if team.Email == user.Email {
-		tm.Roles = model.TEAM_USER_ROLE_ID + " " + model.TEAM_ADMIN_ROLE_ID
+		tm.SchemeAdmin = true
 	}
 
 	if etmr := <-a.Srv.Store.Team().GetMember(team.Id, user.Id); etmr.Err == nil {
@@ -342,14 +384,10 @@ func (a *App) JoinUserToTeam(team *model.Team, user *model.User, userRequestorId
 		return uua.Err
 	}
 
-	channelRole := model.CHANNEL_USER_ROLE_ID
-
-	if team.Email == user.Email {
-		channelRole = model.CHANNEL_USER_ROLE_ID + " " + model.CHANNEL_ADMIN_ROLE_ID
-	}
+	shouldBeAdmin := team.Email == user.Email
 
 	// Soft error if there is an issue joining the default channels
-	if err := a.JoinDefaultChannels(team.Id, user, channelRole, userRequestorId); err != nil {
+	if err := a.JoinDefaultChannels(team.Id, user, shouldBeAdmin, userRequestorId); err != nil {
 		l4g.Error(utils.T("api.user.create_user.joining.error"), user.Id, team.Id, err)
 	}