MM-8796: Full implementation of "Schemes" in Store/Model/App layers. (#8357)

* Add Scheme model and stub store.

* Port ChannelStore to be Scheme aware.

* Make almost all the API/APP layer work with ChannelSchemes.

Only thing still hacky is UpdateChannelMemberRoles().

* Add basic SchemeStore implementation.

* Migrate UpdateChannelMemberRoles properly and fix tests.

* Update store tests and mocks so they work.

* Include creating default roles in Scheme create store function.

* Implement role deletion and start scheme deletion.

* Only use non-deleted roles for authorization.

* Add GetByScheme method to Team store.

* Add GetChannelsByScheme.

* Update store mocks.

* Implement scheme deletion in the store.

* Rename is valid function.

* Add offset and limit to queries to fetch teams and channels by scheme.

* Fix queries.

* Implement scheme awareness in Team store and add a migration.

* Tidy up ChannelStore mapping functions and add exhaustive unit tests.

* Add all missing i18n.

* Proper tests for TeamStore internal functions and fix them.

* Make additional TeamMember fields nullable.

* Make new ChannelMember fields nullable.

* Create new nullable columns without defaults.

* Make new fields in large tables nullalble.

* Fix empty list of TeamMembers.

* Deduplicate SQL queries.

* Fix spelling.

* Fix review comment.

* More review fixes.

* More review fixes.

7 files changed, 161 insertions, 45 deletions

diff --git a/model/channel.go b/model/channel.go
index df68202d6..29ec67ed6 100644
--- a/model/channel.go
+++ b/model/channel.go
@@ -32,20 +32,21 @@ const (
 )
 
 type Channel struct {
-	Id            string `json:"id"`
-	CreateAt      int64  `json:"create_at"`
-	UpdateAt      int64  `json:"update_at"`
-	DeleteAt      int64  `json:"delete_at"`
-	TeamId        string `json:"team_id"`
-	Type          string `json:"type"`
-	DisplayName   string `json:"display_name"`
-	Name          string `json:"name"`
-	Header        string `json:"header"`
-	Purpose       string `json:"purpose"`
-	LastPostAt    int64  `json:"last_post_at"`
-	TotalMsgCount int64  `json:"total_msg_count"`
-	ExtraUpdateAt int64  `json:"extra_update_at"`
-	CreatorId     string `json:"creator_id"`
+	Id            string  `json:"id"`
+	CreateAt      int64   `json:"create_at"`
+	UpdateAt      int64   `json:"update_at"`
+	DeleteAt      int64   `json:"delete_at"`
+	TeamId        string  `json:"team_id"`
+	Type          string  `json:"type"`
+	DisplayName   string  `json:"display_name"`
+	Name          string  `json:"name"`
+	Header        string  `json:"header"`
+	Purpose       string  `json:"purpose"`
+	LastPostAt    int64   `json:"last_post_at"`
+	TotalMsgCount int64   `json:"total_msg_count"`
+	ExtraUpdateAt int64   `json:"extra_update_at"`
+	CreatorId     string  `json:"creator_id"`
+	SchemeId      *string `json:"scheme_id"`
 }
 
 type ChannelPatch struct {
diff --git a/model/channel_member.go b/model/channel_member.go
index e9895aea0..709ad3ccd 100644
--- a/model/channel_member.go
+++ b/model/channel_member.go
@@ -28,14 +28,17 @@ type ChannelUnread struct {
 }
 
 type ChannelMember struct {
-	ChannelId    string    `json:"channel_id"`
-	UserId       string    `json:"user_id"`
-	Roles        string    `json:"roles"`
-	LastViewedAt int64     `json:"last_viewed_at"`
-	MsgCount     int64     `json:"msg_count"`
-	MentionCount int64     `json:"mention_count"`
-	NotifyProps  StringMap `json:"notify_props"`
-	LastUpdateAt int64     `json:"last_update_at"`
+	ChannelId     string    `json:"channel_id"`
+	UserId        string    `json:"user_id"`
+	Roles         string    `json:"roles"`
+	LastViewedAt  int64     `json:"last_viewed_at"`
+	MsgCount      int64     `json:"msg_count"`
+	MentionCount  int64     `json:"mention_count"`
+	NotifyProps   StringMap `json:"notify_props"`
+	LastUpdateAt  int64     `json:"last_update_at"`
+	SchemeUser    bool      `json:"scheme_user"`
+	SchemeAdmin   bool      `json:"scheme_admin"`
+	ExplicitRoles string    `json:"explicit_roles"`
 }
 
 type ChannelMembers []ChannelMember
diff --git a/model/cluster_message.go b/model/cluster_message.go
index cf9e3f9f2..d02da3ee1 100644
--- a/model/cluster_message.go
+++ b/model/cluster_message.go
@@ -22,6 +22,7 @@ const (
 	CLUSTER_EVENT_INVALIDATE_CACHE_FOR_USER                         = "inv_user"
 	CLUSTER_EVENT_CLEAR_SESSION_CACHE_FOR_USER                      = "clear_session_user"
 	CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES                        = "inv_roles"
+	CLUSTER_EVENT_INVALIDATE_CACHE_FOR_SCHEMES                      = "inv_schemes"
 
 	CLUSTER_SEND_BEST_EFFORT = "best_effort"
 	CLUSTER_SEND_RELIABLE    = "reliable"
diff --git a/model/role.go b/model/role.go
index f10b52537..80ae1ae34 100644
--- a/model/role.go
+++ b/model/role.go
@@ -39,6 +39,7 @@ type Role struct {
 	DeleteAt      int64    `json:"delete_at"`
 	Permissions   []string `json:"permissions"`
 	SchemeManaged bool     `json:"scheme_managed"`
+	BuiltIn       bool     `json:"built_in"`
 }
 
 type RolePatch struct {
@@ -187,6 +188,7 @@ func MakeDefaultRoles() map[string]*Role {
 			PERMISSION_USE_SLASH_COMMANDS.Id,
 		},
 		SchemeManaged: true,
+		BuiltIn:       true,
 	}
 
 	roles[CHANNEL_ADMIN_ROLE_ID] = &Role{
@@ -197,6 +199,7 @@ func MakeDefaultRoles() map[string]*Role {
 			PERMISSION_MANAGE_CHANNEL_ROLES.Id,
 		},
 		SchemeManaged: true,
+		BuiltIn:       true,
 	}
 
 	roles[TEAM_USER_ROLE_ID] = &Role{
@@ -210,6 +213,7 @@ func MakeDefaultRoles() map[string]*Role {
 			PERMISSION_VIEW_TEAM.Id,
 		},
 		SchemeManaged: true,
+		BuiltIn:       true,
 	}
 
 	roles[TEAM_POST_ALL_ROLE_ID] = &Role{
@@ -219,7 +223,8 @@ func MakeDefaultRoles() map[string]*Role {
 		Permissions: []string{
 			PERMISSION_CREATE_POST.Id,
 		},
-		SchemeManaged: true,
+		SchemeManaged: false,
+		BuiltIn:       true,
 	}
 
 	roles[TEAM_POST_ALL_PUBLIC_ROLE_ID] = &Role{
@@ -229,7 +234,8 @@ func MakeDefaultRoles() map[string]*Role {
 		Permissions: []string{
 			PERMISSION_CREATE_POST_PUBLIC.Id,
 		},
-		SchemeManaged: true,
+		SchemeManaged: false,
+		BuiltIn:       true,
 	}
 
 	roles[TEAM_ADMIN_ROLE_ID] = &Role{
@@ -249,6 +255,7 @@ func MakeDefaultRoles() map[string]*Role {
 			PERMISSION_MANAGE_WEBHOOKS.Id,
 		},
 		SchemeManaged: true,
+		BuiltIn:       true,
 	}
 
 	roles[SYSTEM_USER_ROLE_ID] = &Role{
@@ -261,6 +268,7 @@ func MakeDefaultRoles() map[string]*Role {
 			PERMISSION_PERMANENT_DELETE_USER.Id,
 		},
 		SchemeManaged: true,
+		BuiltIn:       true,
 	}
 
 	roles[SYSTEM_POST_ALL_ROLE_ID] = &Role{
@@ -270,7 +278,8 @@ func MakeDefaultRoles() map[string]*Role {
 		Permissions: []string{
 			PERMISSION_CREATE_POST.Id,
 		},
-		SchemeManaged: true,
+		SchemeManaged: false,
+		BuiltIn:       true,
 	}
 
 	roles[SYSTEM_POST_ALL_PUBLIC_ROLE_ID] = &Role{
@@ -280,7 +289,8 @@ func MakeDefaultRoles() map[string]*Role {
 		Permissions: []string{
 			PERMISSION_CREATE_POST_PUBLIC.Id,
 		},
-		SchemeManaged: true,
+		SchemeManaged: false,
+		BuiltIn:       true,
 	}
 
 	roles[SYSTEM_USER_ACCESS_TOKEN_ROLE_ID] = &Role{
@@ -292,7 +302,8 @@ func MakeDefaultRoles() map[string]*Role {
 			PERMISSION_READ_USER_ACCESS_TOKEN.Id,
 			PERMISSION_REVOKE_USER_ACCESS_TOKEN.Id,
 		},
-		SchemeManaged: true,
+		SchemeManaged: false,
+		BuiltIn:       true,
 	}
 
 	roles[SYSTEM_ADMIN_ROLE_ID] = &Role{
@@ -345,6 +356,7 @@ func MakeDefaultRoles() map[string]*Role {
 			roles[CHANNEL_ADMIN_ROLE_ID].Permissions...,
 		),
 		SchemeManaged: true,
+		BuiltIn:       true,
 	}
 
 	return roles
diff --git a/model/scheme.go b/model/scheme.go
new file mode 100644
index 000000000..9ad153c73
--- /dev/null
+++ b/model/scheme.go
@@ -0,0 +1,95 @@
+// Copyright (c) 2018-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package model
+
+import (
+	"encoding/json"
+	"io"
+)
+
+const (
+	SCHEME_NAME_MAX_LENGTH        = 64
+	SCHEME_DESCRIPTION_MAX_LENGTH = 1024
+	SCHEME_SCOPE_TEAM             = "team"
+	SCHEME_SCOPE_CHANNEL          = "channel"
+)
+
+type Scheme struct {
+	Id                      string `json:"id"`
+	Name                    string `json:"name"`
+	Description             string `json:"description"`
+	CreateAt                int64  `json:"create_at"`
+	UpdateAt                int64  `json:"update_at"`
+	DeleteAt                int64  `json:"delete_at"`
+	Scope                   string `json:"scope"`
+	DefaultTeamAdminRole    string `json:"default_team_admin_role"`
+	DefaultTeamUserRole     string `json:"default_team_user_role"`
+	DefaultChannelAdminRole string `json:"default_channel_admin_role"`
+	DefaultChannelUserRole  string `json:"default_channel_user_role"`
+}
+
+func (scheme *Scheme) ToJson() string {
+	b, _ := json.Marshal(scheme)
+	return string(b)
+}
+
+func SchemeFromJson(data io.Reader) *Scheme {
+	var scheme *Scheme
+	json.NewDecoder(data).Decode(&scheme)
+	return scheme
+}
+
+func (scheme *Scheme) IsValid() bool {
+	if len(scheme.Id) != 26 {
+		return false
+	}
+
+	return scheme.IsValidForCreate()
+}
+
+func (scheme *Scheme) IsValidForCreate() bool {
+	if len(scheme.Name) == 0 || len(scheme.Name) > SCHEME_NAME_MAX_LENGTH {
+		return false
+	}
+
+	if len(scheme.Description) > SCHEME_DESCRIPTION_MAX_LENGTH {
+		return false
+	}
+
+	switch scheme.Scope {
+	case SCHEME_SCOPE_TEAM, SCHEME_SCOPE_CHANNEL:
+	default:
+		return false
+	}
+
+	if !IsValidRoleName(scheme.DefaultChannelAdminRole) {
+		return false
+	}
+
+	if !IsValidRoleName(scheme.DefaultChannelUserRole) {
+		return false
+	}
+
+	if scheme.Scope == SCHEME_SCOPE_TEAM {
+		if !IsValidRoleName(scheme.DefaultTeamAdminRole) {
+			return false
+		}
+
+		if !IsValidRoleName(scheme.DefaultTeamUserRole) {
+			return false
+		}
+	}
+
+	if scheme.Scope == SCHEME_SCOPE_CHANNEL {
+		if len(scheme.DefaultTeamAdminRole) != 0 {
+			return false
+		}
+
+		if len(scheme.DefaultTeamUserRole) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
diff --git a/model/team.go b/model/team.go
index 7968c9d48..edf9d3a41 100644
--- a/model/team.go
+++ b/model/team.go
@@ -26,20 +26,21 @@ const (
 )
 
 type Team struct {
-	Id                 string `json:"id"`
-	CreateAt           int64  `json:"create_at"`
-	UpdateAt           int64  `json:"update_at"`
-	DeleteAt           int64  `json:"delete_at"`
-	DisplayName        string `json:"display_name"`
-	Name               string `json:"name"`
-	Description        string `json:"description"`
-	Email              string `json:"email"`
-	Type               string `json:"type"`
-	CompanyName        string `json:"company_name"`
-	AllowedDomains     string `json:"allowed_domains"`
-	InviteId           string `json:"invite_id"`
-	AllowOpenInvite    bool   `json:"allow_open_invite"`
-	LastTeamIconUpdate int64  `json:"last_team_icon_update,omitempty"`
+	Id                 string  `json:"id"`
+	CreateAt           int64   `json:"create_at"`
+	UpdateAt           int64   `json:"update_at"`
+	DeleteAt           int64   `json:"delete_at"`
+	DisplayName        string  `json:"display_name"`
+	Name               string  `json:"name"`
+	Description        string  `json:"description"`
+	Email              string  `json:"email"`
+	Type               string  `json:"type"`
+	CompanyName        string  `json:"company_name"`
+	AllowedDomains     string  `json:"allowed_domains"`
+	InviteId           string  `json:"invite_id"`
+	AllowOpenInvite    bool    `json:"allow_open_invite"`
+	LastTeamIconUpdate int64   `json:"last_team_icon_update,omitempty"`
+	SchemeId           *string `json:"scheme_id"`
 }
 
 type TeamPatch struct {
diff --git a/model/team_member.go b/model/team_member.go
index 2fcd1e151..0bda96121 100644
--- a/model/team_member.go
+++ b/model/team_member.go
@@ -11,10 +11,13 @@ import (
 )
 
 type TeamMember struct {
-	TeamId   string `json:"team_id"`
-	UserId   string `json:"user_id"`
-	Roles    string `json:"roles"`
-	DeleteAt int64  `json:"delete_at"`
+	TeamId        string `json:"team_id"`
+	UserId        string `json:"user_id"`
+	Roles         string `json:"roles"`
+	DeleteAt      int64  `json:"delete_at"`
+	SchemeUser    bool   `json:"scheme_user"`
+	SchemeAdmin   bool   `json:"scheme_admin"`
+	ExplicitRoles string `json:"explicit_roles"`
 }
 
 type TeamUnread struct {