MM-8622: Improved plugin error reporting (#8737)

* allow `Wait()`ing on the supervisor

In the event the plugin supervisor shuts down a plugin for crashing too
many times, the new `Wait()` interface allows the `ActivatePlugin` to
accept a callback function to trigger when `supervisor.Wait()` returns.
If the supervisor shuts down normally, this callback is invoked with
a nil error, otherwise any error reported by the supervisor is passed
along.

* improve plugin activation/deactivation logic

Avoid triggering activation of previously failed-to-start plugins just
becase something in the configuration changed. Now, intelligently
compare the global enable bit as well as the each individual plugin's
enabled bit.

* expose store to manipulate PluginStatuses

* expose API to fetch plugin statuses

* keep track of whether or not plugin sandboxing is supported

* transition plugin statuses

* restore error on plugin activation if already active

* don't initialize test plugins until successfully loaded

* emit websocket events when plugin statuses change

* skip pruning if already initialized

* MM-8622: maintain plugin statuses in memory

Switch away from persisting plugin statuses to the database, and
maintain in memory instead. This will be followed by a cluster interface
to query the in-memory status of plugin statuses from all cluster nodes.

At the same time, rename `cluster_discovery_id` on the `PluginStatus`
model object to `cluster_id`.

* MM-8622: aggregate plugin statuses across cluster

* fetch cluster plugin statuses when emitting websocket notification

* address unit test fixes after rebasing

* relax (poor) racey unit test re: supervisor.Wait()

* make store-mocks

4 files changed, 96 insertions, 39 deletions

diff --git a/model/client4.go b/model/client4.go
index d245fe6c0..97dd30790 100644
--- a/model/client4.go
+++ b/model/client4.go
@@ -3534,6 +3534,18 @@ func (c *Client4) GetPlugins() (*PluginsResponse, *Response) {
 	}
 }
 
+// GetPluginStatuses will return the plugins installed on any server in the cluster, for reporting
+// to the administrator via the system console.
+// WARNING: PLUGINS ARE STILL EXPERIMENTAL. THIS FUNCTION IS SUBJECT TO CHANGE.
+func (c *Client4) GetPluginStatuses() (PluginStatuses, *Response) {
+	if r, err := c.DoApiGet(c.GetPluginsRoute(), "/statuses"); err != nil {
+		return nil, BuildErrorResponse(r, err)
+	} else {
+		defer closeBody(r)
+		return PluginStatusesFromJson(r.Body), BuildResponse(r)
+	}
+}
+
 // RemovePlugin will deactivate and delete a plugin.
 // WARNING: PLUGINS ARE STILL EXPERIMENTAL. THIS FUNCTION IS SUBJECT TO CHANGE.
 func (c *Client4) RemovePlugin(id string) (bool, *Response) {
diff --git a/model/cluster_discovery.go b/model/cluster_discovery.go
index 89e5fc95e..5d5b0465d 100644
--- a/model/cluster_discovery.go
+++ b/model/cluster_discovery.go
@@ -86,7 +86,7 @@ func FilterClusterDiscovery(vs []*ClusterDiscovery, f func(*ClusterDiscovery) bo
 
 func (o *ClusterDiscovery) IsValid() *AppError {
 	if len(o.Id) != 26 {
-		return NewAppError("Channel.IsValid", "model.channel.is_valid.id.app_error", nil, "", http.StatusBadRequest)
+		return NewAppError("ClusterDiscovery.IsValid", "model.cluster.is_valid.id.app_error", nil, "", http.StatusBadRequest)
 	}
 
 	if len(o.ClusterName) == 0 {
diff --git a/model/plugin_status.go b/model/plugin_status.go
new file mode 100644
index 000000000..1ae64ff89
--- /dev/null
+++ b/model/plugin_status.go
@@ -0,0 +1,44 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package model
+
+import (
+	"encoding/json"
+	"io"
+)
+
+const (
+	PluginStateNotRunning          = 0
+	PluginStateStarting            = 1
+	PluginStateRunning             = 2
+	PluginStateFailedToStart       = 3
+	PluginStateFailedToStayRunning = 4
+	PluginStateStopping            = 5
+)
+
+// PluginStatus provides a cluster-aware view of installed plugins.
+type PluginStatus struct {
+	PluginId      string `json:"plugin_id"`
+	ClusterId     string `json:"cluster_id"`
+	PluginPath    string `json:"plugin_path"`
+	State         int    `json:"state"`
+	IsSandboxed   bool   `json:"is_sandboxed"`
+	IsPrepackaged bool   `json:"is_prepackaged"`
+	Name          string `json:"name"`
+	Description   string `json:"description"`
+	Version       string `json:"version"`
+}
+
+type PluginStatuses []*PluginStatus
+
+func (m *PluginStatuses) ToJson() string {
+	b, _ := json.Marshal(m)
+	return string(b)
+}
+
+func PluginStatusesFromJson(data io.Reader) PluginStatuses {
+	var m PluginStatuses
+	json.NewDecoder(data).Decode(&m)
+	return m
+}
diff --git a/model/websocket_message.go b/model/websocket_message.go
index 08c238480..071975d6c 100644
--- a/model/websocket_message.go
+++ b/model/websocket_message.go
@@ -10,44 +10,45 @@ import (
 )
 
 const (
-	WEBSOCKET_EVENT_TYPING                 = "typing"
-	WEBSOCKET_EVENT_POSTED                 = "posted"
-	WEBSOCKET_EVENT_POST_EDITED            = "post_edited"
-	WEBSOCKET_EVENT_POST_DELETED           = "post_deleted"
-	WEBSOCKET_EVENT_CHANNEL_DELETED        = "channel_deleted"
-	WEBSOCKET_EVENT_CHANNEL_CREATED        = "channel_created"
-	WEBSOCKET_EVENT_CHANNEL_UPDATED        = "channel_updated"
-	WEBSOCKET_EVENT_CHANNEL_MEMBER_UPDATED = "channel_member_updated"
-	WEBSOCKET_EVENT_DIRECT_ADDED           = "direct_added"
-	WEBSOCKET_EVENT_GROUP_ADDED            = "group_added"
-	WEBSOCKET_EVENT_NEW_USER               = "new_user"
-	WEBSOCKET_EVENT_ADDED_TO_TEAM          = "added_to_team"
-	WEBSOCKET_EVENT_LEAVE_TEAM             = "leave_team"
-	WEBSOCKET_EVENT_UPDATE_TEAM            = "update_team"
-	WEBSOCKET_EVENT_DELETE_TEAM            = "delete_team"
-	WEBSOCKET_EVENT_USER_ADDED             = "user_added"
-	WEBSOCKET_EVENT_USER_UPDATED           = "user_updated"
-	WEBSOCKET_EVENT_USER_ROLE_UPDATED      = "user_role_updated"
-	WEBSOCKET_EVENT_MEMBERROLE_UPDATED     = "memberrole_updated"
-	WEBSOCKET_EVENT_USER_REMOVED           = "user_removed"
-	WEBSOCKET_EVENT_PREFERENCE_CHANGED     = "preference_changed"
-	WEBSOCKET_EVENT_PREFERENCES_CHANGED    = "preferences_changed"
-	WEBSOCKET_EVENT_PREFERENCES_DELETED    = "preferences_deleted"
-	WEBSOCKET_EVENT_EPHEMERAL_MESSAGE      = "ephemeral_message"
-	WEBSOCKET_EVENT_STATUS_CHANGE          = "status_change"
-	WEBSOCKET_EVENT_HELLO                  = "hello"
-	WEBSOCKET_EVENT_WEBRTC                 = "webrtc"
-	WEBSOCKET_AUTHENTICATION_CHALLENGE     = "authentication_challenge"
-	WEBSOCKET_EVENT_REACTION_ADDED         = "reaction_added"
-	WEBSOCKET_EVENT_REACTION_REMOVED       = "reaction_removed"
-	WEBSOCKET_EVENT_RESPONSE               = "response"
-	WEBSOCKET_EVENT_EMOJI_ADDED            = "emoji_added"
-	WEBSOCKET_EVENT_CHANNEL_VIEWED         = "channel_viewed"
-	WEBSOCKET_EVENT_PLUGIN_ACTIVATED       = "plugin_activated"   // EXPERIMENTAL - SUBJECT TO CHANGE
-	WEBSOCKET_EVENT_PLUGIN_DEACTIVATED     = "plugin_deactivated" // EXPERIMENTAL - SUBJECT TO CHANGE
-	WEBSOCKET_EVENT_ROLE_UPDATED           = "role_updated"
-	WEBSOCKET_EVENT_LICENSE_CHANGED        = "license_changed"
-	WEBSOCKET_EVENT_CONFIG_CHANGED         = "config_changed"
+	WEBSOCKET_EVENT_TYPING                  = "typing"
+	WEBSOCKET_EVENT_POSTED                  = "posted"
+	WEBSOCKET_EVENT_POST_EDITED             = "post_edited"
+	WEBSOCKET_EVENT_POST_DELETED            = "post_deleted"
+	WEBSOCKET_EVENT_CHANNEL_DELETED         = "channel_deleted"
+	WEBSOCKET_EVENT_CHANNEL_CREATED         = "channel_created"
+	WEBSOCKET_EVENT_CHANNEL_UPDATED         = "channel_updated"
+	WEBSOCKET_EVENT_CHANNEL_MEMBER_UPDATED  = "channel_member_updated"
+	WEBSOCKET_EVENT_DIRECT_ADDED            = "direct_added"
+	WEBSOCKET_EVENT_GROUP_ADDED             = "group_added"
+	WEBSOCKET_EVENT_NEW_USER                = "new_user"
+	WEBSOCKET_EVENT_ADDED_TO_TEAM           = "added_to_team"
+	WEBSOCKET_EVENT_LEAVE_TEAM              = "leave_team"
+	WEBSOCKET_EVENT_UPDATE_TEAM             = "update_team"
+	WEBSOCKET_EVENT_DELETE_TEAM             = "delete_team"
+	WEBSOCKET_EVENT_USER_ADDED              = "user_added"
+	WEBSOCKET_EVENT_USER_UPDATED            = "user_updated"
+	WEBSOCKET_EVENT_USER_ROLE_UPDATED       = "user_role_updated"
+	WEBSOCKET_EVENT_MEMBERROLE_UPDATED      = "memberrole_updated"
+	WEBSOCKET_EVENT_USER_REMOVED            = "user_removed"
+	WEBSOCKET_EVENT_PREFERENCE_CHANGED      = "preference_changed"
+	WEBSOCKET_EVENT_PREFERENCES_CHANGED     = "preferences_changed"
+	WEBSOCKET_EVENT_PREFERENCES_DELETED     = "preferences_deleted"
+	WEBSOCKET_EVENT_EPHEMERAL_MESSAGE       = "ephemeral_message"
+	WEBSOCKET_EVENT_STATUS_CHANGE           = "status_change"
+	WEBSOCKET_EVENT_HELLO                   = "hello"
+	WEBSOCKET_EVENT_WEBRTC                  = "webrtc"
+	WEBSOCKET_AUTHENTICATION_CHALLENGE      = "authentication_challenge"
+	WEBSOCKET_EVENT_REACTION_ADDED          = "reaction_added"
+	WEBSOCKET_EVENT_REACTION_REMOVED        = "reaction_removed"
+	WEBSOCKET_EVENT_RESPONSE                = "response"
+	WEBSOCKET_EVENT_EMOJI_ADDED             = "emoji_added"
+	WEBSOCKET_EVENT_CHANNEL_VIEWED          = "channel_viewed"
+	WEBSOCKET_EVENT_PLUGIN_ACTIVATED        = "plugin_activated"        // EXPERIMENTAL - SUBJECT TO CHANGE
+	WEBSOCKET_EVENT_PLUGIN_DEACTIVATED      = "plugin_deactivated"      // EXPERIMENTAL - SUBJECT TO CHANGE
+	WEBSOCKET_EVENT_PLUGIN_STATUSES_CHANGED = "plugin_statuses_changed" // EXPERIMENTAL - SUBJECT TO CHANGE
+	WEBSOCKET_EVENT_ROLE_UPDATED            = "role_updated"
+	WEBSOCKET_EVENT_LICENSE_CHANGED         = "license_changed"
+	WEBSOCKET_EVENT_CONFIG_CHANGED          = "config_changed"
 )
 
 type WebSocketMessage interface {