MM-8622: Improved plugin error reporting (#8737)

* allow `Wait()`ing on the supervisor

In the event the plugin supervisor shuts down a plugin for crashing too
many times, the new `Wait()` interface allows the `ActivatePlugin` to
accept a callback function to trigger when `supervisor.Wait()` returns.
If the supervisor shuts down normally, this callback is invoked with
a nil error, otherwise any error reported by the supervisor is passed
along.

* improve plugin activation/deactivation logic

Avoid triggering activation of previously failed-to-start plugins just
becase something in the configuration changed. Now, intelligently
compare the global enable bit as well as the each individual plugin's
enabled bit.

* expose store to manipulate PluginStatuses

* expose API to fetch plugin statuses

* keep track of whether or not plugin sandboxing is supported

* transition plugin statuses

* restore error on plugin activation if already active

* don't initialize test plugins until successfully loaded

* emit websocket events when plugin statuses change

* skip pruning if already initialized

* MM-8622: maintain plugin statuses in memory

Switch away from persisting plugin statuses to the database, and
maintain in memory instead. This will be followed by a cluster interface
to query the in-memory status of plugin statuses from all cluster nodes.

At the same time, rename `cluster_discovery_id` on the `PluginStatus`
model object to `cluster_id`.

* MM-8622: aggregate plugin statuses across cluster

* fetch cluster plugin statuses when emitting websocket notification

* address unit test fixes after rebasing

* relax (poor) racey unit test re: supervisor.Wait()

* make store-mocks

1 files changed, 218 insertions, 22 deletions

diff --git a/app/plugin.go b/app/plugin.go
index 0d3415f4c..f6cb6bdda 100644
--- a/app/plugin.go
+++ b/app/plugin.go
@@ -37,6 +37,31 @@ var prepackagedPlugins map[string]func(string) ([]byte, error) = map[string]func
 	"zoom": zoom.Asset,
 }
 
+func (a *App) notifyPluginStatusesChanged() error {
+	pluginStatuses, err := a.GetClusterPluginStatuses()
+	if err != nil {
+		return err
+	}
+
+	// Notify any system admins.
+	message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_PLUGIN_STATUSES_CHANGED, "", "", "", nil)
+	message.Add("plugin_statuses", pluginStatuses)
+	message.Broadcast.ContainsSensitiveData = true
+	a.Publish(message)
+
+	return nil
+}
+
+func (a *App) setPluginStatusState(id string, state int) error {
+	if _, ok := a.pluginStatuses[id]; !ok {
+		return nil
+	}
+
+	a.pluginStatuses[id].State = state
+
+	return a.notifyPluginStatusesChanged()
+}
+
 func (a *App) initBuiltInPlugins() {
 	plugins := map[string]builtinplugin.Plugin{
 		"ldapextras": &ldapextras.Plugin{},
@@ -77,30 +102,100 @@ func (a *App) setPluginsActive(activate bool) {
 			continue
 		}
 
-		id := plugin.Manifest.Id
+		enabled := false
+		if state, ok := a.Config().PluginSettings.PluginStates[plugin.Manifest.Id]; ok {
+			enabled = state.Enable
+		}
+
+		a.pluginStatuses[plugin.Manifest.Id] = &model.PluginStatus{
+			ClusterId:   a.GetClusterId(),
+			PluginId:    plugin.Manifest.Id,
+			PluginPath:  filepath.Dir(plugin.ManifestPath),
+			IsSandboxed: a.IsPluginSandboxSupported,
+			Name:        plugin.Manifest.Name,
+			Description: plugin.Manifest.Description,
+			Version:     plugin.Manifest.Version,
+		}
+
+		if activate && enabled {
+			a.setPluginActive(plugin, activate)
+		} else if !activate {
+			a.setPluginActive(plugin, activate)
+		}
+	}
+
+	if err := a.notifyPluginStatusesChanged(); err != nil {
+		mlog.Error("failed to notify plugin status changed", mlog.Err(err))
+	}
+}
+
+func (a *App) setPluginActiveById(id string, activate bool) {
+	plugins, err := a.PluginEnv.Plugins()
+	if err != nil {
+		mlog.Error(fmt.Sprintf("Cannot setPluginActiveById(%t)", activate), mlog.String("plugin_id", id), mlog.Err(err))
+		return
+	}
 
-		pluginState := &model.PluginState{Enable: false}
-		if state, ok := a.Config().PluginSettings.PluginStates[id]; ok {
-			pluginState = state
+	for _, plugin := range plugins {
+		if plugin.Manifest != nil && plugin.Manifest.Id == id {
+			a.setPluginActive(plugin, activate)
 		}
+	}
+}
+
+func (a *App) setPluginActive(plugin *model.BundleInfo, activate bool) {
+	if plugin.Manifest == nil {
+		return
+	}
 
-		active := a.PluginEnv.IsPluginActive(id)
+	id := plugin.Manifest.Id
 
-		if activate && pluginState.Enable && !active {
+	active := a.PluginEnv.IsPluginActive(id)
+
+	if activate {
+		if !active {
 			if err := a.activatePlugin(plugin.Manifest); err != nil {
 				mlog.Error("Plugin failed to activate", mlog.String("plugin_id", plugin.Manifest.Id), mlog.String("err", err.DetailedError))
 			}
+		}
 
-		} else if (!activate || !pluginState.Enable) && active {
+	} else if !activate {
+		if active {
 			if err := a.deactivatePlugin(plugin.Manifest); err != nil {
 				mlog.Error("Plugin failed to deactivate", mlog.String("plugin_id", plugin.Manifest.Id), mlog.String("err", err.DetailedError))
 			}
+		} else {
+			if err := a.setPluginStatusState(plugin.Manifest.Id, model.PluginStateNotRunning); err != nil {
+				mlog.Error("Plugin status state failed to update", mlog.String("plugin_id", plugin.Manifest.Id), mlog.String("err", err.Error()))
+			}
 		}
 	}
 }
 
 func (a *App) activatePlugin(manifest *model.Manifest) *model.AppError {
-	if err := a.PluginEnv.ActivatePlugin(manifest.Id); err != nil {
+	mlog.Debug("Activating plugin", mlog.String("plugin_id", manifest.Id))
+
+	if err := a.setPluginStatusState(manifest.Id, model.PluginStateStarting); err != nil {
+		return model.NewAppError("activatePlugin", "app.plugin.set_plugin_status_state.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	onError := func(err error) {
+		mlog.Debug("Plugin failed to stay running", mlog.String("plugin_id", manifest.Id), mlog.Err(err))
+
+		if err := a.setPluginStatusState(manifest.Id, model.PluginStateFailedToStayRunning); err != nil {
+			mlog.Error("Failed to record plugin status", mlog.String("plugin_id", manifest.Id), mlog.Err(err))
+		}
+	}
+
+	if err := a.PluginEnv.ActivatePlugin(manifest.Id, onError); err != nil {
+		if err := a.setPluginStatusState(manifest.Id, model.PluginStateFailedToStart); err != nil {
+			return model.NewAppError("activatePlugin", "app.plugin.activate.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+
+		return model.NewAppError("activatePlugin", "app.plugin.activate.app_error", nil, err.Error(), http.StatusBadRequest)
+	}
+
+	if err := a.setPluginStatusState(manifest.Id, model.PluginStateRunning); err != nil {
 		return model.NewAppError("activatePlugin", "app.plugin.activate.app_error", nil, err.Error(), http.StatusBadRequest)
 	}
 
@@ -115,6 +210,12 @@ func (a *App) activatePlugin(manifest *model.Manifest) *model.AppError {
 }
 
 func (a *App) deactivatePlugin(manifest *model.Manifest) *model.AppError {
+	mlog.Debug("Deactivating plugin", mlog.String("plugin_id", manifest.Id))
+
+	if err := a.setPluginStatusState(manifest.Id, model.PluginStateStopping); err != nil {
+		return model.NewAppError("EnablePlugin", "app.plugin.deactivate.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
 	if err := a.PluginEnv.DeactivatePlugin(manifest.Id); err != nil {
 		return model.NewAppError("deactivatePlugin", "app.plugin.deactivate.app_error", nil, err.Error(), http.StatusBadRequest)
 	}
@@ -127,6 +228,10 @@ func (a *App) deactivatePlugin(manifest *model.Manifest) *model.AppError {
 		a.Publish(message)
 	}
 
+	if err := a.setPluginStatusState(manifest.Id, model.PluginStateNotRunning); err != nil {
+		return model.NewAppError("deactivatePlugin", "app.plugin.deactivate.app_error", nil, err.Error(), http.StatusBadRequest)
+	}
+
 	mlog.Info("Deactivated plugin", mlog.String("plugin_id", manifest.Id))
 	return nil
 }
@@ -166,7 +271,8 @@ func (a *App) installPlugin(pluginFile io.Reader, allowPrepackaged bool) (*model
 		return nil, model.NewAppError("installPlugin", "app.plugin.manifest.app_error", nil, err.Error(), http.StatusBadRequest)
 	}
 
-	if _, ok := prepackagedPlugins[manifest.Id]; ok && !allowPrepackaged {
+	_, isPrepackaged := prepackagedPlugins[manifest.Id]
+	if isPrepackaged && !allowPrepackaged {
 		return nil, model.NewAppError("installPlugin", "app.plugin.prepackaged.app_error", nil, "", http.StatusBadRequest)
 	}
 
@@ -185,16 +291,33 @@ func (a *App) installPlugin(pluginFile io.Reader, allowPrepackaged bool) (*model
 		}
 	}
 
-	err = utils.CopyDir(tmpPluginDir, filepath.Join(a.PluginEnv.SearchPath(), manifest.Id))
+	pluginPath := filepath.Join(a.PluginEnv.SearchPath(), manifest.Id)
+	err = utils.CopyDir(tmpPluginDir, pluginPath)
 	if err != nil {
 		return nil, model.NewAppError("installPlugin", "app.plugin.mvdir.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 
-	// Should add manifest validation and error handling here
+	a.pluginStatuses[manifest.Id] = &model.PluginStatus{
+		ClusterId:     a.GetClusterId(),
+		PluginId:      manifest.Id,
+		PluginPath:    pluginPath,
+		State:         model.PluginStateNotRunning,
+		IsSandboxed:   a.IsPluginSandboxSupported,
+		IsPrepackaged: isPrepackaged,
+		Name:          manifest.Name,
+		Description:   manifest.Description,
+		Version:       manifest.Version,
+	}
+
+	if err := a.notifyPluginStatusesChanged(); err != nil {
+		mlog.Error("failed to notify plugin status changed", mlog.Err(err))
+	}
 
 	return manifest, nil
 }
 
+// GetPlugins returned the plugins installed on this server, including the manifests needed to
+// enable plugins with web functionality.
 func (a *App) GetPlugins() (*model.PluginsResponse, *model.AppError) {
 	if a.PluginEnv == nil || !*a.Config().PluginSettings.Enable {
 		return nil, model.NewAppError("GetPlugins", "app.plugin.disabled.app_error", nil, "", http.StatusNotImplemented)
@@ -240,6 +363,39 @@ func (a *App) GetActivePluginManifests() ([]*model.Manifest, *model.AppError) {
 	return manifests, nil
 }
 
+// GetPluginStatuses returns the status for plugins installed on this server.
+func (a *App) GetPluginStatuses() (model.PluginStatuses, *model.AppError) {
+	if !*a.Config().PluginSettings.Enable {
+		return nil, model.NewAppError("GetPluginStatuses", "app.plugin.disabled.app_error", nil, "", http.StatusNotImplemented)
+	}
+
+	pluginStatuses := make([]*model.PluginStatus, 0, len(a.pluginStatuses))
+	for _, pluginStatus := range a.pluginStatuses {
+		pluginStatuses = append(pluginStatuses, pluginStatus)
+	}
+
+	return pluginStatuses, nil
+}
+
+// GetClusterPluginStatuses returns the status for plugins installed anywhere in the cluster.
+func (a *App) GetClusterPluginStatuses() (model.PluginStatuses, *model.AppError) {
+	pluginStatuses, err := a.GetPluginStatuses()
+	if err != nil {
+		return nil, err
+	}
+
+	if a.Cluster != nil && *a.Config().ClusterSettings.Enable {
+		clusterPluginStatuses, err := a.Cluster.GetPluginStatuses()
+		if err != nil {
+			return nil, model.NewAppError("GetClusterPluginStatuses", "app.plugin.get_cluster_plugin_statuses.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+
+		pluginStatuses = append(pluginStatuses, clusterPluginStatuses...)
+	}
+
+	return pluginStatuses, nil
+}
+
 func (a *App) RemovePlugin(id string) *model.AppError {
 	return a.removePlugin(id, false)
 }
@@ -284,10 +440,16 @@ func (a *App) removePlugin(id string, allowPrepackaged bool) *model.AppError {
 		return model.NewAppError("removePlugin", "app.plugin.remove.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 
+	delete(a.pluginStatuses, manifest.Id)
+	if err := a.notifyPluginStatusesChanged(); err != nil {
+		mlog.Error("failed to notify plugin status changed", mlog.Err(err))
+	}
+
 	return nil
 }
 
-// EnablePlugin will set the config for an installed plugin to enabled, triggering activation if inactive.
+// EnablePlugin will set the config for an installed plugin to enabled, triggering asynchronous
+// activation if inactive anywhere in the cluster.
 func (a *App) EnablePlugin(id string) *model.AppError {
 	if a.PluginEnv == nil || !*a.Config().PluginSettings.Enable {
 		return model.NewAppError("EnablePlugin", "app.plugin.disabled.app_error", nil, "", http.StatusNotImplemented)
@@ -310,8 +472,8 @@ func (a *App) EnablePlugin(id string) *model.AppError {
 		return model.NewAppError("EnablePlugin", "app.plugin.not_installed.app_error", nil, "", http.StatusBadRequest)
 	}
 
-	if err := a.activatePlugin(manifest); err != nil {
-		return err
+	if err := a.setPluginStatusState(manifest.Id, model.PluginStateStarting); err != nil {
+		return model.NewAppError("EnablePlugin", "app.plugin.set_plugin_status_state.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 
 	a.UpdateConfig(func(cfg *model.Config) {
@@ -351,6 +513,10 @@ func (a *App) DisablePlugin(id string) *model.AppError {
 		return model.NewAppError("DisablePlugin", "app.plugin.not_installed.app_error", nil, "", http.StatusBadRequest)
 	}
 
+	if err := a.setPluginStatusState(manifest.Id, model.PluginStateStopping); err != nil {
+		return model.NewAppError("EnablePlugin", "app.plugin.set_plugin_status_state.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
 	a.UpdateConfig(func(cfg *model.Config) {
 		cfg.PluginSettings.PluginStates[id] = &model.PluginState{Enable: false}
 	})
@@ -363,16 +529,18 @@ func (a *App) DisablePlugin(id string) *model.AppError {
 }
 
 func (a *App) InitPlugins(pluginPath, webappPath string, supervisorOverride pluginenv.SupervisorProviderFunc) {
-	if !*a.Config().PluginSettings.Enable {
+	if a.PluginEnv != nil {
 		return
 	}
 
-	if a.PluginEnv != nil {
+	if !*a.Config().PluginSettings.Enable {
 		return
 	}
 
 	mlog.Info("Starting up plugins")
 
+	a.pluginStatuses = make(map[string]*model.PluginStatus)
+
 	if err := os.Mkdir(pluginPath, 0744); err != nil && !os.IsExist(err) {
 		mlog.Error("Failed to start up plugins", mlog.Err(err))
 		return
@@ -398,13 +566,19 @@ func (a *App) InitPlugins(pluginPath, webappPath string, supervisorOverride plug
 		}),
 	}
 
-	if supervisorOverride != nil {
-		options = append(options, pluginenv.SupervisorProvider(supervisorOverride))
-	} else if err := sandbox.CheckSupport(); err != nil {
+	if err := sandbox.CheckSupport(); err != nil {
+		a.IsPluginSandboxSupported = false
 		mlog.Warn("plugin sandboxing is not supported. plugins will run with the same access level as the server. See documentation to learn more: https://developers.mattermost.com/extend/plugins/security/", mlog.Err(err))
-		options = append(options, pluginenv.SupervisorProvider(rpcplugin.SupervisorProvider))
 	} else {
+		a.IsPluginSandboxSupported = true
+	}
+
+	if supervisorOverride != nil {
+		options = append(options, pluginenv.SupervisorProvider(supervisorOverride))
+	} else if a.IsPluginSandboxSupported {
 		options = append(options, pluginenv.SupervisorProvider(sandbox.SupervisorProvider))
+	} else {
+		options = append(options, pluginenv.SupervisorProvider(rpcplugin.SupervisorProvider))
 	}
 
 	if env, err := pluginenv.New(options...); err != nil {
@@ -431,12 +605,34 @@ func (a *App) InitPlugins(pluginPath, webappPath string, supervisorOverride plug
 	}
 
 	a.RemoveConfigListener(a.PluginConfigListenerId)
-	a.PluginConfigListenerId = a.AddConfigListener(func(_, cfg *model.Config) {
+	a.PluginConfigListenerId = a.AddConfigListener(func(oldCfg *model.Config, cfg *model.Config) {
 		if a.PluginEnv == nil {
 			return
 		}
 
-		a.setPluginsActive(*cfg.PluginSettings.Enable)
+		if *oldCfg.PluginSettings.Enable != *cfg.PluginSettings.Enable {
+			a.setPluginsActive(*cfg.PluginSettings.Enable)
+		} else {
+			plugins := map[string]bool{}
+			for id := range oldCfg.PluginSettings.PluginStates {
+				plugins[id] = true
+			}
+			for id := range cfg.PluginSettings.PluginStates {
+				plugins[id] = true
+			}
+
+			for id := range plugins {
+				oldPluginState := oldCfg.PluginSettings.PluginStates[id]
+				pluginState := cfg.PluginSettings.PluginStates[id]
+
+				wasEnabled := oldPluginState != nil && oldPluginState.Enable
+				isEnabled := pluginState != nil && pluginState.Enable
+
+				if wasEnabled != isEnabled {
+					a.setPluginActiveById(id, isEnabled)
+				}
+			}
+		}
 
 		for _, err := range a.PluginEnv.Hooks().OnConfigurationChange() {
 			mlog.Error(err.Error())