From 835c0871a01d9c0460ee34e22e9dfc53a69a2ac9 Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Fri, 27 Jul 2018 11:37:17 -0400 Subject: MM-11431: handle plugin deadlocks (#9167) * ensure plugin is always shutdown Once we call `.client.Client()` the plugin has started, and must be shut down. `newSupervisor` sometimes returned with an error (and without a reference to the supervisor), leaving the client running indefinitely. * Clarify the documentation to explain that plugin hooks will not trigger until `OnActivate` returns successfully, and will stop triggering just before `OnDeactivate` is called. * test for plugin deadlock * plugin/environment.go: switch to sync.Map From: https://golang.org/pkg/sync/#Map > If a goroutine holds a RWMutex for reading and another goroutine might call Lock, no goroutine should expect to be able to acquire a read lock until the initial read lock is released. In particular, this prohibits recursive read locking. This is to ensure that the lock eventually becomes available; a blocked Lock call excludes new readers from acquiring the lock. The previous `RWMutex` was not safe given that we effectively acquired read locks recursively (hook -> api -> hook). This worked up until we activated or deactivated plugins, tried to acquire a write lock, and the plugin used the API to effectively trigger another hook. Switching to sync.Map avoids this by divesting the need to lock at all, avoiding the potential for a recursive lock in the first place. --- plugin/supervisor.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'plugin/supervisor.go') diff --git a/plugin/supervisor.go b/plugin/supervisor.go index f6264f47c..33243e9cf 100644 --- a/plugin/supervisor.go +++ b/plugin/supervisor.go @@ -23,8 +23,13 @@ type supervisor struct { implemented [TotalHooksId]bool } -func newSupervisor(pluginInfo *model.BundleInfo, parentLogger *mlog.Logger, apiImpl API) (*supervisor, error) { +func newSupervisor(pluginInfo *model.BundleInfo, parentLogger *mlog.Logger, apiImpl API) (retSupervisor *supervisor, retErr error) { supervisor := supervisor{} + defer func() { + if retErr != nil { + supervisor.Shutdown() + } + }() wrappedLogger := pluginInfo.WrapLogger(parentLogger) @@ -90,7 +95,9 @@ func newSupervisor(pluginInfo *model.BundleInfo, parentLogger *mlog.Logger, apiI } func (sup *supervisor) Shutdown() { - sup.client.Kill() + if sup.client != nil { + sup.client.Kill() + } } func (sup *supervisor) Hooks() Hooks { -- cgit v1.2.3-1-g7c22