summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorNarayan Desai <desai@mcs.anl.gov>2009-10-02 20:03:33 +0000
committerNarayan Desai <desai@mcs.anl.gov>2009-10-02 20:03:33 +0000
commita434cd446c015980dad5ff97ac5ae33e007a5bb3 (patch)
treef7d1dca489928cfd0ae3d82984ee2506b6b95f66 /src
parentdffae2e4c1b978996ffcb42657554a4fa4b721c7 (diff)
downloadbcfg2-a434cd446c015980dad5ff97ac5ae33e007a5bb3.tar.gz
bcfg2-a434cd446c015980dad5ff97ac5ae33e007a5bb3.tar.bz2
bcfg2-a434cd446c015980dad5ff97ac5ae33e007a5bb3.zip
Fix nasty core hang, which would manifest itself as a lockup upon shutdown for bcfg2-{server,info,admin}
The root cause of this issue was our use of external threads. Without a channel for data requesting plugin (and hence thread) shutdown, plugin threads would continue, unabated, forever. This would cause the python interpreter to hang forever waiting for a futex. Added a new shutdown method to Bcfg2.Server.Plugin.Plugin which sets a state flag upon shutdown request. Plugin threads must cycle periodically, even if no work is available, to ensure that the shutdown flag is honored in a reasonable amount of time. Also, the shutdown method is triggered using atexit. This means that scripts may need to explicitly sys.exit or raise SystemExit, depending on circumstances. This addresses at least one, but probably more tickets, however I am currently offline. git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5472 ce84e21b-d406-0410-9b95-82705330c041
Diffstat (limited to 'src')
-rw-r--r--src/lib/Server/Admin/Viz.py1
-rw-r--r--src/lib/Server/Core.py7
-rw-r--r--src/lib/Server/Plugin.py4
-rw-r--r--src/lib/Server/Plugins/Snapshots.py7
4 files changed, 16 insertions, 3 deletions
diff --git a/src/lib/Server/Admin/Viz.py b/src/lib/Server/Admin/Viz.py
index 3451c56ae..a6dd6682c 100644
--- a/src/lib/Server/Admin/Viz.py
+++ b/src/lib/Server/Admin/Viz.py
@@ -58,6 +58,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore):
data = self.Visualize(self.get_repo_path(), hset, bset,
kset, outputfile)
print data
+ raise SystemExit, 0
def Visualize(self, repopath, hosts=False,
bundles=False, key=False, output=False):
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index 758241c98..acee2542f 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -1,6 +1,7 @@
'''Bcfg2.Server.Core provides the runtime support for bcfg2 modules'''
__revision__ = '$Revision$'
+import atexit
import logging
import lxml.etree
import select
@@ -55,6 +56,7 @@ class Core(Component):
self.revision = '-1'
self.password = password
self.encoding = encoding
+ atexit.register(self.shutdown)
if '' in plugins:
plugins.remove('')
@@ -132,7 +134,10 @@ class Core(Component):
except:
logger.error("Unexpected instantiation failure for plugin %s" %
(plugin), exc_info=1)
-
+
+ def shutdown(self):
+ for plugin in self.plugins.values():
+ plugin.shutdown()
def validate_data(self, metadata, data, base_cls):
for plugin in self.plugins.values():
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 6c6d5aab3..712f12ea9 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -63,6 +63,7 @@ class Plugin(object):
self.core = core
self.data = "%s/%s" % (datastore, self.name)
self.logger = logging.getLogger('Bcfg2.Plugins.%s' % (self.name))
+ self.running = True
@staticmethod
def make_path(path):
@@ -79,6 +80,9 @@ class Plugin(object):
path = "%s/%s" % (repo, cls.name)
cls.make_path(path)
+ def shutdown(self):
+ self.running = False
+
class Generator(object):
'''Generator plugins contribute to literal client configurations'''
def HandlesEntry(self, entry, metadata):
diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py
index 315dc34c0..3b2949e7a 100644
--- a/src/lib/Server/Plugins/Snapshots.py
+++ b/src/lib/Server/Plugins/Snapshots.py
@@ -59,8 +59,11 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics,
self.loader.start()
def load_snapshot(self):
- while True:
- (metadata, data) = self.work_queue.get(block=True)
+ while self.running:
+ try:
+ (metadata, data) = self.work_queue.get(block=True, timeout=5)
+ except:
+ continue
self.statistics_from_old_stats(metadata, data)
def process_statistics(self, metadata, data):