summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--debian/control11
-rwxr-xr-xdebian/rules7
-rw-r--r--doc/development/core.txt5
-rw-r--r--doc/server/configuration.txt37
-rw-r--r--src/lib/Bcfg2/Options.py13
-rw-r--r--src/lib/Bcfg2/Server/Admin/Client.py1
-rw-r--r--src/lib/Bcfg2/Server/Core.py4
-rw-r--r--src/lib/Bcfg2/Server/MultiprocessingCore.py204
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py6
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py5
-rw-r--r--src/lib/Bcfg2/Server/SSLServer.py9
-rwxr-xr-xsrc/sbin/bcfg2-server31
12 files changed, 292 insertions, 41 deletions
diff --git a/debian/control b/debian/control
index 6c7278e4e..20cef93c8 100644
--- a/debian/control
+++ b/debian/control
@@ -4,25 +4,24 @@ Priority: optional
Maintainer: Arto Jantunen <viiru@debian.org>
Uploaders: Sami Haahtinen <ressu@debian.org>
Build-Depends: debhelper (>= 7.0.50~),
- python (>= 2.3.5-7),
+ python (>= 2.6),
python-setuptools,
python-sphinx (>= 1.0.7+dfsg) | python3-sphinx,
python-lxml,
python-daemon,
python-cherrypy,
+ python-gamin,
python-pyinotify,
python-m2crypto,
python-doc,
python-mock-doc
Build-Depends-Indep: python-support (>= 0.5.3)
Standards-Version: 3.8.0.0
-XS-Python-Version: >= 2.3
Homepage: http://bcfg2.org/
Package: bcfg2
Architecture: all
-Depends: ${python:Depends}, ${misc:Depends}, debsums, python-apt, ucf, lsb-base (>= 3.1-9), python-m2crypto | python-ssl | python2.6 | python3.0 | python3.1 | python3.2
-XB-Python-Version: >= 2.3
+Depends: ${python:Depends}, ${misc:Depends}, debsums, python-apt, ucf, lsb-base (>= 3.1-9), python (>= 2.6)
Description: Configuration management client
Bcfg2 is a configuration management system that generates configuration sets
for clients bound by client profiles.
@@ -31,8 +30,7 @@ Description: Configuration management client
Package: bcfg2-server
Architecture: all
-Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python-ssl | python2.6 | python3.0 | python3.1 | python3.2, python-pyinotify | python-gamin, python-daemon
-XB-Python-Version: >= 2.4
+Depends: ${python:Depends}, ${misc:Depends}, python-lxml (>= 0.9), libxml2-utils (>= 2.6.23), lsb-base (>= 3.1-9), ucf, bcfg2 (= ${binary:Version}), openssl, python (>= 2.6), python-pyinotify | python-gamin, python-daemon
Recommends: graphviz, patch
Suggests: python-cheetah, python-genshi (>= 0.4.4), python-profiler, python-sqlalchemy (>= 0.5.0), python-django, mail-transport-agent, bcfg2-doc (= ${binary:Version})
Description: Configuration management server
@@ -45,7 +43,6 @@ Package: bcfg2-web
Architecture: all
Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django,
Suggests: python-mysqldb, python-psycopg2, python-sqlite, libapache2-mod-wsgi
-XB-Python-Version: >= 2.4
Description: Configuration management web interface
Bcfg2 is a configuration management system that generates configuration sets
for clients bound by client profiles.
diff --git a/debian/rules b/debian/rules
index fcbf6346c..5694e4e37 100755
--- a/debian/rules
+++ b/debian/rules
@@ -3,13 +3,6 @@
%:
dh $@ --with python-support,sphinxdoc
-override_dh_auto_install:
- # Make the build destination dir consistent between pre-7.3 and 7.3 and
- # later debhelper - see http://bcfg2.org/ticket/791
- dh_auto_install
- test -d debian/tmp/usr/local && mv debian/tmp/usr/local/* debian/tmp/usr || exit 0
- test -d debian/tmp/usr/local && rmdir debian/tmp/usr/local || exit 0
-
override_dh_installinit:
# Install bcfg2 initscript without starting it on postinst
dh_installinit --package=bcfg2 --no-start
diff --git a/doc/development/core.txt b/doc/development/core.txt
index 205eb5c59..3953d3402 100644
--- a/doc/development/core.txt
+++ b/doc/development/core.txt
@@ -71,6 +71,11 @@ XML-RPC Server
.. automodule:: Bcfg2.Server.SSLServer
+Multiprocessing Core
+--------------------
+
+.. automodule:: Bcfg2.Server.MultiprocessingCore
+
CherryPy Core
-------------
diff --git a/doc/server/configuration.txt b/doc/server/configuration.txt
index be421207c..f93b172ef 100644
--- a/doc/server/configuration.txt
+++ b/doc/server/configuration.txt
@@ -20,6 +20,9 @@ Bcfg2, although it has become easier in 1.3.0. The steps to do so are
described in three sections below: Common steps for all versions;
steps for older versions only; and steps for 1.3.0.
+Many of the steps below may have already been performed by your OS
+packages.
+
Common Steps
------------
@@ -129,7 +132,7 @@ is even invoked.
Restart ``bcfg2-server`` and you should see it running as non-root in
``ps`` output::
- % ps -ef | grep '[b]cfg2-server'
+ % ps -ef | grep '[b]cfg2-server'
1000 11581 1 0 07:55 ? 00:00:15 python usr/sbin/bcfg2-server -C /etc/bcfg2.conf -D /var/run/bcfg2-server/bcfg2-server.pid
Steps on Bcfg2 1.3.0
@@ -157,7 +160,7 @@ natively in 1.3. Simply add the following lines to ``bcfg2.conf``::
Restart ``bcfg2-server`` and you should see it running as non-root in
``ps`` output::
- % ps -ef | grep '[b]cfg2-server'
+ % ps -ef | grep '[b]cfg2-server'
1000 11581 1 0 07:55 ? 00:00:15 python usr/sbin/bcfg2-server -C /etc/bcfg2.conf -D /var/run/bcfg2-server/bcfg2-server.pid
.. _server-backends:
@@ -167,10 +170,11 @@ Server Backends
.. versionadded:: 1.3.0
-Bcfg2 supports two different server backends: a builtin server
-based on the Python SimpleXMLRPCServer object, and a server that uses
-CherryPy (http://www.cherrypy.org). Each one has advantages and
-disadvantages.
+Bcfg2 supports three different server backends: a builtin server based
+on the Python SimpleXMLRPCServer object; a server that uses CherryPy
+(http://www.cherrypy.org); and a version of the builtin server that
+uses the Python :mod:`multiprocessing` module. Each one has
+advantages and disadvantages.
The builtin server:
@@ -179,27 +183,36 @@ The builtin server:
* Works on Python 2.4;
* Is slow with larger numbers of clients.
+The multiprocessing server:
+
+* Leverages most of the stability and maturity of the builtin server,
+ but does have some new bits;
+* Introduces concurrent processing to Bcfg2, which may break in
+ various edge cases;
+* Supports certificate authentication;
+* Requires Python 2.6;
+* Is faster with large numbers of concurrent runs.
+
The CherryPy server:
* Is very new and potentially buggy;
* Does not support certificate authentication yet, only password
authentication;
-* Requires CherryPy 3.2, which requires Python 2.5;
+* Requires CherryPy 3.3, which requires Python 2.5;
* Is smarter about daemonization, particularly if you are
:ref:`server-dropping-privs`;
* Is faster with large numbers of clients.
Basically, the builtin server should be used unless you have a
-particular need for performance, and can sacrifice certificate
-authentication.
+particular need for performance. The CherryPy server is purely
+experimental at this point.
To select which backend to use, set the ``backend`` option in the
``[server]`` section of ``/etc/bcfg2.conf``. Options are:
* ``cherrypy``
* ``builtin``
+* ``multiprocessing``
* ``best`` (the default; currently the same as ``builtin``)
-If the certificate authentication issues (a limitation in CherryPy
-itself) can be resolved and the CherryPy server proves to be stable,
-it will likely become the default (and ``best``) in a future release.
+``best`` may change in future releases.
diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py
index 67dcf901e..7e7232820 100644
--- a/src/lib/Bcfg2/Options.py
+++ b/src/lib/Bcfg2/Options.py
@@ -603,6 +603,16 @@ SERVER_AUTHENTICATION = \
default='cert+password',
odesc='{cert|bootstrap|cert+password}',
cf=('communication', 'authentication'))
+SERVER_CHILDREN = \
+ Option('Spawn this number of children for the multiprocessing core. '
+ 'By default spawns children equivalent to the number of processors '
+ 'in the machine.',
+ default=None,
+ cmd='--children',
+ odesc='<children>',
+ cf=('server', 'children'),
+ cook=get_int,
+ long_arg=True)
# database options
DB_ENGINE = \
@@ -1109,7 +1119,8 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY,
vcs_root=SERVER_VCS_ROOT,
authentication=SERVER_AUTHENTICATION,
perflog=LOG_PERFORMANCE,
- perflog_interval=PERFLOG_INTERVAL)
+ perflog_interval=PERFLOG_INTERVAL,
+ children=SERVER_CHILDREN)
CRYPT_OPTIONS = dict(encrypt=ENCRYPT,
decrypt=DECRYPT,
diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py
index b7916fab9..570e993ed 100644
--- a/src/lib/Bcfg2/Server/Admin/Client.py
+++ b/src/lib/Bcfg2/Server/Admin/Client.py
@@ -8,6 +8,7 @@ from Bcfg2.Server.Plugin import MetadataConsistencyError
class Client(Bcfg2.Server.Admin.MetadataCore):
""" Create, delete, or list client entries """
__usage__ = "[options] [add|del|list] [attr=val]"
+ __plugin_whitelist__ = ["Metadata"]
def __call__(self, args):
if len(args) == 0:
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index d61543256..8ef9e3e96 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -301,6 +301,7 @@ class BaseCore(object):
self.logger.info("Performance statistics: "
"%s min=%.06f, max=%.06f, average=%.06f, "
"count=%d" % ((name, ) + stats))
+ self.logger.debug("Performance logging thread terminated")
def _file_monitor_thread(self):
""" The thread that runs the
@@ -321,6 +322,7 @@ class BaseCore(object):
except:
continue
self._update_vcs_revision()
+ self.logger.debug("File monitor thread terminated")
@Bcfg2.Server.Statistics.track_statistics()
def _update_vcs_revision(self):
@@ -441,8 +443,10 @@ class BaseCore(object):
if not self.terminate.isSet():
self.terminate.set()
self.fam.shutdown()
+ self.logger.debug("FAM shut down")
for plugin in list(self.plugins.values()):
plugin.shutdown()
+ self.logger.debug("All plugins shut down")
@property
def metadata_cache_mode(self):
diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py
new file mode 100644
index 000000000..81fba7092
--- /dev/null
+++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py
@@ -0,0 +1,204 @@
+""" The multiprocessing server core is a reimplementation of the
+:mod:`Bcfg2.Server.BuiltinCore` that uses the Python
+:mod:`multiprocessing` library to offload work to multiple child
+processes. As such, it requires Python 2.6+.
+"""
+
+import threading
+import lxml.etree
+import multiprocessing
+from Bcfg2.Compat import Queue
+from Bcfg2.Server.Core import BaseCore, exposed
+from Bcfg2.Server.BuiltinCore import Core as BuiltinCore
+
+
+class DualEvent(object):
+ """ DualEvent is a clone of :class:`threading.Event` that
+ internally implements both :class:`threading.Event` and
+ :class:`multiprocessing.Event`. """
+
+ def __init__(self, threading_event=None, multiprocessing_event=None):
+ self._threading_event = threading_event or threading.Event()
+ self._multiproc_event = multiprocessing_event or \
+ multiprocessing.Event()
+ if threading_event or multiprocessing_event:
+ # initialize internal flag to false, regardless of the
+ # state of either object passed in
+ self.clear()
+
+ def is_set(self):
+ """ Return true if and only if the internal flag is true. """
+ return self._threading_event.is_set()
+
+ isSet = is_set
+
+ def set(self):
+ """ Set the internal flag to true. """
+ self._threading_event.set()
+ self._multiproc_event.set()
+
+ def clear(self):
+ """ Reset the internal flag to false. """
+ self._threading_event.clear()
+ self._multiproc_event.clear()
+
+ def wait(self, timeout=None):
+ """ Block until the internal flag is true, or until the
+ optional timeout occurs. """
+ return self._threading_event.wait(timeout=timeout)
+
+
+class ChildCore(BaseCore):
+ """ A child process for :class:`Bcfg2.MultiprocessingCore.Core`.
+ This core builds configurations from a given
+ :class:`multiprocessing.Pipe`. Note that this is a full-fledged
+ server core; the only input it gets from the parent process is the
+ hostnames of clients to render. All other state comes from the
+ FAM. However, this core only is used to render configs; it doesn't
+ handle anything else (authentication, probes, etc.) because those
+ are all much faster. There's no reason that it couldn't handle
+ those, though, if the pipe communication "protocol" were made more
+ robust. """
+
+ #: How long to wait while polling for new clients to build. This
+ #: doesn't affect the speed with which a client is built, but
+ #: setting it too high will result in longer shutdown times, since
+ #: we only check for the termination event from the main process
+ #: every ``poll_wait`` seconds.
+ poll_wait = 5.0
+
+ def __init__(self, setup, pipe, terminate):
+ """
+ :param setup: A Bcfg2 options dict
+ :type setup: Bcfg2.Options.OptionParser
+ :param pipe: The pipe to which client hostnames are added for
+ ChildCore objects to build configurations, and to
+ which client configurations are added after
+ having been built by ChildCore objects.
+ :type pipe: multiprocessing.Pipe
+ :param terminate: An event that flags ChildCore objects to shut
+ themselves down.
+ :type terminate: multiprocessing.Event
+ """
+ BaseCore.__init__(self, setup)
+
+ #: The pipe to which client hostnames are added for ChildCore
+ #: objects to build configurations, and to which client
+ #: configurations are added after having been built by
+ #: ChildCore objects.
+ self.pipe = pipe
+
+ #: The :class:`multiprocessing.Event` that will be monitored
+ #: to determine when this child should shut down.
+ self.terminate = terminate
+
+ def _daemonize(self):
+ return True
+
+ def _run(self):
+ return True
+
+ def _block(self):
+ while not self.terminate.isSet():
+ try:
+ if self.pipe.poll(self.poll_wait):
+ if not self.metadata.use_database:
+ # handle FAM events, in case (for instance) the
+ # client has just been added to clients.xml, or a
+ # profile has just been asserted. but really, you
+ # should be using the metadata database if you're
+ # using this core.
+ self.fam.handle_events_in_interval(0.1)
+ client = self.pipe.recv()
+ self.logger.debug("Building configuration for %s" % client)
+ config = \
+ lxml.etree.tostring(self.BuildConfiguration(client))
+ self.logger.debug("Returning configuration for %s to main "
+ "process" % client)
+ self.pipe.send(config)
+ self.logger.debug("Returned configuration for %s to main "
+ "process" % client)
+ except KeyboardInterrupt:
+ break
+ self.shutdown()
+
+
+class Core(BuiltinCore):
+ """ A multiprocessing core that delegates building the actual
+ client configurations to
+ :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` objects. The
+ parent process doesn't build any children itself; all calls to
+ :func:`GetConfig` are delegated to children. All other calls are
+ handled by the parent process. """
+
+ #: How long to wait for a child process to shut down cleanly
+ #: before it is terminated.
+ shutdown_timeout = 10.0
+
+ def __init__(self, setup):
+ BuiltinCore.__init__(self, setup)
+ if setup['children'] is None:
+ setup['children'] = multiprocessing.cpu_count()
+
+ #: A dict of child name -> one end of the
+ #: :class:`multiprocessing.Pipe` object used to communicate
+ #: with that child. (The child is given the other end of the
+ #: Pipe.)
+ self.pipes = dict()
+
+ #: A queue that keeps track of which children are available to
+ #: render a configuration. A child is popped from the queue
+ #: when it starts to render a config, then it's pushed back on
+ #: when it's done. This lets us use a blocking call to
+ #: :func:`Queue.Queue.get` when waiting for an available
+ #: child.
+ self.available_children = Queue(maxsize=self.setup['children'])
+
+ # sigh. multiprocessing was added in py2.6, which is when the
+ # camelCase methods for threading objects were deprecated in
+ # favor of the Pythonic under_score methods. So
+ # multiprocessing.Event *only* has is_set(), while
+ # threading.Event has *both* isSet() and is_set(). In order
+ # to make the core work with Python 2.4+, and with both
+ # multiprocessing and threading Event objects, we just
+ # monkeypatch self.terminate to have isSet().
+ self.terminate = DualEvent(threading_event=self.terminate)
+
+ def _run(self):
+ for cnum in range(self.setup['children']):
+ name = "Child-%s" % cnum
+ (mainpipe, childpipe) = multiprocessing.Pipe()
+ self.pipes[name] = mainpipe
+ self.logger.debug("Starting child %s" % name)
+ childcore = ChildCore(self.setup, childpipe, self.terminate)
+ child = multiprocessing.Process(target=childcore.run, name=name)
+ child.start()
+ self.logger.debug("Child %s started with PID %s" % (name,
+ child.pid))
+ self.available_children.put(name)
+ return BuiltinCore._run(self)
+
+ def shutdown(self):
+ BuiltinCore.shutdown(self)
+ for child in multiprocessing.active_children():
+ self.logger.debug("Shutting down child %s" % child.name)
+ child.join(self.shutdown_timeout)
+ if child.is_alive():
+ self.logger.error("Waited %s seconds to shut down %s, "
+ "terminating" % (self.shutdown_timeout,
+ child.name))
+ child.terminate()
+ else:
+ self.logger.debug("Child %s shut down" % child.name)
+ self.logger.debug("All children shut down")
+
+ @exposed
+ def GetConfig(self, address):
+ client = self.resolve_client(address)[0]
+ childname = self.available_children.get()
+ self.logger.debug("Building configuration on child %s" % childname)
+ pipe = self.pipes[childname]
+ pipe.send(client)
+ config = pipe.recv()
+ self.available_children.put_nowait(childname)
+ return config
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 2bc82caa9..507973fa6 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -556,6 +556,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
open(os.path.join(repo, cls.name, fname),
"w").write(kwargs[aname])
+ @property
+ def use_database(self):
+ """ Expose self._use_db publicly for use in
+ :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` """
+ return self._use_db
+
def _handle_file(self, fname):
""" set up the necessary magic for handling a metadata file
(clients.xml or groups.xml, e.g.) """
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 567a16c40..8c272cf53 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -512,8 +512,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
collection = cclass(metadata, relevant, self.cachepath, self.data,
debug=self.debug_flag)
ckey = collection.cachekey
- self.clients[metadata.hostname] = ckey
- self.collections[ckey] = collection
+ if cclass != Collection:
+ self.clients[metadata.hostname] = ckey
+ self.collections[ckey] = collection
return collection
def get_additional_data(self, metadata):
diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py
index eea2183f7..8bdcf0500 100644
--- a/src/lib/Bcfg2/Server/SSLServer.py
+++ b/src/lib/Bcfg2/Server/SSLServer.py
@@ -290,7 +290,10 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
raise
except socket.error:
err = sys.exc_info()[1]
- if err[0] == 32:
+ if isinstance(err, socket.timeout):
+ self.logger.warning("Connection timed out for %s" %
+ self.client_address[0])
+ elif err[0] == 32:
self.logger.warning("Connection dropped from %s" %
self.client_address[0])
elif err[0] == 104:
@@ -423,7 +426,9 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer,
def serve_forever(self):
"""Serve single requests until (self.serve == False)."""
self.serve = True
- self.task_thread = threading.Thread(target=self._tasks_thread)
+ self.task_thread = \
+ threading.Thread(name="%sThread" % self.__class__.__name__,
+ target=self._tasks_thread)
self.task_thread.start()
self.logger.info("serve_forever() [start]")
signal.signal(signal.SIGINT, self._handle_shutdown_signal)
diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server
index 33ee327fc..95413d6cf 100755
--- a/src/sbin/bcfg2-server
+++ b/src/sbin/bcfg2-server
@@ -24,18 +24,29 @@ def main():
print("Could not read %s" % setup['configfile'])
sys.exit(1)
- if setup['backend'] not in ['best', 'cherrypy', 'builtin']:
+ # TODO: normalize case of various core modules so we can add a new
+ # core without modifying this script
+ backends = dict(cherrypy='CherryPyCore',
+ builtin='BuiltinCore',
+ best='BuiltinCore',
+ multiprocessing='MultiprocessingCore')
+
+ if setup['backend'] not in backends:
print("Unknown server backend %s, using 'best'" % setup['backend'])
setup['backend'] = 'best'
- if setup['backend'] == 'cherrypy':
- try:
- from Bcfg2.Server.CherryPyCore import Core
- except ImportError:
- err = sys.exc_info()[1]
- print("Unable to import CherryPy server core: %s" % err)
- raise
- elif setup['backend'] == 'builtin' or setup['backend'] == 'best':
- from Bcfg2.Server.BuiltinCore import Core
+
+ coremodule = backends[setup['backend']]
+ try:
+ Core = getattr(__import__("Bcfg2.Server.%s" % coremodule).Server,
+ coremodule).Core
+ except ImportError:
+ err = sys.exc_info()[1]
+ print("Unable to import %s server core: %s" % (setup['backend'], err))
+ raise
+ except AttributeError:
+ err = sys.exc_info()[1]
+ print("Unable to load %s server core: %s" % (setup['backend'], err))
+ raise
try:
core = Core()
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-06-02 10:25:55 +0000