summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/development/core.txt62
-rw-r--r--src/lib/Bcfg2/Server/Core.py445
-rw-r--r--src/lib/Bcfg2/Server/Plugin/helpers.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Bundler.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Decisions.py88
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Probes.py2
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py18
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py9
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py4
-rw-r--r--testsuite/Testsrc/test_code_checks.py3
11 files changed, 505 insertions, 132 deletions
diff --git a/doc/development/core.txt b/doc/development/core.txt
new file mode 100644
index 000000000..145be3338
--- /dev/null
+++ b/doc/development/core.txt
@@ -0,0 +1,62 @@
+.. -*- mode: rst -*-
+
+.. _development-core:
+
+=========================
+ Server Core Development
+=========================
+
+.. versionadded:: 1.3.0
+
+Bcfg2 1.3 added a pluggable server core system so that the server core
+itself can be easily swapped out to use different technologies. It
+currently ships with two backends: a builtin core written from scratch
+using the various server tools in the Python standard library; and an
+experimental `CherryPy <http://www.cherrypy.org/>`_ based core. This
+page documents the server core interface so that other cores can be
+written to take advantage of other technologies, e.g., `Tornado
+<http://www.tornadoweb.org/>`_ or `Twisted
+<http://twistedmatrix.com/trac/>`_.
+
+A core implementation needs to:
+
+* Override :func:`Bcfg2.Server.Core.Core._daemonize` to handle
+ daemonization, writing the PID file, and dropping privileges.
+* Override :func:`Bcfg2.Server.Core.Core._run` to handle server
+ startup.
+* Override :func:`Bcfg2.Server.Core.Core._block` to run the blocking
+ server loop.
+* Call :func:`Bcfg2.Server.Core.Core.shutdown` on orderly shutdown.
+
+Nearly all XML-RPC handling is delegated entirely to the core
+implementation. It needs to:
+
+* Call :func:`Bcfg2.Server.Core.Core.authenticate` to authenticate
+ clients.
+* Handle :exc:`xmlrpclib.Fault` exceptions raised by the exposed
+ XML-RPC methods as appropriate.
+* Dispatch XML-RPC method invocations to the appropriate method,
+ including Plugin RMI.
+
+Additionally, running and configuring the server is delegated to the
+core. It needs to honor the configuration options that influence how
+and where the server runs, including the server location (host and
+port), listening interfaces, and SSL certificate and key.
+
+Base Core
+=========
+
+.. automodule:: Bcfg2.Server.Core
+
+Core Implementations
+====================
+
+Builtin Core
+------------
+
+.. automodule:: Bcfg2.Server.BuiltinCore
+
+CherryPy Core
+-------------
+
+.. automodule:: Bcfg2.Server.CherryPyCore
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 6ab18e36f..ee8c34fb8 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -1,4 +1,5 @@
-"""Bcfg2.Server.Core provides the runtime support for Bcfg2 modules."""
+""" Bcfg2.Server.Core provides the base core object that server core
+implementations inherit from. """
import os
import atexit
@@ -30,14 +31,30 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings'
def exposed(func):
- """ decorator that sets the 'exposed' attribute of a function to
- expose it via XML-RPC """
+ """ Decorator that sets the ``exposed`` attribute of a function to
+ ``True`` expose it via XML-RPC. This currently works for both the
+ builtin and CherryPy cores, although if other cores are added this
+ may need to be made a core-specific function.
+
+ :param func: The function to decorate
+ :type func: callable
+ :returns: callable - the decorated function"""
func.exposed = True
return func
def sort_xml(node, key=None):
- """ sort XML in a deterministic fashion """
+ """ Recursively sort an XML document in a deterministic fashion.
+ This shouldn't be used to perform a *useful* sort, merely to put
+ XML in a deterministic, replicable order. The document is sorted
+ in-place.
+
+ :param node: The root node of the XML tree to sort
+ :type node: lxml.etree._Element or lxml.etree.ElementTree
+ :param key: The key to sort by
+ :type key: callable
+ :returns: None
+ """
for child in node:
sort_xml(child, key)
@@ -49,12 +66,13 @@ def sort_xml(node, key=None):
class CoreInitError(Exception):
- """This error is raised when the core cannot be initialized."""
+ """ Raised when the server core cannot be initialized. """
pass
class NoExposedMethod (Exception):
- """There is no method exposed with the given name."""
+ """ Raised when an XML-RPC method is called, but there is no
+ method exposed with the given name. """
# pylint: disable=W0702
@@ -63,11 +81,22 @@ class NoExposedMethod (Exception):
class BaseCore(object):
- """The Core object is the container for all
- Bcfg2 Server logic and modules.
- """
+ """ The server core is the container for all Bcfg2 server logic
+ and modules. All core implementations must inherit from
+ ``BaseCore``. """
def __init__(self, setup): # pylint: disable=R0912,R0915
+ """
+ :param setup: A Bcfg2 options dict
+ :type setup: Bcfg2.Options.OptionParser
+
+ .. automethod:: _daemonize
+ .. automethod:: _run
+ .. automethod:: _block
+ .. -----
+ .. automethod:: _file_monitor_thread
+ """
+ #: The Bcfg2 repository directory
self.datastore = setup['repo']
if setup['debug']:
@@ -86,6 +115,8 @@ class BaseCore(object):
to_syslog=setup['syslog'],
to_file=setup['logging'],
level=level)
+
+ #: A :class:`logging.Logger` object for use by the core
self.logger = logging.getLogger('bcfg2-server')
try:
@@ -100,29 +131,51 @@ class BaseCore(object):
famargs['ignore'] = setup['ignore']
if 'debug' in setup:
famargs['debug'] = setup['debug']
+
try:
+ #: The :class:`Bcfg2.Server.FileMonitor.FileMonitor`
+ #: object used by the core to monitor for Bcfg2 data
+ #: changes.
self.fam = filemonitor(**famargs)
except IOError:
msg = "Failed to instantiate fam driver %s" % setup['filemonitor']
self.logger.error(msg, exc_info=1)
raise CoreInitError(msg)
- self.pubspace = {}
+
+ #: Path to bcfg2.conf
self.cfile = setup['configfile']
- self.cron = {}
+
+ #: Dict of plugins that are enabled. Keys are the plugin
+ #: names (just the plugin name, in the correct case; e.g.,
+ #: "Cfg", not "Bcfg2.Server.Plugins.Cfg"), and values are
+ #: plugin objects.
self.plugins = {}
+
+ #: Blacklist of plugins that conflict with enabled plugins.
+ #: If two plugins are loaded that conflict with each other,
+ #: the first one loaded wins.
self.plugin_blacklist = {}
+
+ #: Revision of the Bcfg2 specification. This will be sent to
+ #: the client in the configuration, and can be set by a
+ #: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin.
self.revision = '-1'
- self.password = setup['password']
- self.encoding = setup['encoding']
+
+ #: The Bcfg2 options dict
self.setup = setup
+
atexit.register(self.shutdown)
- # Create an event to signal worker threads to shutdown
+
+ #: Threading event to signal worker threads (e.g.,
+ #: :attr:`fam_thread`) to shutdown
self.terminate = threading.Event()
# generate Django ORM settings. this must be done _before_ we
# load plugins
Bcfg2.settings.read_config(repo=self.datastore)
+ #: Whether or not it's possible to use the Django database
+ #: backend for plugins that have that capability
self._database_available = False
if Bcfg2.settings.HAS_DJANGO:
from django.core.exceptions import ImproperlyConfigured
@@ -143,7 +196,7 @@ class BaseCore(object):
for plugin in setup['plugins']:
if not plugin in self.plugins:
- self.init_plugins(plugin)
+ self.init_plugin(plugin)
# Remove blacklisted plugins
for plugin, blacklist in list(self.plugin_blacklist.items()):
if len(blacklist) > 0:
@@ -166,41 +219,82 @@ class BaseCore(object):
(" ".join([x.name for x in depr])))
mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata)
- if len(mlist) == 1:
+ if len(mlist) >= 1:
+ #: The Metadata plugin
self.metadata = mlist[0]
+ if len(mlist) > 1:
+ self.logger.error("Multiple Metadata plugins loaded; "
+ "using %s" % self.metadata)
else:
- self.logger.error("No Metadata Plugin loaded; "
+ self.logger.error("No Metadata plugin loaded; "
"failed to instantiate Core")
raise CoreInitError("No Metadata Plugin")
+
+ #: The list of plugins that handle
+ #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics`
self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics)
+
+ #: The list of plugins that implement the
+ #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource`
+ #: interface
self.pull_sources = \
self.plugins_by_type(Bcfg2.Server.Plugin.PullSource)
+
+ #: The list of
+ #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins
self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator)
+
+ #: The list of plugins that handle
+ #: :class:`Bcfg2.Server.Plugin.interfaces.Structure`
+ #: generation
self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure)
+
+ #: The list of plugins that implement the
+ #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface
self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector)
+
+ #: The CA that signed the server cert
self.ca = setup['ca']
+
+ #: The FAM :class:`threading.Thread`,
+ #: :func:`_file_monitor_thread`
self.fam_thread = \
threading.Thread(name="%sFAMThread" % setup['filemonitor'],
target=self._file_monitor_thread)
+
+ #: A :func:`threading.Lock` for use by
+ #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set`
self.lock = threading.Lock()
+ #: A :class:`Bcfg2.Cache.Cache` object for caching client
+ #: metadata
self.metadata_cache = Cache()
def plugins_by_type(self, base_cls):
- """Return a list of loaded plugins that match the passed type.
+ """ Return a list of loaded plugins that match the passed type.
- The returned list is sorted in ascending order by the Plugins'
- sort_order value. The sort_order defaults to 500 in Plugin.py,
- but can be overridden by individual plugins. Plugins with the
- same numerical sort_order value are sorted in alphabetical
+ The returned list is sorted in ascending order by the plugins'
+ ``sort_order`` value. The
+ :attr:`Bcfg2.Server.Plugin.base.Plugin.sort_order` defaults to
+ 500, but can be overridden by individual plugins. Plugins with
+ the same numerical sort_order value are sorted in alphabetical
order by their name.
+
+ :param base_cls: The base plugin interface class to match (see
+ :mod:`Bcfg2.Server.Plugin.interfaces`)
+ :type base_cls: type
+ :returns: list of :attr:`Bcfg2.Server.Plugin.base.Plugin`
+ objects
"""
return sorted([plugin for plugin in self.plugins.values()
if isinstance(plugin, base_cls)],
key=lambda p: (p.sort_order, p.name))
def _file_monitor_thread(self):
- """The thread for monitor the files."""
+ """ The thread that runs the
+ :class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also
+ queries :class:`Bcfg2.Server.Plugin.interfaces.Version`
+ plugins for the current revision of the Bcfg2 repo. """
famfd = self.fam.fileno()
terminate = self.terminate
while not terminate.isSet():
@@ -217,8 +311,16 @@ class BaseCore(object):
for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version):
self.revision = plugin.get_revision()
- def init_plugins(self, plugin):
- """Handling for the plugins."""
+ def init_plugin(self, plugin):
+ """ Import and instantiate a single plugin. The plugin is
+ stored to :attr:`plugins`.
+
+ :param plugin: The name of the plugin. This is just the name
+ of the plugin, in the appropriate case. I.e.,
+ ``Cfg``, not ``Bcfg2.Server.Plugins.Cfg``.
+ :type plugin: string
+ :returns: None
+ """
self.logger.debug("Loading plugin %s" % plugin)
try:
mod = getattr(__import__("Bcfg2.Server.Plugins.%s" %
@@ -250,7 +352,7 @@ class BaseCore(object):
% plugin, exc_info=1)
def shutdown(self):
- """Shutting down the plugins."""
+ """ Perform plugin and FAM shutdown tasks. """
if not self.terminate.isSet():
self.terminate.set()
self.fam.shutdown()
@@ -259,8 +361,9 @@ class BaseCore(object):
@property
def metadata_cache_mode(self):
- """ get the client metadata cache mode. options are off,
- initial, cautious, aggressive, on (synonym for cautious) """
+ """ Get the client :attr:`metadata_cache` mode. Options are
+ off, initial, cautious, aggressive, on (synonym for
+ cautious). See :ref:`server-caching` for more details. """
mode = self.setup.cfp.get("caching", "client_metadata",
default="off").lower()
if mode == "on":
@@ -269,7 +372,20 @@ class BaseCore(object):
return mode
def client_run_hook(self, hook, metadata):
- """invoke client run hooks for a given stage."""
+ """ Invoke hooks from
+ :class:`Bcfg2.Server.Plugin.interfaces.ClientRunHooks` plugins
+ for a given stage.
+
+ :param hook: The name of the stage to run hooks for. A stage
+ can be any abstract function defined in the
+ :class:`Bcfg2.Server.Plugin.interfaces.ClientRunHooks`
+ interface.
+ :type hook: string
+ :param metadata: Client metadata to run the hook for. This
+ will be passed as the sole argument to each
+ hook.
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ """
start = time.time()
try:
for plugin in \
@@ -291,7 +407,18 @@ class BaseCore(object):
@track_statistics()
def validate_structures(self, metadata, data):
- """Checks the data structure."""
+ """ Checks the data structures by calling the
+ :func:`Bcfg2.Server.Plugin.interfaces.StructureValidator.validate_structures`
+ method of
+ :class:`Bcfg2.Server.Plugin.interfaces.StructureValidator`
+ plugins.
+
+ :param metadata: Client metadata to validate structures for
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ :param data: The list of structures (i.e., bundles) for this
+ client
+ :type data: list of lxml.etree._Element objects
+ """
for plugin in \
self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator):
try:
@@ -307,7 +434,17 @@ class BaseCore(object):
@track_statistics()
def validate_goals(self, metadata, data):
- """Checks that the config matches the goals enforced by the plugins."""
+ """ Checks that the config matches the goals enforced by
+ :class:`Bcfg2.Server.Plugin.interfaces.GoalValidator` plugins
+ by calling
+ :func:`Bcfg2.Server.Plugin.interfaces.GoalValidator.validate_goals`.
+
+ :param metadata: Client metadata to validate goals for
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ :param data: The list of structures (i.e., bundles) for this
+ client
+ :type data: list of lxml.etree._Element objects
+ """
for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator):
try:
plugin.validate_goals(metadata, data)
@@ -322,7 +459,12 @@ class BaseCore(object):
@track_statistics()
def GetStructures(self, metadata):
- """Get all structures for client specified by metadata."""
+ """ Get all structures (i.e., bundles) for the given client
+
+ :param metadata: Client metadata to get structures for
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ :returns: list of :class:`lxml.etree._Element` objects
+ """
structures = reduce(lambda x, y: x + y,
[struct.BuildStructures(metadata)
for struct in self.structures], [])
@@ -335,8 +477,17 @@ class BaseCore(object):
@track_statistics()
def BindStructures(self, structures, metadata, config):
- """ Given a list of structures, bind all the entries in them
- and add the structures to the config. """
+ """ Given a list of structures (i.e. bundles), bind all the
+ entries in them and add the structures to the config.
+
+ :param structures: The list of structures for this client
+ :type structures: list of lxml.etree._Element objects
+ :param metadata: Client metadata to bind structures for
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ :param config: The configuration document to add fully-bound
+ structures to. Modified in-place.
+ :type config: lxml.etree._Element
+ """
for astruct in structures:
try:
self.BindStructure(astruct, metadata)
@@ -346,7 +497,13 @@ class BaseCore(object):
@track_statistics()
def BindStructure(self, structure, metadata):
- """Bind a complete structure."""
+ """ Bind all elements in a single structure (i.e., bundle).
+
+ :param structure: The structure to bind. Modified in-place.
+ :type structures: lxml.etree._Element
+ :param metadata: Client metadata to bind structure for
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ """
for entry in structure.getchildren():
if entry.tag.startswith("Bound"):
entry.tag = entry.tag[5:]
@@ -367,7 +524,13 @@ class BaseCore(object):
% (entry.tag, entry.get('name')), exc_info=1)
def Bind(self, entry, metadata):
- """Bind an entry using the appropriate generator."""
+ """ Bind a single entry using the appropriate generator.
+
+ :param entry: The entry to bind. Modified in-place.
+ :type entry: lxml.etree._Element
+ :param metadata: Client metadata to bind structure for
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ """
start = time.time()
if 'altsrc' in entry.attrib:
oldname = entry.get('name')
@@ -405,13 +568,19 @@ class BaseCore(object):
raise PluginExecutionError("No matching generator: %s:%s" %
(entry.tag, entry.get('name')))
finally:
- Bcfg2.Statistics.stats.add_value("%s:Bind:%s" %
+ Bcfg2.Statistics.stats.add_value("%s:Bind:%s" %
(self.__class__.__name__,
entry.tag),
time.time() - start)
def BuildConfiguration(self, client):
- """Build configuration for clients."""
+ """ Build the complete configuration for a client.
+
+ :param client: The hostname of the client to build the
+ configuration for
+ :type client: string
+ :returns: :class:`lxml.etree._Element` - A complete Bcfg2
+ configuration document """
start = time.time()
config = lxml.etree.Element("Configuration", version='2.0',
revision=self.revision)
@@ -458,7 +627,11 @@ class BaseCore(object):
return config
def HandleEvent(self, event):
- """ handle a change in the config file """
+ """ Handle a change in the Bcfg2 config file.
+
+ :param event: The event to handle
+ :type event: Bcfg2.Server.FileMonitor.Event
+ """
if event.filename != self.cfile:
print("Got event for unknown file: %s" % event.filename)
return
@@ -468,8 +641,12 @@ class BaseCore(object):
self.metadata_cache.expire()
def run(self):
- """ run the server core. note that it is the responsibility of
- the server core implementation to call shutdown() """
+ """ Run the server core. This calls :func:`_daemonize`,
+ :func:`_run`, starts the :attr:`fam_thread`, and calls
+ :func:`_block`, but note that it is the responsibility of the
+ server core implementation to call :func:`shutdown` under
+ normal operation. This also handles creation of the directory
+ containing the pidfile, if necessary. """
if self.setup['daemon']:
# if we're dropping privs, then the pidfile is likely
# /var/run/bcfg2-server/bcfg2-server.pid or similar.
@@ -497,24 +674,35 @@ class BaseCore(object):
self._block()
def _daemonize(self):
- """ daemonize the server and write the pidfile """
+ """ Daemonize the server and write the pidfile. This must be
+ overridden by a core implementation. """
raise NotImplementedError
def _run(self):
- """ start up the server; this method should return immediately """
+ """ Start up the server; this method should return
+ immediately. This must be overridden by a core
+ implementation. """
raise NotImplementedError
def _block(self):
- """ enter the infinite loop. this method should not return
- until the server is killed """
+ """ Enter the infinite loop. This method should not return
+ until the server is killed. This must be overridden by a core
+ implementation. """
raise NotImplementedError
def GetDecisions(self, metadata, mode):
- """Get data for the decision list."""
+ """ Get the decision list for a client.
+
+ :param metadata: Client metadata to get the decision list for
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ :param mode: The decision mode ("whitelist" or "blacklist")
+ :type mode: string
+ :returns: list of Decision tuples ``(<entry tag>, <entry name>)``
+ """
result = []
for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision):
try:
- result += plugin.GetDecisions(metadata, mode)
+ result.extend(plugin.GetDecisions(metadata, mode))
except:
self.logger.error("Plugin: %s failed to generate decision list"
% plugin.name, exc_info=1)
@@ -522,7 +710,13 @@ class BaseCore(object):
@track_statistics()
def build_metadata(self, client_name):
- """Build the metadata structure."""
+ """ Build initial client metadata for a client
+
+ :param client_name: The name of the client to build metadata
+ for
+ :type client_name: string
+ :returns: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
+ """
if not hasattr(self, 'metadata'):
# some threads start before metadata is even loaded
raise Bcfg2.Server.Plugin.MetadataRuntimeError
@@ -546,7 +740,14 @@ class BaseCore(object):
return imd
def process_statistics(self, client_name, statistics):
- """Proceed statistics for client."""
+ """ Process uploaded statistics for client.
+
+ :param client_name: The name of the client to process
+ statistics for
+ :type client_name: string
+ :param statistics: The statistics document to process
+ :type statistics: lxml.etree._Element
+ """
meta = self.build_metadata(client_name)
state = statistics.find(".//Statistics")
if state.get('version') >= '2.0':
@@ -563,8 +764,28 @@ class BaseCore(object):
self.client_run_hook("end_statistics", meta)
def resolve_client(self, address, cleanup_cache=False, metadata=True):
- """ given a client address, get the client hostname and
- optionally metadata """
+ """ Given a client address, get the client hostname and
+ optionally metadata.
+
+ :param address: The address pair of the client to get the
+ canonical hostname for.
+ :type address: tuple of (<ip address>, <hostname>)
+ :param cleanup_cache: Tell the
+ :class:`Bcfg2.Server.Plugin.interfaces.Metadata`
+ plugin in :attr:`metadata` to clean up
+ any client or session cache it might
+ keep
+ :type cleanup_cache: bool
+ :param metadata: Build a
+ :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
+ object for this client as well. This is
+ offered for convenience.
+ :type metadata: bool
+ :returns: tuple - If ``metadata`` is False, returns
+ ``(<canonical hostname>, None)``; if ``metadata`` is
+ True, returns ``(<canonical hostname>, <client
+ metadata object>)``
+ """
try:
client = self.metadata.resolve_client(address,
cleanup_cache=cleanup_cache)
@@ -582,11 +803,17 @@ class BaseCore(object):
(address[0], err))
return (client, meta)
- def critical_error(self, operation):
- """Log and err, traceback and return an xmlrpc fault to client."""
- self.logger.error(operation, exc_info=1)
+ def critical_error(self, message):
+ """ Log an error with its traceback and return an XML-RPC fault
+ to the client.
+
+ :param message: The message to log and return to the client
+ :type message: string
+ :raises: :exc:`xmlrpclib.Fault`
+ """
+ self.logger.error(message, exc_info=1)
raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR,
- "Critical failure: %s" % operation)
+ "Critical failure: %s" % message)
def _get_rmi(self):
""" Get a list of RMI calls exposed by plugins """
@@ -598,11 +825,12 @@ class BaseCore(object):
return rmi
def _resolve_exposed_method(self, method_name):
- """Resolve an exposed method.
-
- Arguments:
- method_name -- name of the method to resolve
+ """ Resolve a method name to the callable that implements that
+ method.
+ :param method_name: Name of the method to resolve
+ :type method_name: string
+ :returns: callable
"""
try:
func = getattr(self, method_name)
@@ -616,7 +844,12 @@ class BaseCore(object):
@exposed
def listMethods(self, address): # pylint: disable=W0613
- """ list all exposed methods, including plugin RMI """
+ """ List all exposed methods, including plugin RMI.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: list of exposed method names
+ """
methods = [name
for name, func in inspect.getmembers(self, callable)
if getattr(func, "exposed", False)]
@@ -625,7 +858,14 @@ class BaseCore(object):
@exposed
def methodHelp(self, address, method_name): # pylint: disable=W0613
- """ get help on an exposed method """
+ """ Get help from the docstring of an exposed method
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :param method_name: The name of the method to get help on
+ :type method_name: string
+ :returns: string - The help message from the method's docstring
+ """
try:
func = self._resolve_exposed_method(method_name)
except NoExposedMethod:
@@ -634,7 +874,15 @@ class BaseCore(object):
@exposed
def DeclareVersion(self, address, version):
- """ declare the client version """
+ """ Declare the client version.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :param version: The client's declared version
+ :type version: string
+ :returns: bool - True on success
+ :raises: :exc:`xmlrpclib.Fault`
+ """
client = self.resolve_client(address, metadata=False)[0]
try:
self.metadata.set_version(client, version)
@@ -647,7 +895,14 @@ class BaseCore(object):
@exposed
def GetProbes(self, address):
- """Fetch probes for a particular client."""
+ """ Fetch probes for the client.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: lxml.etree._Element - XML tree describing probes for
+ this client
+ :raises: :exc:`xmlrpclib.Fault`
+ """
resp = lxml.etree.Element('probes')
client, metadata = self.resolve_client(address, cleanup_cache=True)
try:
@@ -663,7 +918,13 @@ class BaseCore(object):
@exposed
def RecvProbeData(self, address, probedata):
- """Receive probe data from clients."""
+ """ Receive probe data from clients.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: bool - True on success
+ :raises: :exc:`xmlrpclib.Fault`
+ """
client, metadata = self.resolve_client(address)
if self.metadata_cache_mode == 'cautious':
# clear the metadata cache right after building the
@@ -701,7 +962,13 @@ class BaseCore(object):
@exposed
def AssertProfile(self, address, profile):
- """Set profile for a client."""
+ """ Set profile for a client.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: bool - True on success
+ :raises: :exc:`xmlrpclib.Fault`
+ """
client = self.resolve_client(address, metadata=False)[0]
try:
self.metadata.set_profile(client, profile, address)
@@ -714,7 +981,15 @@ class BaseCore(object):
@exposed
def GetConfig(self, address):
- """Build config for a client."""
+ """ Build config for a client by calling
+ :func:`BuildConfiguration`.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: lxml.etree._Element - The full configuration
+ document for the client
+ :raises: :exc:`xmlrpclib.Fault`
+ """
client = self.resolve_client(address)[0]
try:
config = self.BuildConfiguration(client)
@@ -725,15 +1000,33 @@ class BaseCore(object):
@exposed
def RecvStats(self, address, stats):
- """Act on statistics upload."""
+ """ Act on statistics upload with :func:`process_statistics`.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: bool - True on success
+ :raises: :exc:`xmlrpclib.Fault`
+ """
client = self.resolve_client(address)[0]
sdata = lxml.etree.XML(stats.encode('utf-8'),
parser=Bcfg2.Server.XMLParser)
self.process_statistics(client, sdata)
- return "<ok/>"
+ return True
def authenticate(self, cert, user, password, address):
- """ Authenticate a client connection """
+ """ Authenticate a client connection with
+ :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`.
+
+ :param cert: an x509 certificate
+ :type cert: dict
+ :param user: The username of the user trying to authenticate
+ :type user: string
+ :param password: The password supplied by the client
+ :type password: string
+ :param address: An address pair of ``(<ip address>, <hostname>)``
+ :type address: tuple
+ :return: bool - True if the authenticate succeeds, False otherwise
+ """
if self.ca:
acert = cert
else:
@@ -744,16 +1037,24 @@ class BaseCore(object):
@exposed
def GetDecisionList(self, address, mode):
- """Get the data of the decision list."""
+ """ Get the decision list for the client with :func:`GetDecisions`.
+
+ :param address: Client (address, hostname) pair
+ :type address: tuple
+ :returns: list of decision tuples
+ :raises: :exc:`xmlrpclib.Fault`
+ """
metadata = self.resolve_client(address)[1]
return self.GetDecisions(metadata, mode)
@property
def database_available(self):
- """Is the database configured and available"""
+ """ True if the database is configured and available, False
+ otherwise. """
return self._database_available
@exposed
def get_statistics(self, _):
- """Get current statistics about component execution"""
+ """ Get current statistics about component execution from
+ :attr:`Bcfg2.Statistics.stats`. """
return Bcfg2.Statistics.stats.display()
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 4a5f9cb90..5bc79a29a 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -1417,7 +1417,7 @@ class GroupSpool(Plugin, Generator):
self.entries = {}
self.handles = {}
self.AddDirectoryMonitor('')
- self.encoding = core.encoding
+ self.encoding = core.setup['encoding']
__init__.__doc__ = Plugin.__init__.__doc__
def add_entry(self, event):
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index 3bd6b7910..7933fe9be 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -86,7 +86,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
def __init__(self, core, datastore):
Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
Bcfg2.Server.Plugin.Structure.__init__(self)
- self.encoding = core.encoding
+ self.encoding = core.setup['encoding']
self.__child__ = self.template_dispatch
try:
Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self,
diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py
index 90d9ecbe3..5521ea045 100644
--- a/src/lib/Bcfg2/Server/Plugins/Decisions.py
+++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py
@@ -1,67 +1,67 @@
-import logging
-import lxml.etree
-import sys
+""" The Decisions plugin provides a flexible method to whitelist or
+blacklist certain entries. """
+import os
+import sys
+import lxml.etree
import Bcfg2.Server.Plugin
-logger = logging.getLogger('Bcfg2.Plugins.Decisions')
+
class DecisionFile(Bcfg2.Server.Plugin.SpecificData):
+ """ Representation of a Decisions XML file """
+
+ def __init__(self, name, specific, encoding):
+ Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific,
+ encoding)
+ self.contents = None
+
def handle_event(self, event):
Bcfg2.Server.Plugin.SpecificData.handle_event(self, event)
self.contents = lxml.etree.XML(self.data)
def get_decisions(self):
- return [(x.get('type'), x.get('name')) for x in self.contents.xpath('.//Decision')]
+ """ Get a list of whitelist or blacklist tuples """
+ return [(x.get('type'), x.get('name'))
+ for x in self.contents.xpath('.//Decision')]
-class DecisionSet(Bcfg2.Server.Plugin.EntrySet):
- basename_is_regex = True
- def __init__(self, path, fam, encoding):
- """Container for decision specification files.
+class Decisions(Bcfg2.Server.Plugin.EntrySet,
+ Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Decision):
+ """ Decisions plugin
- Arguments:
- - `path`: repository path
- - `fam`: reference to the file monitor
- - `encoding`: XML character encoding
+ Arguments:
+ - `core`: Bcfg2.Core instance
+ - `datastore`: File repository location
+ """
+ basename_is_regex = True
+ __author__ = 'bcfg-dev@mcs.anl.gov'
- """
- Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list', path,
- DecisionFile, encoding)
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Decision.__init__(self)
+
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list',
+ self.data,
+ DecisionFile,
+ core.setup['encoding'])
try:
- fam.AddMonitor(path, self)
+ core.fam.AddMonitor(self.data, self)
except OSError:
- e = sys.exc_info()[1]
- logger.error('Adding filemonitor for %s failed. '
- 'Make sure directory exists' % path)
- raise Bcfg2.Server.Plugin.PluginInitError(e)
+ err = sys.exc_info()[1]
+ msg = 'Adding filemonitor for %s failed: %s' % (self.data, err)
+ self.logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginInitError(msg)
def HandleEvent(self, event):
+ """ Handle events on Decision files by passing them off to
+ EntrySet.handle_event """
if event.filename != self.path:
return self.handle_event(event)
def GetDecisions(self, metadata, mode):
ret = []
- candidates = [c for c in self.get_matching(metadata)
- if c.name.split('/')[-1].startswith(mode)]
- for c in candidates:
- ret += c.get_decisions()
+ for cdt in self.get_matching(metadata):
+ if os.path.basename(cdt).startswith(mode):
+ ret.extend(cdt.get_decisions())
return ret
-
-class Decisions(DecisionSet,
- Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Decision):
- name = 'Decisions'
- __author__ = 'bcfg-dev@mcs.anl.gov'
-
- def __init__(self, core, datastore):
- """Decisions plugins
-
- Arguments:
- - `core`: Bcfg2.Core instance
- - `datastore`: File repository location
-
- """
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Decision.__init__(self)
- DecisionSet.__init__(self, self.data, core.fam, core.encoding)
-
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 761ebc2b7..606f34ea3 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -419,7 +419,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.session_cache = {}
self.default = None
self.pdirty = False
- self.password = core.password
+ self.password = core.setup['password']
self.query = MetadataQuery(core.build_metadata,
lambda: list(self.clients),
self.get_client_names_by_groups,
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 7ada08af7..c63f015e5 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -181,7 +181,7 @@ class Probes(Bcfg2.Server.Plugin.Probing,
Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore)
try:
- self.probes = ProbeSet(self.data, core.fam, core.encoding,
+ self.probes = ProbeSet(self.data, core.fam, core.setup['encoding'],
self.name)
except:
err = sys.exc_info()[1]
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
index 8a1d5a949..df55e2bf3 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
@@ -87,7 +87,7 @@ class TestDatabaseBacked(TestPlugin):
core.setup.cfp.getboolean.return_value = False
db = self.get_obj(core)
self.assertFalse(db._use_db)
-
+
Bcfg2.Server.Plugin.helpers.HAS_DJANGO = False
core = Mock()
db = self.get_obj(core)
@@ -1764,6 +1764,17 @@ class TestGroupSpool(TestPlugin, TestGenerator):
test_obj = GroupSpool
def get_obj(self, core=None):
+ if core is None:
+ print "creating core as a magicmock"
+ core = MagicMock()
+ core.setup = MagicMock()
+ else:
+ try:
+ core.setup['encoding']
+ except TypeError:
+ print "creating core.setup.__getitem__"
+ core.setup.__getitem__ = MagicMock()
+
@patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__,
self.test_obj.__name__),
Mock())
@@ -1773,11 +1784,10 @@ class TestGroupSpool(TestPlugin, TestGenerator):
return inner()
def test__init(self):
- core = Mock()
@patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__,
self.test_obj.__name__))
def inner(mock_Add):
- gs = self.test_obj(core, datastore)
+ gs = self.test_obj(MagicMock(), datastore)
mock_Add.assert_called_with('')
self.assertItemsEqual(gs.Entries, {gs.entry_type: {}})
@@ -1792,7 +1802,7 @@ class TestGroupSpool(TestPlugin, TestGenerator):
gs.event_id = Mock()
gs.event_path = Mock()
gs.AddDirectoryMonitor = Mock()
-
+
def reset():
gs.es_cls.reset_mock()
gs.es_child_cls.reset_mock()
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
index 5610d9071..38f4f2161 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
@@ -90,6 +90,7 @@ def get_groups_test_tree():
def get_metadata_object(core=None, watch_clients=False, use_db=False):
if core is None:
core = Mock()
+ core.setup = MagicMock()
core.metadata_cache = MagicMock()
core.setup.cfp.getboolean = Mock(return_value=use_db)
return Metadata(core, datastore, watch_clients=watch_clients)
@@ -248,7 +249,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
self.assertEqual(config.base_xdata, "<test/>")
def test_add_monitor(self):
- core = Mock()
+ core = MagicMock()
config = self.get_obj(core=core)
fname = "test.xml"
@@ -441,7 +442,7 @@ class TestMetadata(_TestMetadata, TestStatistics, TestDatabaseBacked):
def test__init(self):
# test with watch_clients=False
- core = Mock()
+ core = MagicMock()
metadata = self.get_obj(core=core)
self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Plugin)
self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Metadata)
@@ -452,7 +453,7 @@ class TestMetadata(_TestMetadata, TestStatistics, TestDatabaseBacked):
self.assertEqual(metadata.states, dict())
# test with watch_clients=True
- core.fam = Mock()
+ core.fam = MagicMock()
metadata = self.get_obj(core=core, watch_clients=True)
self.assertEqual(len(metadata.states), 2)
core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data,
@@ -1206,7 +1207,7 @@ class TestMetadataBase(TestMetadata):
@patch('os.path.exists')
def test__init(self, mock_exists):
- core = Mock()
+ core = MagicMock()
core.fam = Mock()
mock_exists.return_value = False
metadata = self.get_obj(core=core, watch_clients=True)
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
index a1d41b693..38d3c08e6 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
@@ -201,7 +201,7 @@ class TestProbes(TestProbing, TestConnector, TestDatabaseBacked):
def get_obj(self, core=None):
if core is None:
- core = Mock()
+ core = MagicMock()
return self.test_obj(core, datastore)
def get_test_probedata(self):
@@ -233,7 +233,7 @@ text
"bar.example.com": []}
def get_probes_object(self, use_db=False, load_data=None):
- core = Mock()
+ core = MagicMock()
core.setup.cfp.getboolean = Mock()
core.setup.cfp.getboolean.return_value = use_db
if load_data is None:
diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py
index 1d1d271d6..f5a4655ae 100644
--- a/testsuite/Testsrc/test_code_checks.py
+++ b/testsuite/Testsrc/test_code_checks.py
@@ -81,8 +81,7 @@ error_checks = {
"RcUpdate.py",
"VCS.py",
"YUM24.py"],
- "lib/Bcfg2/Server/Plugins": ["Decisions.py",
- "Deps.py",
+ "lib/Bcfg2/Server/Plugins": ["Deps.py",
"Ldap.py",
"Pkgmgr.py"]
}
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-07-01 12:40:49 +0000