From 1a6160ebeecffc57b5066ebf343188edf6a63eaa Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 12 Oct 2012 11:37:14 -0400 Subject: wrote sphinx docs for base server Core --- doc/development/core.txt | 62 +++ src/lib/Bcfg2/Server/Core.py | 445 +++++++++++++++++---- src/lib/Bcfg2/Server/Plugin/helpers.py | 2 +- src/lib/Bcfg2/Server/Plugins/Bundler.py | 2 +- src/lib/Bcfg2/Server/Plugins/Decisions.py | 88 ++-- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- src/lib/Bcfg2/Server/Plugins/Probes.py | 2 +- .../Testlib/TestServer/TestPlugin/Testhelpers.py | 18 +- .../Testlib/TestServer/TestPlugins/TestMetadata.py | 9 +- .../Testlib/TestServer/TestPlugins/TestProbes.py | 4 +- testsuite/Testsrc/test_code_checks.py | 3 +- 11 files changed, 505 insertions(+), 132 deletions(-) create mode 100644 doc/development/core.txt diff --git a/doc/development/core.txt b/doc/development/core.txt new file mode 100644 index 000000000..145be3338 --- /dev/null +++ b/doc/development/core.txt @@ -0,0 +1,62 @@ +.. -*- mode: rst -*- + +.. _development-core: + +========================= + Server Core Development +========================= + +.. versionadded:: 1.3.0 + +Bcfg2 1.3 added a pluggable server core system so that the server core +itself can be easily swapped out to use different technologies. It +currently ships with two backends: a builtin core written from scratch +using the various server tools in the Python standard library; and an +experimental `CherryPy `_ based core. This +page documents the server core interface so that other cores can be +written to take advantage of other technologies, e.g., `Tornado +`_ or `Twisted +`_. + +A core implementation needs to: + +* Override :func:`Bcfg2.Server.Core.Core._daemonize` to handle + daemonization, writing the PID file, and dropping privileges. +* Override :func:`Bcfg2.Server.Core.Core._run` to handle server + startup. +* Override :func:`Bcfg2.Server.Core.Core._block` to run the blocking + server loop. +* Call :func:`Bcfg2.Server.Core.Core.shutdown` on orderly shutdown. + +Nearly all XML-RPC handling is delegated entirely to the core +implementation. It needs to: + +* Call :func:`Bcfg2.Server.Core.Core.authenticate` to authenticate + clients. +* Handle :exc:`xmlrpclib.Fault` exceptions raised by the exposed + XML-RPC methods as appropriate. +* Dispatch XML-RPC method invocations to the appropriate method, + including Plugin RMI. + +Additionally, running and configuring the server is delegated to the +core. It needs to honor the configuration options that influence how +and where the server runs, including the server location (host and +port), listening interfaces, and SSL certificate and key. + +Base Core +========= + +.. automodule:: Bcfg2.Server.Core + +Core Implementations +==================== + +Builtin Core +------------ + +.. automodule:: Bcfg2.Server.BuiltinCore + +CherryPy Core +------------- + +.. automodule:: Bcfg2.Server.CherryPyCore diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 6ab18e36f..ee8c34fb8 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -1,4 +1,5 @@ -"""Bcfg2.Server.Core provides the runtime support for Bcfg2 modules.""" +""" Bcfg2.Server.Core provides the base core object that server core +implementations inherit from. """ import os import atexit @@ -30,14 +31,30 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' def exposed(func): - """ decorator that sets the 'exposed' attribute of a function to - expose it via XML-RPC """ + """ Decorator that sets the ``exposed`` attribute of a function to + ``True`` expose it via XML-RPC. This currently works for both the + builtin and CherryPy cores, although if other cores are added this + may need to be made a core-specific function. + + :param func: The function to decorate + :type func: callable + :returns: callable - the decorated function""" func.exposed = True return func def sort_xml(node, key=None): - """ sort XML in a deterministic fashion """ + """ Recursively sort an XML document in a deterministic fashion. + This shouldn't be used to perform a *useful* sort, merely to put + XML in a deterministic, replicable order. The document is sorted + in-place. + + :param node: The root node of the XML tree to sort + :type node: lxml.etree._Element or lxml.etree.ElementTree + :param key: The key to sort by + :type key: callable + :returns: None + """ for child in node: sort_xml(child, key) @@ -49,12 +66,13 @@ def sort_xml(node, key=None): class CoreInitError(Exception): - """This error is raised when the core cannot be initialized.""" + """ Raised when the server core cannot be initialized. """ pass class NoExposedMethod (Exception): - """There is no method exposed with the given name.""" + """ Raised when an XML-RPC method is called, but there is no + method exposed with the given name. """ # pylint: disable=W0702 @@ -63,11 +81,22 @@ class NoExposedMethod (Exception): class BaseCore(object): - """The Core object is the container for all - Bcfg2 Server logic and modules. - """ + """ The server core is the container for all Bcfg2 server logic + and modules. All core implementations must inherit from + ``BaseCore``. """ def __init__(self, setup): # pylint: disable=R0912,R0915 + """ + :param setup: A Bcfg2 options dict + :type setup: Bcfg2.Options.OptionParser + + .. automethod:: _daemonize + .. automethod:: _run + .. automethod:: _block + .. ----- + .. automethod:: _file_monitor_thread + """ + #: The Bcfg2 repository directory self.datastore = setup['repo'] if setup['debug']: @@ -86,6 +115,8 @@ class BaseCore(object): to_syslog=setup['syslog'], to_file=setup['logging'], level=level) + + #: A :class:`logging.Logger` object for use by the core self.logger = logging.getLogger('bcfg2-server') try: @@ -100,29 +131,51 @@ class BaseCore(object): famargs['ignore'] = setup['ignore'] if 'debug' in setup: famargs['debug'] = setup['debug'] + try: + #: The :class:`Bcfg2.Server.FileMonitor.FileMonitor` + #: object used by the core to monitor for Bcfg2 data + #: changes. self.fam = filemonitor(**famargs) except IOError: msg = "Failed to instantiate fam driver %s" % setup['filemonitor'] self.logger.error(msg, exc_info=1) raise CoreInitError(msg) - self.pubspace = {} + + #: Path to bcfg2.conf self.cfile = setup['configfile'] - self.cron = {} + + #: Dict of plugins that are enabled. Keys are the plugin + #: names (just the plugin name, in the correct case; e.g., + #: "Cfg", not "Bcfg2.Server.Plugins.Cfg"), and values are + #: plugin objects. self.plugins = {} + + #: Blacklist of plugins that conflict with enabled plugins. + #: If two plugins are loaded that conflict with each other, + #: the first one loaded wins. self.plugin_blacklist = {} + + #: Revision of the Bcfg2 specification. This will be sent to + #: the client in the configuration, and can be set by a + #: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin. self.revision = '-1' - self.password = setup['password'] - self.encoding = setup['encoding'] + + #: The Bcfg2 options dict self.setup = setup + atexit.register(self.shutdown) - # Create an event to signal worker threads to shutdown + + #: Threading event to signal worker threads (e.g., + #: :attr:`fam_thread`) to shutdown self.terminate = threading.Event() # generate Django ORM settings. this must be done _before_ we # load plugins Bcfg2.settings.read_config(repo=self.datastore) + #: Whether or not it's possible to use the Django database + #: backend for plugins that have that capability self._database_available = False if Bcfg2.settings.HAS_DJANGO: from django.core.exceptions import ImproperlyConfigured @@ -143,7 +196,7 @@ class BaseCore(object): for plugin in setup['plugins']: if not plugin in self.plugins: - self.init_plugins(plugin) + self.init_plugin(plugin) # Remove blacklisted plugins for plugin, blacklist in list(self.plugin_blacklist.items()): if len(blacklist) > 0: @@ -166,41 +219,82 @@ class BaseCore(object): (" ".join([x.name for x in depr]))) mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) - if len(mlist) == 1: + if len(mlist) >= 1: + #: The Metadata plugin self.metadata = mlist[0] + if len(mlist) > 1: + self.logger.error("Multiple Metadata plugins loaded; " + "using %s" % self.metadata) else: - self.logger.error("No Metadata Plugin loaded; " + self.logger.error("No Metadata plugin loaded; " "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") + + #: The list of plugins that handle + #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics` self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) + + #: The list of plugins that implement the + #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource` + #: interface self.pull_sources = \ self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) + + #: The list of + #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) + + #: The list of plugins that handle + #: :class:`Bcfg2.Server.Plugin.interfaces.Structure` + #: generation self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) + + #: The list of plugins that implement the + #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) + + #: The CA that signed the server cert self.ca = setup['ca'] + + #: The FAM :class:`threading.Thread`, + #: :func:`_file_monitor_thread` self.fam_thread = \ threading.Thread(name="%sFAMThread" % setup['filemonitor'], target=self._file_monitor_thread) + + #: A :func:`threading.Lock` for use by + #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() + #: A :class:`Bcfg2.Cache.Cache` object for caching client + #: metadata self.metadata_cache = Cache() def plugins_by_type(self, base_cls): - """Return a list of loaded plugins that match the passed type. + """ Return a list of loaded plugins that match the passed type. - The returned list is sorted in ascending order by the Plugins' - sort_order value. The sort_order defaults to 500 in Plugin.py, - but can be overridden by individual plugins. Plugins with the - same numerical sort_order value are sorted in alphabetical + The returned list is sorted in ascending order by the plugins' + ``sort_order`` value. The + :attr:`Bcfg2.Server.Plugin.base.Plugin.sort_order` defaults to + 500, but can be overridden by individual plugins. Plugins with + the same numerical sort_order value are sorted in alphabetical order by their name. + + :param base_cls: The base plugin interface class to match (see + :mod:`Bcfg2.Server.Plugin.interfaces`) + :type base_cls: type + :returns: list of :attr:`Bcfg2.Server.Plugin.base.Plugin` + objects """ return sorted([plugin for plugin in self.plugins.values() if isinstance(plugin, base_cls)], key=lambda p: (p.sort_order, p.name)) def _file_monitor_thread(self): - """The thread for monitor the files.""" + """ The thread that runs the + :class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also + queries :class:`Bcfg2.Server.Plugin.interfaces.Version` + plugins for the current revision of the Bcfg2 repo. """ famfd = self.fam.fileno() terminate = self.terminate while not terminate.isSet(): @@ -217,8 +311,16 @@ class BaseCore(object): for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): self.revision = plugin.get_revision() - def init_plugins(self, plugin): - """Handling for the plugins.""" + def init_plugin(self, plugin): + """ Import and instantiate a single plugin. The plugin is + stored to :attr:`plugins`. + + :param plugin: The name of the plugin. This is just the name + of the plugin, in the appropriate case. I.e., + ``Cfg``, not ``Bcfg2.Server.Plugins.Cfg``. + :type plugin: string + :returns: None + """ self.logger.debug("Loading plugin %s" % plugin) try: mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % @@ -250,7 +352,7 @@ class BaseCore(object): % plugin, exc_info=1) def shutdown(self): - """Shutting down the plugins.""" + """ Perform plugin and FAM shutdown tasks. """ if not self.terminate.isSet(): self.terminate.set() self.fam.shutdown() @@ -259,8 +361,9 @@ class BaseCore(object): @property def metadata_cache_mode(self): - """ get the client metadata cache mode. options are off, - initial, cautious, aggressive, on (synonym for cautious) """ + """ Get the client :attr:`metadata_cache` mode. Options are + off, initial, cautious, aggressive, on (synonym for + cautious). See :ref:`server-caching` for more details. """ mode = self.setup.cfp.get("caching", "client_metadata", default="off").lower() if mode == "on": @@ -269,7 +372,20 @@ class BaseCore(object): return mode def client_run_hook(self, hook, metadata): - """invoke client run hooks for a given stage.""" + """ Invoke hooks from + :class:`Bcfg2.Server.Plugin.interfaces.ClientRunHooks` plugins + for a given stage. + + :param hook: The name of the stage to run hooks for. A stage + can be any abstract function defined in the + :class:`Bcfg2.Server.Plugin.interfaces.ClientRunHooks` + interface. + :type hook: string + :param metadata: Client metadata to run the hook for. This + will be passed as the sole argument to each + hook. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ start = time.time() try: for plugin in \ @@ -291,7 +407,18 @@ class BaseCore(object): @track_statistics() def validate_structures(self, metadata, data): - """Checks the data structure.""" + """ Checks the data structures by calling the + :func:`Bcfg2.Server.Plugin.interfaces.StructureValidator.validate_structures` + method of + :class:`Bcfg2.Server.Plugin.interfaces.StructureValidator` + plugins. + + :param metadata: Client metadata to validate structures for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The list of structures (i.e., bundles) for this + client + :type data: list of lxml.etree._Element objects + """ for plugin in \ self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): try: @@ -307,7 +434,17 @@ class BaseCore(object): @track_statistics() def validate_goals(self, metadata, data): - """Checks that the config matches the goals enforced by the plugins.""" + """ Checks that the config matches the goals enforced by + :class:`Bcfg2.Server.Plugin.interfaces.GoalValidator` plugins + by calling + :func:`Bcfg2.Server.Plugin.interfaces.GoalValidator.validate_goals`. + + :param metadata: Client metadata to validate goals for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The list of structures (i.e., bundles) for this + client + :type data: list of lxml.etree._Element objects + """ for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): try: plugin.validate_goals(metadata, data) @@ -322,7 +459,12 @@ class BaseCore(object): @track_statistics() def GetStructures(self, metadata): - """Get all structures for client specified by metadata.""" + """ Get all structures (i.e., bundles) for the given client + + :param metadata: Client metadata to get structures for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: list of :class:`lxml.etree._Element` objects + """ structures = reduce(lambda x, y: x + y, [struct.BuildStructures(metadata) for struct in self.structures], []) @@ -335,8 +477,17 @@ class BaseCore(object): @track_statistics() def BindStructures(self, structures, metadata, config): - """ Given a list of structures, bind all the entries in them - and add the structures to the config. """ + """ Given a list of structures (i.e. bundles), bind all the + entries in them and add the structures to the config. + + :param structures: The list of structures for this client + :type structures: list of lxml.etree._Element objects + :param metadata: Client metadata to bind structures for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param config: The configuration document to add fully-bound + structures to. Modified in-place. + :type config: lxml.etree._Element + """ for astruct in structures: try: self.BindStructure(astruct, metadata) @@ -346,7 +497,13 @@ class BaseCore(object): @track_statistics() def BindStructure(self, structure, metadata): - """Bind a complete structure.""" + """ Bind all elements in a single structure (i.e., bundle). + + :param structure: The structure to bind. Modified in-place. + :type structures: lxml.etree._Element + :param metadata: Client metadata to bind structure for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ for entry in structure.getchildren(): if entry.tag.startswith("Bound"): entry.tag = entry.tag[5:] @@ -367,7 +524,13 @@ class BaseCore(object): % (entry.tag, entry.get('name')), exc_info=1) def Bind(self, entry, metadata): - """Bind an entry using the appropriate generator.""" + """ Bind a single entry using the appropriate generator. + + :param entry: The entry to bind. Modified in-place. + :type entry: lxml.etree._Element + :param metadata: Client metadata to bind structure for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ start = time.time() if 'altsrc' in entry.attrib: oldname = entry.get('name') @@ -405,13 +568,19 @@ class BaseCore(object): raise PluginExecutionError("No matching generator: %s:%s" % (entry.tag, entry.get('name'))) finally: - Bcfg2.Statistics.stats.add_value("%s:Bind:%s" % + Bcfg2.Statistics.stats.add_value("%s:Bind:%s" % (self.__class__.__name__, entry.tag), time.time() - start) def BuildConfiguration(self, client): - """Build configuration for clients.""" + """ Build the complete configuration for a client. + + :param client: The hostname of the client to build the + configuration for + :type client: string + :returns: :class:`lxml.etree._Element` - A complete Bcfg2 + configuration document """ start = time.time() config = lxml.etree.Element("Configuration", version='2.0', revision=self.revision) @@ -458,7 +627,11 @@ class BaseCore(object): return config def HandleEvent(self, event): - """ handle a change in the config file """ + """ Handle a change in the Bcfg2 config file. + + :param event: The event to handle + :type event: Bcfg2.Server.FileMonitor.Event + """ if event.filename != self.cfile: print("Got event for unknown file: %s" % event.filename) return @@ -468,8 +641,12 @@ class BaseCore(object): self.metadata_cache.expire() def run(self): - """ run the server core. note that it is the responsibility of - the server core implementation to call shutdown() """ + """ Run the server core. This calls :func:`_daemonize`, + :func:`_run`, starts the :attr:`fam_thread`, and calls + :func:`_block`, but note that it is the responsibility of the + server core implementation to call :func:`shutdown` under + normal operation. This also handles creation of the directory + containing the pidfile, if necessary. """ if self.setup['daemon']: # if we're dropping privs, then the pidfile is likely # /var/run/bcfg2-server/bcfg2-server.pid or similar. @@ -497,24 +674,35 @@ class BaseCore(object): self._block() def _daemonize(self): - """ daemonize the server and write the pidfile """ + """ Daemonize the server and write the pidfile. This must be + overridden by a core implementation. """ raise NotImplementedError def _run(self): - """ start up the server; this method should return immediately """ + """ Start up the server; this method should return + immediately. This must be overridden by a core + implementation. """ raise NotImplementedError def _block(self): - """ enter the infinite loop. this method should not return - until the server is killed """ + """ Enter the infinite loop. This method should not return + until the server is killed. This must be overridden by a core + implementation. """ raise NotImplementedError def GetDecisions(self, metadata, mode): - """Get data for the decision list.""" + """ Get the decision list for a client. + + :param metadata: Client metadata to get the decision list for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param mode: The decision mode ("whitelist" or "blacklist") + :type mode: string + :returns: list of Decision tuples ``(, )`` + """ result = [] for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): try: - result += plugin.GetDecisions(metadata, mode) + result.extend(plugin.GetDecisions(metadata, mode)) except: self.logger.error("Plugin: %s failed to generate decision list" % plugin.name, exc_info=1) @@ -522,7 +710,13 @@ class BaseCore(object): @track_statistics() def build_metadata(self, client_name): - """Build the metadata structure.""" + """ Build initial client metadata for a client + + :param client_name: The name of the client to build metadata + for + :type client_name: string + :returns: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` + """ if not hasattr(self, 'metadata'): # some threads start before metadata is even loaded raise Bcfg2.Server.Plugin.MetadataRuntimeError @@ -546,7 +740,14 @@ class BaseCore(object): return imd def process_statistics(self, client_name, statistics): - """Proceed statistics for client.""" + """ Process uploaded statistics for client. + + :param client_name: The name of the client to process + statistics for + :type client_name: string + :param statistics: The statistics document to process + :type statistics: lxml.etree._Element + """ meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': @@ -563,8 +764,28 @@ class BaseCore(object): self.client_run_hook("end_statistics", meta) def resolve_client(self, address, cleanup_cache=False, metadata=True): - """ given a client address, get the client hostname and - optionally metadata """ + """ Given a client address, get the client hostname and + optionally metadata. + + :param address: The address pair of the client to get the + canonical hostname for. + :type address: tuple of (, ) + :param cleanup_cache: Tell the + :class:`Bcfg2.Server.Plugin.interfaces.Metadata` + plugin in :attr:`metadata` to clean up + any client or session cache it might + keep + :type cleanup_cache: bool + :param metadata: Build a + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` + object for this client as well. This is + offered for convenience. + :type metadata: bool + :returns: tuple - If ``metadata`` is False, returns + ``(, None)``; if ``metadata`` is + True, returns ``(, )`` + """ try: client = self.metadata.resolve_client(address, cleanup_cache=cleanup_cache) @@ -582,11 +803,17 @@ class BaseCore(object): (address[0], err)) return (client, meta) - def critical_error(self, operation): - """Log and err, traceback and return an xmlrpc fault to client.""" - self.logger.error(operation, exc_info=1) + def critical_error(self, message): + """ Log an error with its traceback and return an XML-RPC fault + to the client. + + :param message: The message to log and return to the client + :type message: string + :raises: :exc:`xmlrpclib.Fault` + """ + self.logger.error(message, exc_info=1) raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR, - "Critical failure: %s" % operation) + "Critical failure: %s" % message) def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ @@ -598,11 +825,12 @@ class BaseCore(object): return rmi def _resolve_exposed_method(self, method_name): - """Resolve an exposed method. - - Arguments: - method_name -- name of the method to resolve + """ Resolve a method name to the callable that implements that + method. + :param method_name: Name of the method to resolve + :type method_name: string + :returns: callable """ try: func = getattr(self, method_name) @@ -616,7 +844,12 @@ class BaseCore(object): @exposed def listMethods(self, address): # pylint: disable=W0613 - """ list all exposed methods, including plugin RMI """ + """ List all exposed methods, including plugin RMI. + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: list of exposed method names + """ methods = [name for name, func in inspect.getmembers(self, callable) if getattr(func, "exposed", False)] @@ -625,7 +858,14 @@ class BaseCore(object): @exposed def methodHelp(self, address, method_name): # pylint: disable=W0613 - """ get help on an exposed method """ + """ Get help from the docstring of an exposed method + + :param address: Client (address, hostname) pair + :type address: tuple + :param method_name: The name of the method to get help on + :type method_name: string + :returns: string - The help message from the method's docstring + """ try: func = self._resolve_exposed_method(method_name) except NoExposedMethod: @@ -634,7 +874,15 @@ class BaseCore(object): @exposed def DeclareVersion(self, address, version): - """ declare the client version """ + """ Declare the client version. + + :param address: Client (address, hostname) pair + :type address: tuple + :param version: The client's declared version + :type version: string + :returns: bool - True on success + :raises: :exc:`xmlrpclib.Fault` + """ client = self.resolve_client(address, metadata=False)[0] try: self.metadata.set_version(client, version) @@ -647,7 +895,14 @@ class BaseCore(object): @exposed def GetProbes(self, address): - """Fetch probes for a particular client.""" + """ Fetch probes for the client. + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: lxml.etree._Element - XML tree describing probes for + this client + :raises: :exc:`xmlrpclib.Fault` + """ resp = lxml.etree.Element('probes') client, metadata = self.resolve_client(address, cleanup_cache=True) try: @@ -663,7 +918,13 @@ class BaseCore(object): @exposed def RecvProbeData(self, address, probedata): - """Receive probe data from clients.""" + """ Receive probe data from clients. + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - True on success + :raises: :exc:`xmlrpclib.Fault` + """ client, metadata = self.resolve_client(address) if self.metadata_cache_mode == 'cautious': # clear the metadata cache right after building the @@ -701,7 +962,13 @@ class BaseCore(object): @exposed def AssertProfile(self, address, profile): - """Set profile for a client.""" + """ Set profile for a client. + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - True on success + :raises: :exc:`xmlrpclib.Fault` + """ client = self.resolve_client(address, metadata=False)[0] try: self.metadata.set_profile(client, profile, address) @@ -714,7 +981,15 @@ class BaseCore(object): @exposed def GetConfig(self, address): - """Build config for a client.""" + """ Build config for a client by calling + :func:`BuildConfiguration`. + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: lxml.etree._Element - The full configuration + document for the client + :raises: :exc:`xmlrpclib.Fault` + """ client = self.resolve_client(address)[0] try: config = self.BuildConfiguration(client) @@ -725,15 +1000,33 @@ class BaseCore(object): @exposed def RecvStats(self, address, stats): - """Act on statistics upload.""" + """ Act on statistics upload with :func:`process_statistics`. + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - True on success + :raises: :exc:`xmlrpclib.Fault` + """ client = self.resolve_client(address)[0] sdata = lxml.etree.XML(stats.encode('utf-8'), parser=Bcfg2.Server.XMLParser) self.process_statistics(client, sdata) - return "" + return True def authenticate(self, cert, user, password, address): - """ Authenticate a client connection """ + """ Authenticate a client connection with + :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`. + + :param cert: an x509 certificate + :type cert: dict + :param user: The username of the user trying to authenticate + :type user: string + :param password: The password supplied by the client + :type password: string + :param address: An address pair of ``(, )`` + :type address: tuple + :return: bool - True if the authenticate succeeds, False otherwise + """ if self.ca: acert = cert else: @@ -744,16 +1037,24 @@ class BaseCore(object): @exposed def GetDecisionList(self, address, mode): - """Get the data of the decision list.""" + """ Get the decision list for the client with :func:`GetDecisions`. + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: list of decision tuples + :raises: :exc:`xmlrpclib.Fault` + """ metadata = self.resolve_client(address)[1] return self.GetDecisions(metadata, mode) @property def database_available(self): - """Is the database configured and available""" + """ True if the database is configured and available, False + otherwise. """ return self._database_available @exposed def get_statistics(self, _): - """Get current statistics about component execution""" + """ Get current statistics about component execution from + :attr:`Bcfg2.Statistics.stats`. """ return Bcfg2.Statistics.stats.display() diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 4a5f9cb90..5bc79a29a 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1417,7 +1417,7 @@ class GroupSpool(Plugin, Generator): self.entries = {} self.handles = {} self.AddDirectoryMonitor('') - self.encoding = core.encoding + self.encoding = core.setup['encoding'] __init__.__doc__ = Plugin.__init__.__doc__ def add_entry(self, event): diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 3bd6b7910..7933fe9be 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -86,7 +86,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) - self.encoding = core.encoding + self.encoding = core.setup['encoding'] self.__child__ = self.template_dispatch try: Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 90d9ecbe3..5521ea045 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -1,67 +1,67 @@ -import logging -import lxml.etree -import sys +""" The Decisions plugin provides a flexible method to whitelist or +blacklist certain entries. """ +import os +import sys +import lxml.etree import Bcfg2.Server.Plugin -logger = logging.getLogger('Bcfg2.Plugins.Decisions') + class DecisionFile(Bcfg2.Server.Plugin.SpecificData): + """ Representation of a Decisions XML file """ + + def __init__(self, name, specific, encoding): + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, + encoding) + self.contents = None + def handle_event(self, event): Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) self.contents = lxml.etree.XML(self.data) def get_decisions(self): - return [(x.get('type'), x.get('name')) for x in self.contents.xpath('.//Decision')] + """ Get a list of whitelist or blacklist tuples """ + return [(x.get('type'), x.get('name')) + for x in self.contents.xpath('.//Decision')] -class DecisionSet(Bcfg2.Server.Plugin.EntrySet): - basename_is_regex = True - def __init__(self, path, fam, encoding): - """Container for decision specification files. +class Decisions(Bcfg2.Server.Plugin.EntrySet, + Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Decision): + """ Decisions plugin - Arguments: - - `path`: repository path - - `fam`: reference to the file monitor - - `encoding`: XML character encoding + Arguments: + - `core`: Bcfg2.Core instance + - `datastore`: File repository location + """ + basename_is_regex = True + __author__ = 'bcfg-dev@mcs.anl.gov' - """ - Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list', path, - DecisionFile, encoding) + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Decision.__init__(self) + + Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list', + self.data, + DecisionFile, + core.setup['encoding']) try: - fam.AddMonitor(path, self) + core.fam.AddMonitor(self.data, self) except OSError: - e = sys.exc_info()[1] - logger.error('Adding filemonitor for %s failed. ' - 'Make sure directory exists' % path) - raise Bcfg2.Server.Plugin.PluginInitError(e) + err = sys.exc_info()[1] + msg = 'Adding filemonitor for %s failed: %s' % (self.data, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) def HandleEvent(self, event): + """ Handle events on Decision files by passing them off to + EntrySet.handle_event """ if event.filename != self.path: return self.handle_event(event) def GetDecisions(self, metadata, mode): ret = [] - candidates = [c for c in self.get_matching(metadata) - if c.name.split('/')[-1].startswith(mode)] - for c in candidates: - ret += c.get_decisions() + for cdt in self.get_matching(metadata): + if os.path.basename(cdt).startswith(mode): + ret.extend(cdt.get_decisions()) return ret - -class Decisions(DecisionSet, - Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Decision): - name = 'Decisions' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - """Decisions plugins - - Arguments: - - `core`: Bcfg2.Core instance - - `datastore`: File repository location - - """ - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Decision.__init__(self) - DecisionSet.__init__(self, self.data, core.fam, core.encoding) - diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 761ebc2b7..606f34ea3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -419,7 +419,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.session_cache = {} self.default = None self.pdirty = False - self.password = core.password + self.password = core.setup['password'] self.query = MetadataQuery(core.build_metadata, lambda: list(self.clients), self.get_client_names_by_groups, diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7ada08af7..c63f015e5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -181,7 +181,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: - self.probes = ProbeSet(self.data, core.fam, core.encoding, + self.probes = ProbeSet(self.data, core.fam, core.setup['encoding'], self.name) except: err = sys.exc_info()[1] diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 8a1d5a949..df55e2bf3 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -87,7 +87,7 @@ class TestDatabaseBacked(TestPlugin): core.setup.cfp.getboolean.return_value = False db = self.get_obj(core) self.assertFalse(db._use_db) - + Bcfg2.Server.Plugin.helpers.HAS_DJANGO = False core = Mock() db = self.get_obj(core) @@ -1764,6 +1764,17 @@ class TestGroupSpool(TestPlugin, TestGenerator): test_obj = GroupSpool def get_obj(self, core=None): + if core is None: + print "creating core as a magicmock" + core = MagicMock() + core.setup = MagicMock() + else: + try: + core.setup['encoding'] + except TypeError: + print "creating core.setup.__getitem__" + core.setup.__getitem__ = MagicMock() + @patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__, self.test_obj.__name__), Mock()) @@ -1773,11 +1784,10 @@ class TestGroupSpool(TestPlugin, TestGenerator): return inner() def test__init(self): - core = Mock() @patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__, self.test_obj.__name__)) def inner(mock_Add): - gs = self.test_obj(core, datastore) + gs = self.test_obj(MagicMock(), datastore) mock_Add.assert_called_with('') self.assertItemsEqual(gs.Entries, {gs.entry_type: {}}) @@ -1792,7 +1802,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): gs.event_id = Mock() gs.event_path = Mock() gs.AddDirectoryMonitor = Mock() - + def reset(): gs.es_cls.reset_mock() gs.es_child_cls.reset_mock() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index 5610d9071..38f4f2161 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -90,6 +90,7 @@ def get_groups_test_tree(): def get_metadata_object(core=None, watch_clients=False, use_db=False): if core is None: core = Mock() + core.setup = MagicMock() core.metadata_cache = MagicMock() core.setup.cfp.getboolean = Mock(return_value=use_db) return Metadata(core, datastore, watch_clients=watch_clients) @@ -248,7 +249,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertEqual(config.base_xdata, "") def test_add_monitor(self): - core = Mock() + core = MagicMock() config = self.get_obj(core=core) fname = "test.xml" @@ -441,7 +442,7 @@ class TestMetadata(_TestMetadata, TestStatistics, TestDatabaseBacked): def test__init(self): # test with watch_clients=False - core = Mock() + core = MagicMock() metadata = self.get_obj(core=core) self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Plugin) self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Metadata) @@ -452,7 +453,7 @@ class TestMetadata(_TestMetadata, TestStatistics, TestDatabaseBacked): self.assertEqual(metadata.states, dict()) # test with watch_clients=True - core.fam = Mock() + core.fam = MagicMock() metadata = self.get_obj(core=core, watch_clients=True) self.assertEqual(len(metadata.states), 2) core.fam.AddMonitor.assert_any_call(os.path.join(metadata.data, @@ -1206,7 +1207,7 @@ class TestMetadataBase(TestMetadata): @patch('os.path.exists') def test__init(self, mock_exists): - core = Mock() + core = MagicMock() core.fam = Mock() mock_exists.return_value = False metadata = self.get_obj(core=core, watch_clients=True) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index a1d41b693..38d3c08e6 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -201,7 +201,7 @@ class TestProbes(TestProbing, TestConnector, TestDatabaseBacked): def get_obj(self, core=None): if core is None: - core = Mock() + core = MagicMock() return self.test_obj(core, datastore) def get_test_probedata(self): @@ -233,7 +233,7 @@ text "bar.example.com": []} def get_probes_object(self, use_db=False, load_data=None): - core = Mock() + core = MagicMock() core.setup.cfp.getboolean = Mock() core.setup.cfp.getboolean.return_value = use_db if load_data is None: diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 1d1d271d6..f5a4655ae 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -81,8 +81,7 @@ error_checks = { "RcUpdate.py", "VCS.py", "YUM24.py"], - "lib/Bcfg2/Server/Plugins": ["Decisions.py", - "Deps.py", + "lib/Bcfg2/Server/Plugins": ["Deps.py", "Ldap.py", "Pkgmgr.py"] } -- cgit v1.2.3-1-g7c22