diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/__init__.py | 11 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/base.py | 106 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/exceptions.py | 36 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/helpers.py (renamed from src/lib/Bcfg2/Server/Plugin.py) | 689 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/interfaces.py | 548 |
5 files changed, 707 insertions, 683 deletions
diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py new file mode 100644 index 000000000..487a457e6 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugin/__init__.py @@ -0,0 +1,11 @@ +""" Bcfg2 server plugin base classes, interfaces, and helper +objects. """ + +import os +import sys +sys.path.append(os.path.dirname(__file__)) + +from base import * +from interfaces import * +from helpers import * +from exceptions import * diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py new file mode 100644 index 000000000..98427e726 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -0,0 +1,106 @@ +"""This module provides the base class for Bcfg2 server plugins.""" + +import os +import logging + +class Debuggable(object): + """ Mixin to add a debugging interface to an object and expose it + via XML-RPC on :class:`Bcfg2.Server.Plugin.Plugin` objects """ + + #: List of names of methods to be exposed as XML-RPC functions + __rmi__ = ['toggle_debug'] + + def __init__(self, name=None): + if name is None: + name = "%s.%s" % (self.__class__.__module__, + self.__class__.__name__) + self.debug_flag = False + self.logger = logging.getLogger(name) + + def toggle_debug(self): + """ Turn debugging output on or off. + + :returns: bool - The new value of the debug flag + """ + self.debug_flag = not self.debug_flag + self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, + self.debug_flag), + flag=True) + return self.debug_flag + + def debug_log(self, message, flag=None): + """ Log a message at the debug level. + + :param message: The message to log + :type message: string + :param flag: Override the current debug flag with this value + :type flag: bool + :returns: None + """ + if (flag is None and self.debug_flag) or flag: + self.logger.error(message) + + +class Plugin(Debuggable): + """ The base class for all Bcfg2 Server plugins. """ + + #: The name of the plugin. + name = 'Plugin' + + #: The email address of the plugin author. + __author__ = 'bcfg-dev@mcs.anl.gov' + + #: Plugin is experimental. Use of this plugin will produce a log + #: message alerting the administrator that an experimental plugin + #: is in use. + experimental = False + + #: Plugin is deprecated and will be removed in a future release. + #: Use of this plugin will produce a log message alerting the + #: administrator that an experimental plugin is in use. + deprecated = False + + #: Plugin conflicts with the list of other plugin names + conflicts = [] + + #: Plugins of the same type are processed in order of ascending + #: sort_order value. Plugins with the same sort_order are sorted + #: alphabetically by their name. + sort_order = 500 + + def __init__(self, core, datastore): + """ Initialize the plugin. + + :param core: The Bcfg2.Server.Core initializing the plugin + :type core: Bcfg2.Server.Core + :param datastore: The path to the Bcfg2 repository on the + filesystem + :type datastore: string + :raises: Bcfg2.Server.Plugin.PluginInitError + """ + object.__init__(self) + self.Entries = {} + self.core = core + self.data = os.path.join(datastore, self.name) + self.running = True + Debuggable.__init__(self, name=self.name) + + @classmethod + def init_repo(cls, repo): + """ Perform any tasks necessary to create an initial Bcfg2 + repository. + + :param repo: The path to the Bcfg2 repository on the filesystem + :type repo: string + :returns: None + """ + os.makedirs(os.path.join(repo, cls.name)) + + def shutdown(self): + """ Perform shutdown tasks for the plugin + + :returns: None """ + self.running = False + + def __str__(self): + return "%s Plugin" % self.__class__.__name__ diff --git a/src/lib/Bcfg2/Server/Plugin/exceptions.py b/src/lib/Bcfg2/Server/Plugin/exceptions.py new file mode 100644 index 000000000..bc8c62acd --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugin/exceptions.py @@ -0,0 +1,36 @@ +""" Exceptions for Bcfg2 Server Plugins.""" + +class PluginInitError(Exception): + """Error raised in cases of :class:`Bcfg2.Server.Plugin.Plugin` + initialization errors.""" + pass + + +class PluginExecutionError(Exception): + """Error raised in case of :class:`Bcfg2.Server.Plugin.Plugin` + execution errors.""" + pass + + +class MetadataConsistencyError(Exception): + """This error gets raised when metadata is internally inconsistent.""" + pass + + +class MetadataRuntimeError(Exception): + """This error is raised when the metadata engine is called prior + to reading enough data, or for other + :class:`Bcfg2.Server.Plugin.Metadata` errors. """ + pass + + +class ValidationError(Exception): + """ Exception raised by + :class:`Bcfg2.Server.Plugin.StructureValidator` and + :class:`Bcfg2.Server.Plugin.GoalValidator` objects """ + + +class SpecificityError(Exception): + """ Thrown by :class:`Bcfg2.Server.Plugin.Specificity` in case of + filename parse failure.""" + pass diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 0b2f7cee0..74cf4b3c4 100644 --- a/src/lib/Bcfg2/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1,4 +1,4 @@ -"""This module provides the baseclass for Bcfg2 Server Plugins.""" +""" Helper classes for Bcfg2 server plugins """ import os import re @@ -6,12 +6,13 @@ import sys import copy import logging import operator -import threading import lxml.etree import Bcfg2.Server import Bcfg2.Options -from Bcfg2.Compat import ConfigParser, CmpMixin, reduce, Queue, Empty, \ - Full, cPickle +from Bcfg2.Compat import CmpMixin +from base import * +from interfaces import * +from exceptions import * try: import django @@ -31,7 +32,7 @@ default_file_metadata = Bcfg2.Options.OptionParser(opts) default_file_metadata.parse([]) del default_file_metadata['args'] -logger = logging.getLogger('Bcfg2.Server.Plugin') +logger = logging.getLogger(__name__) info_regex = re.compile('owner:(\s)*(?P<owner>\S+)|' + 'group:(\s)*(?P<group>\S+)|' + @@ -57,133 +58,6 @@ def bind_info(entry, metadata, infoxml=None, default=default_file_metadata): entry.set(attr, val) -class PluginInitError(Exception): - """Error raised in cases of :class:`Bcfg2.Server.Plugin.Plugin` - initialization errors.""" - pass - - -class PluginExecutionError(Exception): - """Error raised in case of :class:`Bcfg2.Server.Plugin.Plugin` - execution errors.""" - pass - - -class MetadataConsistencyError(Exception): - """This error gets raised when metadata is internally inconsistent.""" - pass - - -class MetadataRuntimeError(Exception): - """This error is raised when the metadata engine is called prior - to reading enough data, or for other - :class:`Bcfg2.Server.Plugin.Metadata` errors. """ - pass - - -class Debuggable(object): - """ Mixin to add a debugging interface to an object and expose it - via XML-RPC on :class:`Bcfg2.Server.Plugin.Plugin` objects """ - - #: List of names of methods to be exposed as XML-RPC functions - __rmi__ = ['toggle_debug'] - - def __init__(self, name=None): - if name is None: - name = "%s.%s" % (self.__class__.__module__, - self.__class__.__name__) - self.debug_flag = False - self.logger = logging.getLogger(name) - - def toggle_debug(self): - """ Turn debugging output on or off. - - :returns: bool - The new value of the debug flag - """ - self.debug_flag = not self.debug_flag - self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, - self.debug_flag), - flag=True) - return self.debug_flag - - def debug_log(self, message, flag=None): - """ Log a message at the debug level. - - :param message: The message to log - :type message: string - :param flag: Override the current debug flag with this value - :type flag: bool - :returns: None - """ - if (flag is None and self.debug_flag) or flag: - self.logger.error(message) - - -class Plugin(Debuggable): - """ The base class for all Bcfg2 Server plugins. """ - - #: The name of the plugin. - name = 'Plugin' - - #: The email address of the plugin author. - __author__ = 'bcfg-dev@mcs.anl.gov' - - #: Plugin is experimental. Use of this plugin will produce a log - #: message alerting the administrator that an experimental plugin - #: is in use. - experimental = False - - #: Plugin is deprecated and will be removed in a future release. - #: Use of this plugin will produce a log message alerting the - #: administrator that an experimental plugin is in use. - deprecated = False - - #: Plugin conflicts with the list of other plugin names - conflicts = [] - - #: Plugins of the same type are processed in order of ascending - #: sort_order value. Plugins with the same sort_order are sorted - #: alphabetically by their name. - sort_order = 500 - - def __init__(self, core, datastore): - """ Initialize the plugin. - - :param core: The Bcfg2.Server.Core initializing the plugin - :type core: Bcfg2.Server.Core - :param datastore: The path to the Bcfg2 repository on the - filesystem - :type datastore: string - :raises: Bcfg2.Server.Plugin.PluginInitError - """ - object.__init__(self) - self.Entries = {} - self.core = core - self.data = os.path.join(datastore, self.name) - self.running = True - Debuggable.__init__(self, name=self.name) - - @classmethod - def init_repo(cls, repo): - """ Perform any tasks necessary to create an initial Bcfg2 - repository. - - :param repo: The path to the Bcfg2 repository on the filesystem - :type repo: string - :returns: None - """ - os.makedirs(os.path.join(repo, cls.name)) - - def shutdown(self): - """ Perform shutdown tasks for the plugin - - :returns: None """ - self.running = False - - def __str__(self): - return "%s Plugin" % self.__class__.__name__ - - class DatabaseBacked(Plugin): @property def _use_db(self): @@ -204,551 +78,6 @@ class PluginDatabaseModel(object): app_label = "Server" -class Generator(object): - """ Generator plugins contribute to literal client configurations. - That is, they generate entry contents. - - An entry is generated in one of two ways: - - #. The Bcfg2 core looks in the ``Entries`` dict attribute of the - plugin object. ``Entries`` is expected to be a dict whose keys - are entry tags (e.g., ``"Path"``, ``"Service"``, etc.) and - whose values are dicts; those dicts should map the ``name`` - attribute of an entry to a callable that will be called to - generate the content. The callable will receive two arguments: - the abstract entry (as an lxml.etree._Element object), and the - client metadata object the entry is being generated for. - - #. If the entry is not listed in ``Entries``, the Bcfg2 core calls - :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry`; if that - returns True, then it calls - :func:`Bcfg2.Server.Plugin.Generator.HandleEntry`. - """ - - def HandlesEntry(self, entry, metadata): - """ HandlesEntry is the slow path method for routing - configuration binding requests. It is called if the - ``Entries`` dict does not contain a method for binding the - entry. - - :param entry: The entry to bind - :type entry: lxml.etree._Element - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: bool - Whether or not this plugin can handle the entry - :raises: Bcfg2.Server.Plugin.PluginExecutionError - """ - return False - - def HandleEntry(self, entry, metadata): - """ HandlesEntry is the slow path method for binding - configuration binding requests. It is called if the - ``Entries`` dict does not contain a method for binding the - entry, and :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry` - returns True. - - :param entry: The entry to bind - :type entry: lxml.etree._Element - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: lxml.etree._Element - The fully bound entry - :raises: Bcfg2.Server.Plugin.PluginExecutionError - """ - return entry - - -class Structure(object): - """ Structure Plugins contribute to abstract client - configurations. That is, they produce lists of entries that will - be generated for a client. """ - - def BuildStructures(self, metadata): - """ Build a list of lxml.etree._Element objects that will be - added to the top-level ``<Configuration>`` tag of the client - configuration. Consequently, each object in the list returned - by ``BuildStructures()`` must consist of a container tag - (e.g., ``<Bundle>`` or ``<Independent>``) which contains the - entry tags. It must not return a list of entry tags. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of lxml.etree._Element objects - """ - raise NotImplementedError - - -class Metadata(object): - """Signal metadata capabilities for this plugin""" - def viz(self, hosts, bundles, key, only_client, colors): - """ Return a string containing a graphviz document that maps - out the Metadata for :ref:`bcfg2-admin viz <server-admin-viz>` - - :param hosts: Include hosts in the graph - :type hosts: bool - :param bundles: Include bundles in the graph - :type bundles: bool - :param key: Include a key in the graph - :type key: bool - :param only_client: Only include data for the specified client - :type only_client: string - :param colors: Use the specified graphviz colors - :type colors: list of strings - :return: string - """ - return '' - - def set_version(self, client, version): - """ Set the version for the named client to the specified - version string. - - :param client: Hostname of the client - :type client: string - :param profile: Client Bcfg2 version - :type profile: string - :return: None - :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, - Bcfg2.Server.Plugin.MetadataConsistencyError - """ - pass - - def set_profile(self, client, profile, address): - """ Set the profile for the named client to the named profile - group. - - :param client: Hostname of the client - :type client: string - :param profile: Name of the profile group - :type profile: string - :param address: Address pair of ``(<ip address>, <hostname>)`` - :type address: tuple - :return: None - :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, - Bcfg2.Server.Plugin.MetadataConsistencyError - """ - pass - - def resolve_client(self, address, cleanup_cache=False): - """ Resolve the canonical name of this client. If this method - is not implemented, the hostname claimed by the client is - used. (This may be a security risk; it's highly recommended - that you implement ``resolve_client`` if you are writing a - Metadata plugin.) - - :param address: Address pair of ``(<ip address>, <hostname>)`` - :type address: tuple - :param cleanup_cache: Whether or not to remove expire the - entire client hostname resolution class - :type cleanup_cache: bool - :return: string - canonical client hostname - :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, - Bcfg2.Server.Plugin.MetadataConsistencyError - """ - return address[1] - - def AuthenticateConnection(self, cert, user, password, address): - """ Authenticate the given client. - - :param cert: an x509 certificate - :type cert: dict - :param user: The username of the user trying to authenticate - :type user: string - :param password: The password supplied by the client - :type password: string - :param addresspair: An address pair of ``(<ip address>, - <hostname>)`` - :type addresspair: tuple - :return: bool - True if the authenticate succeeds, False otherwise - """ - raise NotImplementedError - - def get_initial_metadata(self, client_name): - """ Return a - :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` object - that fully describes everything the Metadata plugin knows - about the named client. - - :param client_name: The hostname of the client - :type client_name: string - :return: Bcfg2.Server.Plugins.Metadata.ClientMetadata - """ - raise NotImplementedError - - def merge_additional_data(self, imd, source, data): - """ Add arbitrary data from a - :class:`Bcfg2.Server.Plugin.Connector` plugin to the given - metadata object. - - :param imd: An initial metadata object - :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param source: The name of the plugin providing this data - :type source: string - :param data: The data to add - :type data: any - :return: None - """ - raise NotImplementedError - - def merge_additional_groups(self, imd, groups): - """ Add groups from a - :class:`Bcfg2.Server.Plugin.Connector` plugin to the given - metadata object. - - :param imd: An initial metadata object - :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param groups: The groups to add - :type groups: list of strings - :return: None - """ - raise NotImplementedError - - -class Connector(object): - """ Connector plugins augment client metadata instances with - additional data, additional groups, or both. """ - - def get_additional_groups(self, metadata): - """ Return a list of additional groups for the given client. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of strings - """ - return list() - - def get_additional_data(self, metadata): - """ Return arbitrary additional data for the given - ClientMetadata object. By convention this is usually a dict - object, but doesn't need to be. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of strings - """ - return dict() - - -class Probing(object): - """ Probing plugins can collect data from clients and process it. - """ - - def GetProbes(self, metadata): - """ Return a list of probes for the given client. Each probe - should be an lxml.etree._Element object that adheres to - the following specification. Each probe must the following - attributes: - - * ``name``: The unique name of the probe. - * ``source``: The origin of the probe; probably the name of - the plugin that supplies the probe. - * ``interpreter``: The command that will be run on the client - to interpret the probe script. Compiled (i.e., - non-interpreted) probes are not supported. - - The text of the XML tag should be the contents of the probe, - i.e., the code that will be run on the client. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of lxml.etree._Element objects - """ - raise NotImplementedError - - def ReceiveData(self, metadata, datalist): - """ Process data returned from the probes for the given - client. ``datalist`` is a list of lxml.etree._Element - objects, each of which is a single tag; the ``name`` attribute - holds the unique name of the probe that was run, and the text - contents of the tag hold the results of the probe. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param datalist: The probe data - :type datalist: list of lxml.etree._Element objects - :return: None - """ - raise NotImplementedError - - -class Statistics(Plugin): - """ Statistics plugins handle statistics for clients. In general, - you should avoid using Statistics and use - :class:`Bcfg2.Server.Plugin.ThreadedStatistics` instead.""" - - def process_statistics(self, client, xdata): - """ Process the given XML statistics data for the specified - client. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param data: The statistics data - :type data: lxml.etree._Element - :return: None - """ - raise NotImplementedError - - -class ThreadedStatistics(Statistics, threading.Thread): - """ ThreadedStatistics plugins process client statistics in a - separate thread. """ - - def __init__(self, core, datastore): - Statistics.__init__(self, core, datastore) - threading.Thread.__init__(self) - # Event from the core signaling an exit - self.terminate = core.terminate - self.work_queue = Queue(100000) - self.pending_file = os.path.join(datastore, "etc", - "%s.pending" % self.name) - self.daemon = False - self.start() - - def _save(self): - """Save any pending data to a file.""" - pending_data = [] - try: - while not self.work_queue.empty(): - (metadata, data) = self.work_queue.get_nowait() - try: - pending_data.append((metadata.hostname, - lxml.etree.tostring(data, - xml_declaration=False).decode("UTF-8"))) - except: - err = sys.exc_info()[1] - self.logger.warning("Dropping interaction for %s: %s" % - (metadata.hostname, err)) - except Empty: - pass - - try: - savefile = open(self.pending_file, 'w') - cPickle.dump(pending_data, savefile) - savefile.close() - self.logger.info("Saved pending %s data" % self.name) - except: - err = sys.exc_info()[1] - self.logger.warning("Failed to save pending data: %s" % err) - - def _load(self): - """Load any pending data from a file.""" - if not os.path.exists(self.pending_file): - return True - pending_data = [] - try: - savefile = open(self.pending_file, 'r') - pending_data = cPickle.load(savefile) - savefile.close() - except Exception: - e = sys.exc_info()[1] - self.logger.warning("Failed to load pending data: %s" % e) - return False - for (pmetadata, pdata) in pending_data: - # check that shutdown wasnt called early - if self.terminate.isSet(): - return False - - try: - while True: - try: - metadata = self.core.build_metadata(pmetadata) - break - except MetadataRuntimeError: - pass - - self.terminate.wait(5) - if self.terminate.isSet(): - return False - - self.work_queue.put_nowait((metadata, - lxml.etree.XML(pdata, - parser=Bcfg2.Server.XMLParser))) - except Full: - self.logger.warning("Queue.Full: Failed to load queue data") - break - except lxml.etree.LxmlError: - lxml_error = sys.exc_info()[1] - self.logger.error("Unable to load saved interaction: %s" % - lxml_error) - except MetadataConsistencyError: - self.logger.error("Unable to load metadata for save " - "interaction: %s" % pmetadata) - try: - os.unlink(self.pending_file) - except: - self.logger.error("Failed to unlink save file: %s" % - self.pending_file) - self.logger.info("Loaded pending %s data" % self.name) - return True - - def run(self): - if not self._load(): - return - while not self.terminate.isSet() and self.work_queue != None: - try: - (client, xdata) = self.work_queue.get(block=True, timeout=2) - except Empty: - continue - except Exception: - e = sys.exc_info()[1] - self.logger.error("ThreadedStatistics: %s" % e) - continue - self.handle_statistic(client, xdata) - if self.work_queue != None and not self.work_queue.empty(): - self._save() - - def process_statistics(self, metadata, data): - try: - self.work_queue.put_nowait((metadata, copy.copy(data))) - except Full: - self.logger.warning("%s: Queue is full. Dropping interactions." % - self.name) - - def handle_statistic(self, metadata, data): - """ Process the given XML statistics data for the specified - client object. This differs from the - :func:`Bcfg2.Server.Plugin.Statistics.process_statistics` - method only in that ThreadedStatistics first adds the data to - a queue, and then processes them in a separate thread. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param data: The statistics data - :type data: lxml.etree._Element - :return: None - """ - raise NotImplementedError - - -class PullSource(object): - def GetExtra(self, client): - return [] - - def GetCurrentEntry(self, client, e_type, e_name): - raise NotImplementedError - - -class PullTarget(object): - def AcceptChoices(self, entry, metadata): - raise NotImplementedError - - def AcceptPullData(self, specific, new_entry, verbose): - raise NotImplementedError - - -class Decision(object): - """ Decision plugins produce decision lists for affecting which - entries are actually installed on clients. """ - - def GetDecisions(self, metadata, mode): - """ Return a list of tuples of ``(<entry type>, <entry - name>)`` to be used as the decision list for the given - client in the specified mode. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param mode: The decision mode ("whitelist" or "blacklist") - :type mode: string - :return: list of tuples - """ - raise NotImplementedError - - -class ValidationError(Exception): - """ Exception raised by - :class:`Bcfg2.Server.Plugin.StructureValidator` and - :class:`Bcfg2.Server.Plugin.GoalValidator` objects """ - - -class StructureValidator(object): - """ StructureValidator plugins can modify the list of structures - after it has been created but before the entries have been - concretely bound. """ - - def validate_structures(self, metadata, structures): - """ Given a list of structures (i.e., of tags that contain - entry tags), modify that list or the structures in it - in-place. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param config: A list of lxml.etree._Element objects - describing the structures for this client - :type config: list - :returns: None - :raises: Bcfg2.Server.Plugin.ValidationError - """ - raise NotImplementedError - - -class GoalValidator(object): - """ GoalValidator plugins can modify the concretely-bound configuration of - a client as a last stage before the configuration is sent to the - client. """ - - def validate_goals(self, metadata, config): - """ Given a monolithic XML document of the full configuration, - modify the document in-place. - - :param metadata: The client metadata - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param config: The full configuration for the client - :type config: lxml.etree._Element - :returns: None - :raises: Bcfg2.Server.Plugin.ValidationError - """ - raise NotImplementedError - - -class Version(object): - """ Version plugins interact with various version control systems. """ - - def get_revision(self): - """ Return the current revision of the Bcfg2 specification. - This will be included in the ``revision`` attribute of the - top-level tag of the XML configuration sent to the client. - - :returns: string - the current version - """ - raise NotImplementedError - - -class ClientRunHooks(object): - """ ClientRunHooks can hook into various parts of a client run to - perform actions at various times without needing to pretend to be - a different plugin type. """ - - def start_client_run(self, metadata): - """ Invoked at the start of a client run, after all probe data - has been received and decision lists have been queried (if - applicable), but before the configuration is generated. - - :param metadata: The client metadata object - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: None - """ - pass - - def end_client_run(self, metadata): - """ Invoked at the end of a client run, immediately after - :class:`Bcfg2.Server.Plugin.GoalValidator` plugins have been run - and just before the configuration is returned to the client. - - :param metadata: The client metadata object - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: None - """ - pass - - def end_statistics(self, metadata): - """ Invoked after statistics are processed for a client. - - :param metadata: The client metadata object - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: None - """ - pass - -# the rest of the file contains classes for coherent file caching - class FileBacked(object): """This object caches file data in memory. HandleEvent is called whenever fam registers an event. @@ -1321,12 +650,6 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): if not key.startswith('__')]) -# new unified EntrySet backend -class SpecificityError(Exception): - """Thrown in case of filename parse failure.""" - pass - - class Specificity(CmpMixin): def __init__(self, all=False, group=False, hostname=False, prio=0, delta=False): diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py new file mode 100644 index 000000000..a6543e9b9 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -0,0 +1,548 @@ +""" Interface definitions for Bcfg2 server plugins """ + +import os +import sys +import copy +import threading +import lxml.etree +import Bcfg2.Server +from Bcfg2.Compat import Queue, Empty, Full, cPickle +from exceptions import * +from base import Plugin + +class Generator(object): + """ Generator plugins contribute to literal client configurations. + That is, they generate entry contents. + + An entry is generated in one of two ways: + + #. The Bcfg2 core looks in the ``Entries`` dict attribute of the + plugin object. ``Entries`` is expected to be a dict whose keys + are entry tags (e.g., ``"Path"``, ``"Service"``, etc.) and + whose values are dicts; those dicts should map the ``name`` + attribute of an entry to a callable that will be called to + generate the content. The callable will receive two arguments: + the abstract entry (as an lxml.etree._Element object), and the + client metadata object the entry is being generated for. + + #. If the entry is not listed in ``Entries``, the Bcfg2 core calls + :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry`; if that + returns True, then it calls + :func:`Bcfg2.Server.Plugin.Generator.HandleEntry`. + """ + + def HandlesEntry(self, entry, metadata): + """ HandlesEntry is the slow path method for routing + configuration binding requests. It is called if the + ``Entries`` dict does not contain a method for binding the + entry. + + :param entry: The entry to bind + :type entry: lxml.etree._Element + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: bool - Whether or not this plugin can handle the entry + :raises: Bcfg2.Server.Plugin.PluginExecutionError + """ + return False + + def HandleEntry(self, entry, metadata): + """ HandlesEntry is the slow path method for binding + configuration binding requests. It is called if the + ``Entries`` dict does not contain a method for binding the + entry, and :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry` + returns True. + + :param entry: The entry to bind + :type entry: lxml.etree._Element + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: lxml.etree._Element - The fully bound entry + :raises: Bcfg2.Server.Plugin.PluginExecutionError + """ + return entry + + +class Structure(object): + """ Structure Plugins contribute to abstract client + configurations. That is, they produce lists of entries that will + be generated for a client. """ + + def BuildStructures(self, metadata): + """ Build a list of lxml.etree._Element objects that will be + added to the top-level ``<Configuration>`` tag of the client + configuration. Consequently, each object in the list returned + by ``BuildStructures()`` must consist of a container tag + (e.g., ``<Bundle>`` or ``<Independent>``) which contains the + entry tags. It must not return a list of entry tags. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of lxml.etree._Element objects + """ + raise NotImplementedError + + +class Metadata(object): + """Signal metadata capabilities for this plugin""" + def viz(self, hosts, bundles, key, only_client, colors): + """ Return a string containing a graphviz document that maps + out the Metadata for :ref:`bcfg2-admin viz <server-admin-viz>` + + :param hosts: Include hosts in the graph + :type hosts: bool + :param bundles: Include bundles in the graph + :type bundles: bool + :param key: Include a key in the graph + :type key: bool + :param only_client: Only include data for the specified client + :type only_client: string + :param colors: Use the specified graphviz colors + :type colors: list of strings + :return: string + """ + return '' + + def set_version(self, client, version): + """ Set the version for the named client to the specified + version string. + + :param client: Hostname of the client + :type client: string + :param profile: Client Bcfg2 version + :type profile: string + :return: None + :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, + Bcfg2.Server.Plugin.MetadataConsistencyError + """ + pass + + def set_profile(self, client, profile, address): + """ Set the profile for the named client to the named profile + group. + + :param client: Hostname of the client + :type client: string + :param profile: Name of the profile group + :type profile: string + :param address: Address pair of ``(<ip address>, <hostname>)`` + :type address: tuple + :return: None + :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, + Bcfg2.Server.Plugin.MetadataConsistencyError + """ + pass + + def resolve_client(self, address, cleanup_cache=False): + """ Resolve the canonical name of this client. If this method + is not implemented, the hostname claimed by the client is + used. (This may be a security risk; it's highly recommended + that you implement ``resolve_client`` if you are writing a + Metadata plugin.) + + :param address: Address pair of ``(<ip address>, <hostname>)`` + :type address: tuple + :param cleanup_cache: Whether or not to remove expire the + entire client hostname resolution class + :type cleanup_cache: bool + :return: string - canonical client hostname + :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, + Bcfg2.Server.Plugin.MetadataConsistencyError + """ + return address[1] + + def AuthenticateConnection(self, cert, user, password, address): + """ Authenticate the given client. + + :param cert: an x509 certificate + :type cert: dict + :param user: The username of the user trying to authenticate + :type user: string + :param password: The password supplied by the client + :type password: string + :param addresspair: An address pair of ``(<ip address>, + <hostname>)`` + :type addresspair: tuple + :return: bool - True if the authenticate succeeds, False otherwise + """ + raise NotImplementedError + + def get_initial_metadata(self, client_name): + """ Return a + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` object + that fully describes everything the Metadata plugin knows + about the named client. + + :param client_name: The hostname of the client + :type client_name: string + :return: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ + raise NotImplementedError + + def merge_additional_data(self, imd, source, data): + """ Add arbitrary data from a + :class:`Bcfg2.Server.Plugin.Connector` plugin to the given + metadata object. + + :param imd: An initial metadata object + :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param source: The name of the plugin providing this data + :type source: string + :param data: The data to add + :type data: any + :return: None + """ + raise NotImplementedError + + def merge_additional_groups(self, imd, groups): + """ Add groups from a + :class:`Bcfg2.Server.Plugin.Connector` plugin to the given + metadata object. + + :param imd: An initial metadata object + :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param groups: The groups to add + :type groups: list of strings + :return: None + """ + raise NotImplementedError + + +class Connector(object): + """ Connector plugins augment client metadata instances with + additional data, additional groups, or both. """ + + def get_additional_groups(self, metadata): + """ Return a list of additional groups for the given client. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of strings + """ + return list() + + def get_additional_data(self, metadata): + """ Return arbitrary additional data for the given + ClientMetadata object. By convention this is usually a dict + object, but doesn't need to be. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of strings + """ + return dict() + + +class Probing(object): + """ Probing plugins can collect data from clients and process it. + """ + + def GetProbes(self, metadata): + """ Return a list of probes for the given client. Each probe + should be an lxml.etree._Element object that adheres to + the following specification. Each probe must the following + attributes: + + * ``name``: The unique name of the probe. + * ``source``: The origin of the probe; probably the name of + the plugin that supplies the probe. + * ``interpreter``: The command that will be run on the client + to interpret the probe script. Compiled (i.e., + non-interpreted) probes are not supported. + + The text of the XML tag should be the contents of the probe, + i.e., the code that will be run on the client. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of lxml.etree._Element objects + """ + raise NotImplementedError + + def ReceiveData(self, metadata, datalist): + """ Process data returned from the probes for the given + client. ``datalist`` is a list of lxml.etree._Element + objects, each of which is a single tag; the ``name`` attribute + holds the unique name of the probe that was run, and the text + contents of the tag hold the results of the probe. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param datalist: The probe data + :type datalist: list of lxml.etree._Element objects + :return: None + """ + raise NotImplementedError + + +class Statistics(Plugin): + """ Statistics plugins handle statistics for clients. In general, + you should avoid using Statistics and use + :class:`Bcfg2.Server.Plugin.ThreadedStatistics` instead.""" + + def process_statistics(self, client, xdata): + """ Process the given XML statistics data for the specified + client. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The statistics data + :type data: lxml.etree._Element + :return: None + """ + raise NotImplementedError + + +class ThreadedStatistics(Statistics, threading.Thread): + """ ThreadedStatistics plugins process client statistics in a + separate thread. """ + + def __init__(self, core, datastore): + Statistics.__init__(self, core, datastore) + threading.Thread.__init__(self) + # Event from the core signaling an exit + self.terminate = core.terminate + self.work_queue = Queue(100000) + self.pending_file = os.path.join(datastore, "etc", + "%s.pending" % self.name) + self.daemon = False + self.start() + + def _save(self): + """Save any pending data to a file.""" + pending_data = [] + try: + while not self.work_queue.empty(): + (metadata, data) = self.work_queue.get_nowait() + try: + pending_data.append((metadata.hostname, + lxml.etree.tostring(data, + xml_declaration=False).decode("UTF-8"))) + except: + err = sys.exc_info()[1] + self.logger.warning("Dropping interaction for %s: %s" % + (metadata.hostname, err)) + except Empty: + pass + + try: + savefile = open(self.pending_file, 'w') + cPickle.dump(pending_data, savefile) + savefile.close() + self.logger.info("Saved pending %s data" % self.name) + except: + err = sys.exc_info()[1] + self.logger.warning("Failed to save pending data: %s" % err) + + def _load(self): + """Load any pending data from a file.""" + if not os.path.exists(self.pending_file): + return True + pending_data = [] + try: + savefile = open(self.pending_file, 'r') + pending_data = cPickle.load(savefile) + savefile.close() + except Exception: + e = sys.exc_info()[1] + self.logger.warning("Failed to load pending data: %s" % e) + return False + for (pmetadata, pdata) in pending_data: + # check that shutdown wasnt called early + if self.terminate.isSet(): + return False + + try: + while True: + try: + metadata = self.core.build_metadata(pmetadata) + break + except MetadataRuntimeError: + pass + + self.terminate.wait(5) + if self.terminate.isSet(): + return False + + self.work_queue.put_nowait((metadata, + lxml.etree.XML(pdata, + parser=Bcfg2.Server.XMLParser))) + except Full: + self.logger.warning("Queue.Full: Failed to load queue data") + break + except lxml.etree.LxmlError: + lxml_error = sys.exc_info()[1] + self.logger.error("Unable to load saved interaction: %s" % + lxml_error) + except MetadataConsistencyError: + self.logger.error("Unable to load metadata for save " + "interaction: %s" % pmetadata) + try: + os.unlink(self.pending_file) + except: + self.logger.error("Failed to unlink save file: %s" % + self.pending_file) + self.logger.info("Loaded pending %s data" % self.name) + return True + + def run(self): + if not self._load(): + return + while not self.terminate.isSet() and self.work_queue != None: + try: + (client, xdata) = self.work_queue.get(block=True, timeout=2) + except Empty: + continue + except Exception: + e = sys.exc_info()[1] + self.logger.error("ThreadedStatistics: %s" % e) + continue + self.handle_statistic(client, xdata) + if self.work_queue != None and not self.work_queue.empty(): + self._save() + + def process_statistics(self, metadata, data): + try: + self.work_queue.put_nowait((metadata, copy.copy(data))) + except Full: + self.logger.warning("%s: Queue is full. Dropping interactions." % + self.name) + + def handle_statistic(self, metadata, data): + """ Process the given XML statistics data for the specified + client object. This differs from the + :func:`Bcfg2.Server.Plugin.Statistics.process_statistics` + method only in that ThreadedStatistics first adds the data to + a queue, and then processes them in a separate thread. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The statistics data + :type data: lxml.etree._Element + :return: None + """ + raise NotImplementedError + + +class PullSource(object): + def GetExtra(self, client): + return [] + + def GetCurrentEntry(self, client, e_type, e_name): + raise NotImplementedError + + +class PullTarget(object): + def AcceptChoices(self, entry, metadata): + raise NotImplementedError + + def AcceptPullData(self, specific, new_entry, verbose): + raise NotImplementedError + + +class Decision(object): + """ Decision plugins produce decision lists for affecting which + entries are actually installed on clients. """ + + def GetDecisions(self, metadata, mode): + """ Return a list of tuples of ``(<entry type>, <entry + name>)`` to be used as the decision list for the given + client in the specified mode. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param mode: The decision mode ("whitelist" or "blacklist") + :type mode: string + :return: list of tuples + """ + raise NotImplementedError + + +class StructureValidator(object): + """ StructureValidator plugins can modify the list of structures + after it has been created but before the entries have been + concretely bound. """ + + def validate_structures(self, metadata, structures): + """ Given a list of structures (i.e., of tags that contain + entry tags), modify that list or the structures in it + in-place. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param config: A list of lxml.etree._Element objects + describing the structures for this client + :type config: list + :returns: None + :raises: Bcfg2.Server.Plugin.ValidationError + """ + raise NotImplementedError + + +class GoalValidator(object): + """ GoalValidator plugins can modify the concretely-bound configuration of + a client as a last stage before the configuration is sent to the + client. """ + + def validate_goals(self, metadata, config): + """ Given a monolithic XML document of the full configuration, + modify the document in-place. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param config: The full configuration for the client + :type config: lxml.etree._Element + :returns: None + :raises: Bcfg2.Server.Plugin.ValidationError + """ + raise NotImplementedError + + +class Version(object): + """ Version plugins interact with various version control systems. """ + + def get_revision(self): + """ Return the current revision of the Bcfg2 specification. + This will be included in the ``revision`` attribute of the + top-level tag of the XML configuration sent to the client. + + :returns: string - the current version + """ + raise NotImplementedError + + +class ClientRunHooks(object): + """ ClientRunHooks can hook into various parts of a client run to + perform actions at various times without needing to pretend to be + a different plugin type. """ + + def start_client_run(self, metadata): + """ Invoked at the start of a client run, after all probe data + has been received and decision lists have been queried (if + applicable), but before the configuration is generated. + + :param metadata: The client metadata object + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ + pass + + def end_client_run(self, metadata): + """ Invoked at the end of a client run, immediately after + :class:`Bcfg2.Server.Plugin.GoalValidator` plugins have been run + and just before the configuration is returned to the client. + + :param metadata: The client metadata object + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ + pass + + def end_statistics(self, metadata): + """ Invoked after statistics are processed for a client. + + :param metadata: The client metadata object + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ + pass |