diff options
-rw-r--r-- | doc/conf.py | 2 | ||||
-rw-r--r-- | doc/development/plugins.txt | 363 | ||||
-rw-r--r-- | doc/server/plugins/structures/deps.txt | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin.py | 441 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/ServiceCompat.py | 31 | ||||
-rw-r--r-- | testsuite/Testsrc/Testlib/TestServer/TestPlugin.py | 53 | ||||
-rw-r--r-- | testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py | 9 |
7 files changed, 575 insertions, 326 deletions
diff --git a/doc/conf.py b/doc/conf.py index 97a094a59..4497b78d7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -211,3 +211,5 @@ else: # If false, no module index is generated. latex_use_modindex = False + +autodoc_default_flags = ['members', 'show-inheritance'] diff --git a/doc/development/plugins.txt b/doc/development/plugins.txt index 96469602d..2609595a7 100644 --- a/doc/development/plugins.txt +++ b/doc/development/plugins.txt @@ -28,296 +28,195 @@ Server Plugin Types A plugin must implement at least one of the interfaces described below. Each interface is available as a class in -``Bcfg2.Server.Plugin``. In most cases, a plugin must also inherit -from ``Bcfg2.Server.Plugin.Plugin``, which is the base Plugin object -(described below). Some of the interfaces listed below are themselves -Plugin objects, so your custom plugin would only need to inherit from -the plugin type. +:mod:``Bcfg2.Server.Plugin``. In most cases, a plugin must also +inherit from :class:`Bcfg2.Server.Plugin.Plugin`, which is the base +Plugin object (described below). Some of the interfaces listed below +are themselves Plugin objects, so your custom plugin would only need +to inherit from the plugin type. + +Plugin +^^^^^^ + +.. autoclass:: Bcfg2.Server.Plugin.Plugin + :inherited-members: + :show-inheritance: + + .. automethod:: Bcfg2.Server.Plugin.Plugin.__init__ + +With the exceptions of :class:`Bcfg2.Server.Plugin.Statistics` and +:class:`Bcfg2.Server.Plugin.ThreadedStatistics`, the plugin interfaces +listed below do **not** inherit from Plugin; they simply provide +interfaces that a given plugin may or must implement. Generator ^^^^^^^^^ -Generator plugins contribute to literal client configurations. That -is, they generate entry contents. Examples are -:ref:`server-plugins-generators-cfg` and -:ref:`server-plugins-generators-sshbase`. +.. autoclass:: Bcfg2.Server.Plugin.Generator -An entry is generated in one of two ways: - -#. First, the Bcfg2 core looks in the ``Entries`` dict attribute of - the plugin object. ``Entries`` is expected to be a dict whose keys - are entry tags (e.g., ``"Path"``, ``"Service"``, etc.) and whose - values are dicts; those dicts should map the ``name`` attribute of an - entry to a callable that will be called to generate the content. The - callable will receive two arguments: the abstract entry (as an - lxml.etree._Element object), and the client metadata object the entry - is being generated for. -#. Second, if the entry is not listed in ``Entries``, the Bcfg2 core - calls ``HandlesEntry(entry, metadata)``; if that returns True, then - it calls ``HandleEntry(entry, metadata)``. - -The Generator plugin should provide one or both methods to bind -entries, but does not have to provide both. - -Both ``HandleEntry()`` and the callable objects in the ``Entries`` -dict should return an lxml.etree._Element object representing the -fully-bound entry. They should raise -``Bcfg2.Server.Plugin.PluginExecutionError`` with a sensible error -message on failure. +Examples are :ref:`server-plugins-generators-cfg` and +:ref:`server-plugins-generators-sshbase`. Structure ^^^^^^^^^ -Structure Plugins contribute to abstract client configurations. That -is, they produce lists of entries that will be generated for a client. -:ref:`server-plugins-structure-bundler-index` is a Structure plugin. +.. autoclass:: Bcfg2.Server.Plugin.Structure -Structure plugins must implement one method: ``BuildStructures()``, -which is called with a single argument, a client metadata object -structures should be built for. It should return a list of -lxml.etree._Element objects that will be added to the top-level -``<Configuration>`` tag of the client configuration. Consequently, -each object in the list must consist of a container tag (e.g., -``<Bundle>`` or ``<Independent>``) which contains the entry tags. It -must not return a list of entry tags. +:ref:`server-plugins-structures-bundler-index` is a Structure plugin. Metadata ^^^^^^^^ -Metadata plugins provide client metadata. -:ref:`server-plugins-grouping-metadata` is a Metadata plugin. +.. autoclass:: Bcfg2.Server.Plugin.Metadata -Metadata plugins **must** implement the following methods: - -* ``AuthenticateConnection(cert, user, password, addresspair)``: - Authenticate the given client. Arguments are: - * ``cert``: an x509 certificate dict; - * ``user``: The username of the user trying to authenticate; - * ``password``: The password supplied by the client; - * ``addresspair``: A tuple of ``(<ip address>, <hostname>)`` - ``AuthenticateConnection()`` should return True if the authenticate - succeeds, False otherwise. Failures should be logged at the error - level. -* ``get_initial_metadata(client_name)``: Return a ClientMetadata - object that fully describes everything the Metadata plugin knows - about the named client. See :file:``Metadata.py`` for a reference - implementation of the ClientMetadata object. -* ``merge_additional_data(metadata, source, data)``: Add data from the - Connector plugin named by ``<source>`` (a string giving the name of - the Connector plugin) to the given metadata object. - ``merge_additional_data()`` should modify the ``metadata`` object in - place; it doesn't need to return anything. -* ``merge_additional_groups(metadata, groups)``: Add groups from an - anonymous Connector plugin to the given metadata object. - ``merge_additional_groups()`` should modify the ``metadata`` object in - place; it doesn't need to return anything. - -Metadata plugins **may** implement the following methods: - -* ``viz(hosts, bundles, key, only_client, colors)``: Return a string - containing a graphviz document that maps out the Metadata. The - first three options are boolean, and describe whether or not the - named item(s) should be included in the graphviz document. - ``only_client`` is the name of a client which, if included, will be - the only client whose metadata will be included on the map. - ``colors`` is a list of graphviz color names to use. If - unimplemented, the empty string will be returned. -* ``set_version(client, version)``: Set the version for the named - client to the specified version string. If unimplemented, setting - the version of a client will fail silently. -* ``set_profile(client, profile, addresspair)``: Set the profile for - the named client to the named profile group. If unimplemented, - setting a client profile will fail silently. -* ``resolve_client(addresspair, cleanup_cache=False)``: Given a tuple - of ``(<ip address>, <hostname>)``, resolve the canonical name of - this client. If this method is not implemented, the hostname - claimed by the client is used. (This may be a security risk; it's - highly recommended that you implement ``resolve_client`` if you are - writing a Metadata plugin.) +:ref:`server-plugins-grouping-metadata` is a Metadata plugin. Connector ^^^^^^^^^ -Connector plugins augment client metadata instances with additional -data, additional groups, or both. Connector plugins include +.. autoclass:: Bcfg2.Server.Plugin.Connector + +Connector plugins include :ref:`server-plugins-grouping-grouppatterns`, :ref:`server-plugins-connectors-properties`, and :ref:`server-plugins-probes-index`. -Connector plugins should implement one or all of the following -methods: - -* ``get_additional_groups(metadata)``: Return a list of additional - groups for the given ClientMetadata object. If unimplemented, the - empty list is returned. -* ``get_additional_data(metadata)``: Return arbitrary additional data - for the given ClientMetadata object. By convention this is usually - a dict object, but doesn't need to be. If unimplemented, the empty - dict is returned. - Probing ^^^^^^^ -Probing plugins can collect data from clients and process it. +.. autoclass:: Bcfg2.Server.Plugin.Probing + Examples include :ref:`server-plugins-probes-index` and :ref:`server-plugins-probes-fileprobes`. -Probing plugins must implement the following methods: +Statistics +^^^^^^^^^^ -* ``GetProbes(metadata)``: Return a list of probes for the given - ClientMetadata object. Each probe should be an XML document, - described below. -* ``ReceiveData(metadata, datalist)``: Process data returned from the - probes for the given ClientMetadata object. ``datalist`` is a list - of lxml.etree._Element objects, each of which is a single tag; the - ``name`` attribute holds the unique name of the probe that was run, - and the text contents of the tag hold the results of the probe. +.. autoclass:: Bcfg2.Server.Plugin.Statistics -``GetProbes()`` returns a list of probes, each of which is an -``lxml.etree._Element`` object that adheres to the following -specification. Each probe must the following attributes: +The Statistics object is itself a :class:`Bcfg2.Server.Plugin.Plugin` +object, so objects that inherit from Statistics do not have to also +inherit from Plugin. -* ``name``: The unique name of the probe. -* ``source``: The origin of the probe; probably the name of the plugin - that supplies the probe. -* ``interpreter``: The command that will be run on the client to - interpret the probe script. Compiled (i.e., non-interpreted) probes - are not supported. +ThreadedStatistics +^^^^^^^^^^^^^^^^^^ -The text of the XML tag should be the contents of the probe, i.e., the -code that will be run on the client. +.. autoclass:: Bcfg2.Server.Plugin.ThreadedStatistics -Statistics +:ref:`server-plugins-statistics-dbstats` is an example of a +ThreadedStatistics plugin. + +The ThreadedStatistics object is itself a +:class:`Bcfg2.Server.Plugin.Plugin` object, so objects that inherit +from ThreadedStatistics do not have to also inherit from Plugin. + +PullSource +^^^^^^^^^^ + +.. autoclass:: Bcfg2.Server.Plugin.PullSource + +:ref:`server-plugins-statistics-dbstats` is an example of a plugin +that implements the PullSource interface + +PullTarget ^^^^^^^^^^ -Signal statistics handling capability +.. autoclass:: Bcfg2.Server.Plugin.PullTarget + +:ref:`server-plugins-generators-sshbase` is an example of a plugin +that implements the PullTarget interface Decision ^^^^^^^^ -Signal decision handling capability +.. autoclass:: Bcfg2.Server.Plugin.Decision -Version -^^^^^^^ +:ref:`server-plugins-generators-decisions` is an example of a Decision +plugin, and has much more information about how decisions are used. -Interact with various version control systems - -Writing Bcfg2 Server Plugins ----------------------------- - -Bcfg2 plugins are python classes that subclass from -Bcfg2.Server.Plugin.Plugin. Several plugin-specific values must be set -in the new plugin. These values dictate how the new plugin will behave -with respect to the above four functions. The following table describes -all important member fields. - -+-----------------+-----------------------------------+--------------------------+ -| Name | Description | Format | -+=================+===================================+==========================+ -| __name__ | The name of the plugin | string | -+-----------------+-----------------------------------+--------------------------+ -| __version__ | The plugin version (generally | string | -| | tied to revctl keyword expansion) | | -+-----------------+-----------------------------------+--------------------------+ -| __author__ | The plugin author | string | -+-----------------+-----------------------------------+--------------------------+ -| __rmi__ | Set of functions to be exposed as | List of function names | -| | XML-RPC functions | (strings) | -+-----------------+-----------------------------------+--------------------------+ -| Entries | Multidimentional dictionary of | Dictionary of | -| | keys that point to the function | ConfigurationEntityType, | -| | used to bind literal contents for | Name keys, and function | -| | a given configuration entity | reference values | -+-----------------+-----------------------------------+--------------------------+ -| BuildStructures | Function that returns a list of | Member function | -| | the structures for a given client | | -+-----------------+-----------------------------------+--------------------------+ -| GetProbes | Function that returns a list of | Member function | -| | probes that a given client should | | -| | execute | | -+-----------------+-----------------------------------+--------------------------+ -| ReceiveData | Function that accepts the probe | Member function | -| | results for a given client | | -+-----------------+-----------------------------------+--------------------------+ - -Example Plugin -^^^^^^^^^^^^^^ +StructureValidator +^^^^^^^^^^^^^^^^^^ -.. code-block:: python +.. autoclass:: Bcfg2.Server.Plugin.StructureValidator - import Bcfg2.Server.Plugin - class MyPlugin(Bcfg2.Server.Plugin.Plugin): - '''An example plugin''' - # All plugins need to subclass Bcfg2.Server.Plugin.Plugin - __name__ = 'MyPlugin' - __version__ = '1' - __author__ = 'me@me.com' - __rmi__ = ['myfunction'] - # myfunction is now available remotely as MyPlugin.myfunction - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.Entries = {'Path':{'/etc/foo.conf': self.buildFoo}} - - def myfunction(self): - '''function for xmlrpc rmi call''' - #do something - return True - - def buildFoo(self, entry, metadata): - '''Bind per-client information into entry based on metadata''' - entry.attrib.update({'type':'file', 'owner':'root', 'group':'root', 'perms':'644'}) - entry.text = '''contents of foo.conf''' - -Example Connector -^^^^^^^^^^^^^^^^^ +Examples are :ref:`server-plugins-structures-defaults` and +:ref:`server-plugins-structures-deps`. -.. code-block:: python +GoalValidator +^^^^^^^^^^^^^ - import Bcfg2.Server.Plugin +.. autoclass:: Bcfg2.Server.Plugin.GoalValidator - class Foo(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - '''The Foo plugin is here to illustrate a barebones connector''' - name = 'Foo' - experimental = True +An example of a GoalValidator plugin would be the ServiceCompat plugin +that is used to provide old-style Service tag attributes to older +clients from a Bcfg2 1.3.0 server. As a final stage of configuration +generation, it translates the new "restart" and "install" attributes +into the older "mode" attribute. - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.store = XMLFileBacked(self.data, core.fam) +Version +^^^^^^^ + +.. autoclass:: Bcfg2.Server.Plugin.Version - def get_additional_data(self, metadata): +Examples include :ref:`server-plugins-version-git` and +:ref:`server-plugins-version-svn2`. + +ClientRunHooks +^^^^^^^^^^^^^^ - mydata = {} - for data in self.store.entries['foo.xml'].xdata.get("foo", []): +.. autoclass:: Bcfg2.Server.Plugin.ClientRunHooks - mydata[data] = "bar" +Examples are :ref:`server-plugins-misc-trigger` and +:ref:`server-plugins-connectors-puppetenc`. - return dict([('mydata', mydata)]) +Exposing XML-RPC Functions +-------------------------- + +Plugins can expose XML-RPC functions that can then be called with +:ref:`bcfg2-admin xcmd <server-admin-xcmd>`. Note that there is +absolutely no access control beyond the initial authentication, so +take care to not expose any data or behavior via XML-RPC that you +would not want all of your clients to be able to see or use. + +To expose a function, simply add its name to the ``__rmi__`` class +attribute. (RMI stands for "Remote Method Invocation.") Consider +this example from the :ref:`server-plugins-generators-packages` +plugin: + +.. code-block:: python - def get_additional_groups(self, meta): - return self.cgroups.get(meta.hostname, list()) + class Packages(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.StructureValidator, + Bcfg2.Server.Plugin.Generator, + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.ClientRunHooks): + name = 'Packages' + conflicts = ['Pkgmgr'] + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] -Example Metadata plugin -^^^^^^^^^^^^^^^^^^^^^^^ + def Refresh(self): + self._load_config(force_update=True) + return True -If you would like to define your own Metadata plugin (to extend/change -functionality of the existing Metadata plugin), here are the steps to -do so. We will call our new plugin `MyMetadata`. + def Reload(self): + self._load_config() + return True -#. Add MyMetadata.py +This exposes two functions, ``Refresh`` and ``Reload``, in addition to +any default methods that are already exposed. To call one of these +functions, you could run:: - .. code-block:: python + bcfg2-admin xcmd Packages.Refresh - import Bcfg2.Server.Plugins.Metadata +Plugin Helper Classes +--------------------- - class MyMetadata(Bcfg2.Server.Plugins.Metadata.Metadata): - '''This class contains data for bcfg2 server metadata''' - __author__ = 'bcfg-dev@mcs.anl.gov' +.. autoclass:: Bcfg2.Server.Plugin.Debuggable - def __init__(self, core, datastore, watch_clients=True): - Bcfg2.Server.Plugins.Metadata.Metadata.__init__(self, core, datastore, watch_clients) +Plugin Exceptions +----------------- -#. Add MyMetadata to ``src/lib/Server/Plugins/__init__.py`` -#. Replace Metadata with MyMetadata in the plugins line of bcfg2.conf +.. autoexception:: Bcfg2.Server.Plugin.ValidationError +.. autoexception:: Bcfg2.Server.Plugin.PluginInitError +.. autoexception:: Bcfg2.Server.Plugin.PluginExecutionError +.. autoexception:: Bcfg2.Server.Plugin.MetadataConsistencyError +.. autoexception:: Bcfg2.Server.Plugin.MetadataRuntimeError diff --git a/doc/server/plugins/structures/deps.txt b/doc/server/plugins/structures/deps.txt index 7c5861d06..3c5177f4d 100644 --- a/doc/server/plugins/structures/deps.txt +++ b/doc/server/plugins/structures/deps.txt @@ -1,6 +1,6 @@ .. -*- mode: rst -*- -.. _server-plugins-generators-deps: +.. _server-plugins-structures-deps: ==== Deps diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index 80537c200..0b2f7cee0 100644 --- a/src/lib/Bcfg2/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -58,12 +58,14 @@ def bind_info(entry, metadata, infoxml=None, default=default_file_metadata): class PluginInitError(Exception): - """Error raised in cases of Plugin initialization errors.""" + """Error raised in cases of :class:`Bcfg2.Server.Plugin.Plugin` + initialization errors.""" pass class PluginExecutionError(Exception): - """Error raised in case of Plugin execution errors.""" + """Error raised in case of :class:`Bcfg2.Server.Plugin.Plugin` + execution errors.""" pass @@ -73,13 +75,17 @@ class MetadataConsistencyError(Exception): class MetadataRuntimeError(Exception): - """This error is raised when the metadata engine - is called prior to reading enough data. - """ + """This error is raised when the metadata engine is called prior + to reading enough data, or for other + :class:`Bcfg2.Server.Plugin.Metadata` errors. """ pass class Debuggable(object): + """ Mixin to add a debugging interface to an object and expose it + via XML-RPC on :class:`Bcfg2.Server.Plugin.Plugin` objects """ + + #: List of names of methods to be exposed as XML-RPC functions __rmi__ = ['toggle_debug'] def __init__(self, name=None): @@ -90,6 +96,10 @@ class Debuggable(object): self.logger = logging.getLogger(name) def toggle_debug(self): + """ Turn debugging output on or off. + + :returns: bool - The new value of the debug flag + """ self.debug_flag = not self.debug_flag self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, self.debug_flag), @@ -97,37 +107,54 @@ class Debuggable(object): return self.debug_flag def debug_log(self, message, flag=None): + """ Log a message at the debug level. + + :param message: The message to log + :type message: string + :param flag: Override the current debug flag with this value + :type flag: bool + :returns: None + """ if (flag is None and self.debug_flag) or flag: self.logger.error(message) class Plugin(Debuggable): - """This is the base class for all Bcfg2 Server plugins. - Several attributes must be defined in the subclass: - name : the name of the plugin - __author__ : the author/contact for the plugin - - Plugins can provide three basic types of functionality: - - Structure creation (overloading BuildStructures) - - Configuration entry binding (overloading HandlesEntry, or loads the Entries table) - - Data collection (overloading GetProbes/ReceiveData) - """ + """ The base class for all Bcfg2 Server plugins. """ + + #: The name of the plugin. name = 'Plugin' + + #: The email address of the plugin author. __author__ = 'bcfg-dev@mcs.anl.gov' + + #: Plugin is experimental. Use of this plugin will produce a log + #: message alerting the administrator that an experimental plugin + #: is in use. experimental = False + + #: Plugin is deprecated and will be removed in a future release. + #: Use of this plugin will produce a log message alerting the + #: administrator that an experimental plugin is in use. deprecated = False + + #: Plugin conflicts with the list of other plugin names conflicts = [] - # Default sort_order to 500. Plugins of the same type are - # processed in order of ascending sort_order value. Plugins with - # the same sort_order are sorted alphabetically by their name. + #: Plugins of the same type are processed in order of ascending + #: sort_order value. Plugins with the same sort_order are sorted + #: alphabetically by their name. sort_order = 500 def __init__(self, core, datastore): - """Initialize the plugin. - - :param core: the Bcfg2.Server.Core initializing the plugin - :param datastore: the filesystem path of Bcfg2's repository + """ Initialize the plugin. + + :param core: The Bcfg2.Server.Core initializing the plugin + :type core: Bcfg2.Server.Core + :param datastore: The path to the Bcfg2 repository on the + filesystem + :type datastore: string + :raises: Bcfg2.Server.Plugin.PluginInitError """ object.__init__(self) self.Entries = {} @@ -138,9 +165,19 @@ class Plugin(Debuggable): @classmethod def init_repo(cls, repo): + """ Perform any tasks necessary to create an initial Bcfg2 + repository. + + :param repo: The path to the Bcfg2 repository on the filesystem + :type repo: string + :returns: None + """ os.makedirs(os.path.join(repo, cls.name)) def shutdown(self): + """ Perform shutdown tasks for the plugin + + :returns: None """ self.running = False def __str__(self): @@ -168,84 +205,292 @@ class PluginDatabaseModel(object): class Generator(object): - """Generator plugins contribute to literal client - configurations.""" + """ Generator plugins contribute to literal client configurations. + That is, they generate entry contents. + + An entry is generated in one of two ways: + + #. The Bcfg2 core looks in the ``Entries`` dict attribute of the + plugin object. ``Entries`` is expected to be a dict whose keys + are entry tags (e.g., ``"Path"``, ``"Service"``, etc.) and + whose values are dicts; those dicts should map the ``name`` + attribute of an entry to a callable that will be called to + generate the content. The callable will receive two arguments: + the abstract entry (as an lxml.etree._Element object), and the + client metadata object the entry is being generated for. + + #. If the entry is not listed in ``Entries``, the Bcfg2 core calls + :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry`; if that + returns True, then it calls + :func:`Bcfg2.Server.Plugin.Generator.HandleEntry`. + """ def HandlesEntry(self, entry, metadata): - """This is the slow path method for routing configuration - binding requests.""" + """ HandlesEntry is the slow path method for routing + configuration binding requests. It is called if the + ``Entries`` dict does not contain a method for binding the + entry. + + :param entry: The entry to bind + :type entry: lxml.etree._Element + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: bool - Whether or not this plugin can handle the entry + :raises: Bcfg2.Server.Plugin.PluginExecutionError + """ return False def HandleEntry(self, entry, metadata): - """This is the slow-path handler for configuration entry binding.""" + """ HandlesEntry is the slow path method for binding + configuration binding requests. It is called if the + ``Entries`` dict does not contain a method for binding the + entry, and :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry` + returns True. + + :param entry: The entry to bind + :type entry: lxml.etree._Element + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: lxml.etree._Element - The fully bound entry + :raises: Bcfg2.Server.Plugin.PluginExecutionError + """ return entry class Structure(object): - """Structure Plugins contribute to abstract client configurations.""" + """ Structure Plugins contribute to abstract client + configurations. That is, they produce lists of entries that will + be generated for a client. """ + def BuildStructures(self, metadata): - """Return a list of abstract goal structures for client.""" + """ Build a list of lxml.etree._Element objects that will be + added to the top-level ``<Configuration>`` tag of the client + configuration. Consequently, each object in the list returned + by ``BuildStructures()`` must consist of a container tag + (e.g., ``<Bundle>`` or ``<Independent>``) which contains the + entry tags. It must not return a list of entry tags. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of lxml.etree._Element objects + """ raise NotImplementedError class Metadata(object): """Signal metadata capabilities for this plugin""" def viz(self, hosts, bundles, key, only_client, colors): - """Create viz str for viz admin mode.""" + """ Return a string containing a graphviz document that maps + out the Metadata for :ref:`bcfg2-admin viz <server-admin-viz>` + + :param hosts: Include hosts in the graph + :type hosts: bool + :param bundles: Include bundles in the graph + :type bundles: bool + :param key: Include a key in the graph + :type key: bool + :param only_client: Only include data for the specified client + :type only_client: string + :param colors: Use the specified graphviz colors + :type colors: list of strings + :return: string + """ return '' def set_version(self, client, version): + """ Set the version for the named client to the specified + version string. + + :param client: Hostname of the client + :type client: string + :param profile: Client Bcfg2 version + :type profile: string + :return: None + :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, + Bcfg2.Server.Plugin.MetadataConsistencyError + """ pass def set_profile(self, client, profile, address): + """ Set the profile for the named client to the named profile + group. + + :param client: Hostname of the client + :type client: string + :param profile: Name of the profile group + :type profile: string + :param address: Address pair of ``(<ip address>, <hostname>)`` + :type address: tuple + :return: None + :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, + Bcfg2.Server.Plugin.MetadataConsistencyError + """ pass def resolve_client(self, address, cleanup_cache=False): + """ Resolve the canonical name of this client. If this method + is not implemented, the hostname claimed by the client is + used. (This may be a security risk; it's highly recommended + that you implement ``resolve_client`` if you are writing a + Metadata plugin.) + + :param address: Address pair of ``(<ip address>, <hostname>)`` + :type address: tuple + :param cleanup_cache: Whether or not to remove expire the + entire client hostname resolution class + :type cleanup_cache: bool + :return: string - canonical client hostname + :raises: Bcfg2.Server.Plugin.MetadataRuntimeError, + Bcfg2.Server.Plugin.MetadataConsistencyError + """ return address[1] def AuthenticateConnection(self, cert, user, password, address): + """ Authenticate the given client. + + :param cert: an x509 certificate + :type cert: dict + :param user: The username of the user trying to authenticate + :type user: string + :param password: The password supplied by the client + :type password: string + :param addresspair: An address pair of ``(<ip address>, + <hostname>)`` + :type addresspair: tuple + :return: bool - True if the authenticate succeeds, False otherwise + """ raise NotImplementedError def get_initial_metadata(self, client_name): + """ Return a + :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` object + that fully describes everything the Metadata plugin knows + about the named client. + + :param client_name: The hostname of the client + :type client_name: string + :return: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ raise NotImplementedError def merge_additional_data(self, imd, source, data): + """ Add arbitrary data from a + :class:`Bcfg2.Server.Plugin.Connector` plugin to the given + metadata object. + + :param imd: An initial metadata object + :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param source: The name of the plugin providing this data + :type source: string + :param data: The data to add + :type data: any + :return: None + """ raise NotImplementedError def merge_additional_groups(self, imd, groups): + """ Add groups from a + :class:`Bcfg2.Server.Plugin.Connector` plugin to the given + metadata object. + + :param imd: An initial metadata object + :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param groups: The groups to add + :type groups: list of strings + :return: None + """ raise NotImplementedError class Connector(object): - """Connector Plugins augment client metadata instances.""" + """ Connector plugins augment client metadata instances with + additional data, additional groups, or both. """ + def get_additional_groups(self, metadata): - """Determine additional groups for metadata.""" + """ Return a list of additional groups for the given client. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of strings + """ return list() def get_additional_data(self, metadata): - """Determine additional data for metadata instances.""" + """ Return arbitrary additional data for the given + ClientMetadata object. By convention this is usually a dict + object, but doesn't need to be. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of strings + """ return dict() class Probing(object): - """Signal probe capability for this plugin.""" + """ Probing plugins can collect data from clients and process it. + """ + def GetProbes(self, metadata): - """Return a set of probes for execution on client.""" + """ Return a list of probes for the given client. Each probe + should be an lxml.etree._Element object that adheres to + the following specification. Each probe must the following + attributes: + + * ``name``: The unique name of the probe. + * ``source``: The origin of the probe; probably the name of + the plugin that supplies the probe. + * ``interpreter``: The command that will be run on the client + to interpret the probe script. Compiled (i.e., + non-interpreted) probes are not supported. + + The text of the XML tag should be the contents of the probe, + i.e., the code that will be run on the client. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: list of lxml.etree._Element objects + """ raise NotImplementedError def ReceiveData(self, metadata, datalist): - """Receive probe results pertaining to client.""" + """ Process data returned from the probes for the given + client. ``datalist`` is a list of lxml.etree._Element + objects, each of which is a single tag; the ``name`` attribute + holds the unique name of the probe that was run, and the text + contents of the tag hold the results of the probe. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param datalist: The probe data + :type datalist: list of lxml.etree._Element objects + :return: None + """ raise NotImplementedError class Statistics(Plugin): - """Signal statistics handling capability.""" + """ Statistics plugins handle statistics for clients. In general, + you should avoid using Statistics and use + :class:`Bcfg2.Server.Plugin.ThreadedStatistics` instead.""" + def process_statistics(self, client, xdata): - pass + """ Process the given XML statistics data for the specified + client. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The statistics data + :type data: lxml.etree._Element + :return: None + """ + raise NotImplementedError class ThreadedStatistics(Statistics, threading.Thread): - """Threaded statistics handling capability.""" + """ ThreadedStatistics plugins process client statistics in a + separate thread. """ + def __init__(self, core, datastore): Statistics.__init__(self, core, datastore) threading.Thread.__init__(self) @@ -257,7 +502,7 @@ class ThreadedStatistics(Statistics, threading.Thread): self.daemon = False self.start() - def save(self): + def _save(self): """Save any pending data to a file.""" pending_data = [] try: @@ -283,7 +528,7 @@ class ThreadedStatistics(Statistics, threading.Thread): err = sys.exc_info()[1] self.logger.warning("Failed to save pending data: %s" % err) - def load(self): + def _load(self): """Load any pending data from a file.""" if not os.path.exists(self.pending_file): return True @@ -335,7 +580,7 @@ class ThreadedStatistics(Statistics, threading.Thread): return True def run(self): - if not self.load(): + if not self._load(): return while not self.terminate.isSet() and self.work_queue != None: try: @@ -348,7 +593,7 @@ class ThreadedStatistics(Statistics, threading.Thread): continue self.handle_statistic(client, xdata) if self.work_queue != None and not self.work_queue.empty(): - self.save() + self._save() def process_statistics(self, metadata, data): try: @@ -358,8 +603,19 @@ class ThreadedStatistics(Statistics, threading.Thread): self.name) def handle_statistic(self, metadata, data): - """Handle stats here.""" - pass + """ Process the given XML statistics data for the specified + client object. This differs from the + :func:`Bcfg2.Server.Plugin.Statistics.process_statistics` + method only in that ThreadedStatistics first adds the data to + a queue, and then processes them in a separate thread. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param data: The statistics data + :type data: lxml.etree._Element + :return: None + """ + raise NotImplementedError class PullSource(object): @@ -375,51 +631,120 @@ class PullTarget(object): raise NotImplementedError def AcceptPullData(self, specific, new_entry, verbose): - """This is the null per-plugin implementation - of bcfg2-admin pull.""" raise NotImplementedError class Decision(object): - """Signal decision handling capability.""" + """ Decision plugins produce decision lists for affecting which + entries are actually installed on clients. """ + def GetDecisions(self, metadata, mode): - return [] + """ Return a list of tuples of ``(<entry type>, <entry + name>)`` to be used as the decision list for the given + client in the specified mode. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param mode: The decision mode ("whitelist" or "blacklist") + :type mode: string + :return: list of tuples + """ + raise NotImplementedError class ValidationError(Exception): - pass + """ Exception raised by + :class:`Bcfg2.Server.Plugin.StructureValidator` and + :class:`Bcfg2.Server.Plugin.GoalValidator` objects """ class StructureValidator(object): - """Validate/modify goal structures.""" + """ StructureValidator plugins can modify the list of structures + after it has been created but before the entries have been + concretely bound. """ + def validate_structures(self, metadata, structures): + """ Given a list of structures (i.e., of tags that contain + entry tags), modify that list or the structures in it + in-place. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param config: A list of lxml.etree._Element objects + describing the structures for this client + :type config: list + :returns: None + :raises: Bcfg2.Server.Plugin.ValidationError + """ raise NotImplementedError class GoalValidator(object): - """Validate/modify configuration goals.""" - def validate_goals(self, metadata, goals): + """ GoalValidator plugins can modify the concretely-bound configuration of + a client as a last stage before the configuration is sent to the + client. """ + + def validate_goals(self, metadata, config): + """ Given a monolithic XML document of the full configuration, + modify the document in-place. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param config: The full configuration for the client + :type config: lxml.etree._Element + :returns: None + :raises: Bcfg2.Server.Plugin.ValidationError + """ raise NotImplementedError class Version(object): - """Interact with various version control systems.""" + """ Version plugins interact with various version control systems. """ + def get_revision(self): - return [] + """ Return the current revision of the Bcfg2 specification. + This will be included in the ``revision`` attribute of the + top-level tag of the XML configuration sent to the client. - def commit_data(self, file_list, comment=None): - pass + :returns: string - the current version + """ + raise NotImplementedError class ClientRunHooks(object): - """ Provides hooks to interact with client runs """ + """ ClientRunHooks can hook into various parts of a client run to + perform actions at various times without needing to pretend to be + a different plugin type. """ + def start_client_run(self, metadata): + """ Invoked at the start of a client run, after all probe data + has been received and decision lists have been queried (if + applicable), but before the configuration is generated. + + :param metadata: The client metadata object + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ pass def end_client_run(self, metadata): + """ Invoked at the end of a client run, immediately after + :class:`Bcfg2.Server.Plugin.GoalValidator` plugins have been run + and just before the configuration is returned to the client. + + :param metadata: The client metadata object + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ pass def end_statistics(self, metadata): + """ Invoked after statistics are processed for a client. + + :param metadata: The client metadata object + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: None + """ pass # the rest of the file contains classes for coherent file caching diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index aad92b7c7..f1309412a 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -1,7 +1,7 @@ import Bcfg2.Server.Plugin class ServiceCompat(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.StructureValidator): + Bcfg2.Server.Plugin.GoalValidator): """ Use old-style service modes for older clients """ name = 'ServiceCompat' __author__ = 'bcfg-dev@mcs.anl.gov' @@ -9,24 +9,23 @@ class ServiceCompat(Bcfg2.Server.Plugin.Plugin, ('interactive', 'true'): 'interactive_only', ('false', 'false'): 'manual'} - def validate_structures(self, metadata, structures): + def validate_goals(self, metadata, config): """ Apply defaults """ if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): # do not care about a client that is _any_ 1.3.0 release # (including prereleases and RCs) return - for struct in structures: - for entry in struct.xpath("//BoundService|//Service"): - mode_key = (entry.get("restart", "true").lower(), - entry.get("install", "true").lower()) - try: - mode = self.mode_map[mode_key] - except KeyError: - self.logger.info("Could not map restart and install " - "settings of %s:%s to an old-style " - "Service mode for %s; using 'manual'" % - (entry.tag, entry.get("name"), - metadata.hostname)) - mode = "manual" - entry.set("mode", mode) + for entry in config.xpath("//BoundService|//Service"): + mode_key = (entry.get("restart", "true").lower(), + entry.get("install", "true").lower()) + try: + mode = self.mode_map[mode_key] + except KeyError: + self.logger.info("Could not map restart and install settings " + "of %s:%s to an old-style Service mode for " + "%s; using 'manual'" % + (entry.tag, entry.get("name"), + metadata.hostname)) + mode = "manual" + entry.set("mode", mode) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin.py index cbeec965c..5410c550e 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin.py @@ -266,8 +266,15 @@ class TestProbing(Bcfg2TestCase): class TestStatistics(TestPlugin): test_obj = Statistics + def get_obj(self, core=None): + if core is None: + core = Mock() + return self.test_obj(core, datastore) + def test_process_statistics(self): - pass + s = self.get_obj() + self.assertRaises(NotImplementedError, + s.process_statistics, None, None) class TestThreadedStatistics(TestStatistics): @@ -294,7 +301,7 @@ class TestThreadedStatistics(TestStatistics): mock_open.side_effect = OSError # test that save does _not_ raise an exception even when # everything goes pear-shaped - ts.save() + ts._save() queue.empty.assert_any_call() mock_open.assert_called_with(ts.pending_file, 'w') @@ -310,7 +317,7 @@ class TestThreadedStatistics(TestStatistics): queue.get_nowait = Mock(side_effect=lambda: queue.data.pop()) mock_open.side_effect = None - ts.save() + ts._save() queue.empty.assert_any_call() queue.get_nowait.assert_any_call() mock_open.assert_called_with(ts.pending_file, 'w') @@ -345,13 +352,13 @@ class TestThreadedStatistics(TestStatistics): ts.work_queue.data = [] mock_exists.return_value = False - self.assertTrue(ts.load()) + self.assertTrue(ts._load()) mock_exists.assert_called_with(ts.pending_file) reset() mock_exists.return_value = True mock_open.side_effect = OSError - self.assertFalse(ts.load()) + self.assertFalse(ts._load()) mock_exists.assert_called_with(ts.pending_file) mock_open.assert_called_with(ts.pending_file, 'r') @@ -359,7 +366,7 @@ class TestThreadedStatistics(TestStatistics): mock_open.side_effect = None mock_load.return_value = self.data ts.work_queue.put_nowait.side_effect = Full - self.assertTrue(ts.load()) + self.assertTrue(ts._load()) mock_exists.assert_called_with(ts.pending_file) mock_open.assert_called_with(ts.pending_file, 'r') mock_open.return_value.close.assert_any_call() @@ -369,7 +376,7 @@ class TestThreadedStatistics(TestStatistics): core.build_metadata.side_effect = lambda x: x mock_XML.side_effect = lambda x, parser=None: x ts.work_queue.put_nowait.side_effect = None - self.assertTrue(ts.load()) + self.assertTrue(ts._load()) mock_exists.assert_called_with(ts.pending_file) mock_open.assert_called_with(ts.pending_file, 'r') mock_open.return_value.close.assert_any_call() @@ -382,8 +389,8 @@ class TestThreadedStatistics(TestStatistics): mock_unlink.assert_called_with(ts.pending_file) @patch("threading.Thread.start", Mock()) - @patch("Bcfg2.Server.Plugin.ThreadedStatistics.load") - @patch("Bcfg2.Server.Plugin.ThreadedStatistics.save") + @patch("Bcfg2.Server.Plugin.ThreadedStatistics._load") + @patch("Bcfg2.Server.Plugin.ThreadedStatistics._save") @patch("Bcfg2.Server.Plugin.ThreadedStatistics.handle_statistic") def test_run(self, mock_handle, mock_save, mock_load): core = Mock() @@ -426,8 +433,6 @@ class TestThreadedStatistics(TestStatistics): @patch("copy.copy", Mock(side_effect=lambda x: x)) @patch("Bcfg2.Server.Plugin.ThreadedStatistics.run", Mock()) def test_process_statistics(self): - TestStatistics.test_process_statistics(self) - core = Mock() ts = self.get_obj(core) ts.work_queue = Mock() @@ -438,6 +443,11 @@ class TestThreadedStatistics(TestStatistics): ts.work_queue.put_nowait.side_effect = Full # test that no exception is thrown ts.process_statistics(*self.data[0]) + + def test_handle_statistic(self): + ts = self.get_obj() + self.assertRaises(NotImplementedError, + ts.handle_statistic, None, None) class TestPullSource(Bcfg2TestCase): @@ -460,8 +470,15 @@ class TestPullTarget(Bcfg2TestCase): class TestDecision(Bcfg2TestCase): - """ placeholder for future tests """ - pass + test_obj = Decision + + def get_obj(self): + return self.test_obj() + + def test_GetDecisions(self): + d = self.get_obj() + self.assertRaises(NotImplementedError, + d.GetDecisions, None, None) class TestValidationError(Bcfg2TestCase): @@ -484,8 +501,14 @@ class TestGoalValidator(Bcfg2TestCase): class TestVersion(Bcfg2TestCase): - """ placeholder for future tests """ - pass + test_obj = Version + + def get_obj(self): + return self.test_obj() + + def test_get_revision(self): + d = self.get_obj() + self.assertRaises(NotImplementedError, d.get_revision) class TestClientRunHooks(Bcfg2TestCase): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 7673c73d2..4d5aea936 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -199,6 +199,11 @@ group-specific""" class TestProbes(TestProbing, TestConnector, TestDatabaseBacked): test_obj = Probes + def get_obj(self, core=None): + if core is None: + core = Mock() + return self.test_obj(core, datastore) + def get_test_probedata(self): test_xdata = lxml.etree.Element("test") lxml.etree.SubElement(test_xdata, "test", foo="foo") @@ -460,8 +465,6 @@ text @patch("Bcfg2.Server.Plugins.Probes.ProbeSet.get_probe_data") def test_GetProbes(self, mock_get_probe_data): - TestProbing.test_GetProbes(self) - probes = self.get_probes_object() metadata = Mock() probes.GetProbes(metadata) @@ -470,8 +473,6 @@ text @patch("Bcfg2.Server.Plugins.Probes.Probes.write_data") @patch("Bcfg2.Server.Plugins.Probes.Probes.ReceiveDataItem") def test_ReceiveData(self, mock_ReceiveDataItem, mock_write_data): - TestProbing.test_ReceiveData(self) - # we use a simple (read: bogus) datalist here to make this # easy to test datalist = ["a", "b", "c"] |