split up mammoth Plugin.py

5 files changed, 707 insertions, 683 deletions

diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py
new file mode 100644
index 000000000..487a457e6
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugin/__init__.py
@@ -0,0 +1,11 @@
+""" Bcfg2 server plugin base classes, interfaces, and helper
+objects. """
+
+import os
+import sys
+sys.path.append(os.path.dirname(__file__))
+
+from base import *
+from interfaces import *
+from helpers import *
+from exceptions import *
diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py
new file mode 100644
index 000000000..98427e726
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugin/base.py
@@ -0,0 +1,106 @@
+"""This module provides the base class for Bcfg2 server plugins."""
+
+import os
+import logging
+
+class Debuggable(object):
+    """ Mixin to add a debugging interface to an object and expose it
+    via XML-RPC on :class:`Bcfg2.Server.Plugin.Plugin` objects """
+
+    #: List of names of methods to be exposed as XML-RPC functions
+    __rmi__ = ['toggle_debug']
+
+    def __init__(self, name=None):
+        if name is None:
+            name = "%s.%s" % (self.__class__.__module__,
+                              self.__class__.__name__)
+        self.debug_flag = False
+        self.logger = logging.getLogger(name)
+
+    def toggle_debug(self):
+        """ Turn debugging output on or off.
+
+        :returns: bool - The new value of the debug flag
+        """
+        self.debug_flag = not self.debug_flag
+        self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__,
+                                                self.debug_flag),
+                       flag=True)
+        return self.debug_flag
+
+    def debug_log(self, message, flag=None):
+        """ Log a message at the debug level.
+
+        :param message: The message to log
+        :type message: string
+        :param flag: Override the current debug flag with this value
+        :type flag: bool
+        :returns: None
+        """
+        if (flag is None and self.debug_flag) or flag:
+            self.logger.error(message)
+
+
+class Plugin(Debuggable):
+    """ The base class for all Bcfg2 Server plugins. """
+
+    #: The name of the plugin.
+    name = 'Plugin'
+
+    #: The email address of the plugin author.
+    __author__ = 'bcfg-dev@mcs.anl.gov'
+
+    #: Plugin is experimental.  Use of this plugin will produce a log
+    #: message alerting the administrator that an experimental plugin
+    #: is in use.
+    experimental = False
+
+    #: Plugin is deprecated and will be removed in a future release.
+    #: Use of this plugin will produce a log message alerting the
+    #: administrator that an experimental plugin is in use.
+    deprecated = False
+
+    #: Plugin conflicts with the list of other plugin names
+    conflicts = []
+
+    #: Plugins of the same type are processed in order of ascending
+    #: sort_order value. Plugins with the same sort_order are sorted
+    #: alphabetically by their name.
+    sort_order = 500
+
+    def __init__(self, core, datastore):
+        """ Initialize the plugin.
+
+        :param core: The Bcfg2.Server.Core initializing the plugin
+        :type core: Bcfg2.Server.Core
+        :param datastore: The path to the Bcfg2 repository on the
+                          filesystem
+        :type datastore: string
+        :raises: Bcfg2.Server.Plugin.PluginInitError
+        """
+        object.__init__(self)
+        self.Entries = {}
+        self.core = core
+        self.data = os.path.join(datastore, self.name)
+        self.running = True
+        Debuggable.__init__(self, name=self.name)
+
+    @classmethod
+    def init_repo(cls, repo):
+        """ Perform any tasks necessary to create an initial Bcfg2
+        repository.
+
+        :param repo: The path to the Bcfg2 repository on the filesystem
+        :type repo: string
+        :returns: None
+        """
+        os.makedirs(os.path.join(repo, cls.name))
+
+    def shutdown(self):
+        """ Perform shutdown tasks for the plugin
+
+        :returns: None """
+        self.running = False
+
+    def __str__(self):
+        return "%s Plugin" % self.__class__.__name__
diff --git a/src/lib/Bcfg2/Server/Plugin/exceptions.py b/src/lib/Bcfg2/Server/Plugin/exceptions.py
new file mode 100644
index 000000000..bc8c62acd
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugin/exceptions.py
@@ -0,0 +1,36 @@
+""" Exceptions for Bcfg2 Server Plugins."""
+
+class PluginInitError(Exception):
+    """Error raised in cases of :class:`Bcfg2.Server.Plugin.Plugin`
+    initialization errors."""
+    pass
+
+
+class PluginExecutionError(Exception):
+    """Error raised in case of :class:`Bcfg2.Server.Plugin.Plugin`
+    execution errors."""
+    pass
+
+
+class MetadataConsistencyError(Exception):
+    """This error gets raised when metadata is internally inconsistent."""
+    pass
+
+
+class MetadataRuntimeError(Exception):
+    """This error is raised when the metadata engine is called prior
+    to reading enough data, or for other
+    :class:`Bcfg2.Server.Plugin.Metadata` errors. """
+    pass
+
+
+class ValidationError(Exception):
+    """ Exception raised by
+    :class:`Bcfg2.Server.Plugin.StructureValidator` and
+    :class:`Bcfg2.Server.Plugin.GoalValidator` objects """
+
+
+class SpecificityError(Exception):
+    """ Thrown by :class:`Bcfg2.Server.Plugin.Specificity` in case of
+    filename parse failure."""
+    pass
diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 0b2f7cee0..74cf4b3c4 100644
--- a/src/lib/Bcfg2/Server/Plugin.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -1,4 +1,4 @@
-"""This module provides the baseclass for Bcfg2 Server Plugins."""
+""" Helper classes for Bcfg2 server plugins """
 
 import os
 import re
@@ -6,12 +6,13 @@ import sys
 import copy
 import logging
 import operator
-import threading
 import lxml.etree
 import Bcfg2.Server
 import Bcfg2.Options
-from Bcfg2.Compat import ConfigParser, CmpMixin, reduce, Queue, Empty, \
-    Full, cPickle
+from Bcfg2.Compat import CmpMixin
+from base import *
+from interfaces import *
+from exceptions import *
 
 try:
     import django
@@ -31,7 +32,7 @@ default_file_metadata = Bcfg2.Options.OptionParser(opts)
 default_file_metadata.parse([])
 del default_file_metadata['args']
 
-logger = logging.getLogger('Bcfg2.Server.Plugin')
+logger = logging.getLogger(__name__)
 
 info_regex = re.compile('owner:(\s)*(?P<owner>\S+)|' +
                         'group:(\s)*(?P<group>\S+)|' +
@@ -57,133 +58,6 @@ def bind_info(entry, metadata, infoxml=None, default=default_file_metadata):
             entry.set(attr, val)
 
 
-class PluginInitError(Exception):
-    """Error raised in cases of :class:`Bcfg2.Server.Plugin.Plugin`
-    initialization errors."""
-    pass
-
-
-class PluginExecutionError(Exception):
-    """Error raised in case of :class:`Bcfg2.Server.Plugin.Plugin`
-    execution errors."""
-    pass
-
-
-class MetadataConsistencyError(Exception):
-    """This error gets raised when metadata is internally inconsistent."""
-    pass
-
-
-class MetadataRuntimeError(Exception):
-    """This error is raised when the metadata engine is called prior
-    to reading enough data, or for other
-    :class:`Bcfg2.Server.Plugin.Metadata` errors. """
-    pass
-
-
-class Debuggable(object):
-    """ Mixin to add a debugging interface to an object and expose it
-    via XML-RPC on :class:`Bcfg2.Server.Plugin.Plugin` objects """
-
-    #: List of names of methods to be exposed as XML-RPC functions
-    __rmi__ = ['toggle_debug']
-
-    def __init__(self, name=None):
-        if name is None:
-            name = "%s.%s" % (self.__class__.__module__,
-                              self.__class__.__name__)
-        self.debug_flag = False
-        self.logger = logging.getLogger(name)
-
-    def toggle_debug(self):
-        """ Turn debugging output on or off.
-
-        :returns: bool - The new value of the debug flag
-        """
-        self.debug_flag = not self.debug_flag
-        self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__,
-                                                self.debug_flag),
-                       flag=True)
-        return self.debug_flag
-
-    def debug_log(self, message, flag=None):
-        """ Log a message at the debug level.
-
-        :param message: The message to log
-        :type message: string
-        :param flag: Override the current debug flag with this value
-        :type flag: bool
-        :returns: None
-        """
-        if (flag is None and self.debug_flag) or flag:
-            self.logger.error(message)
-
-
-class Plugin(Debuggable):
-    """ The base class for all Bcfg2 Server plugins. """
-
-    #: The name of the plugin.
-    name = 'Plugin'
-
-    #: The email address of the plugin author.
-    __author__ = 'bcfg-dev@mcs.anl.gov'
-
-    #: Plugin is experimental.  Use of this plugin will produce a log
-    #: message alerting the administrator that an experimental plugin
-    #: is in use.
-    experimental = False
-
-    #: Plugin is deprecated and will be removed in a future release.
-    #: Use of this plugin will produce a log message alerting the
-    #: administrator that an experimental plugin is in use.
-    deprecated = False
-
-    #: Plugin conflicts with the list of other plugin names
-    conflicts = []
-
-    #: Plugins of the same type are processed in order of ascending
-    #: sort_order value. Plugins with the same sort_order are sorted
-    #: alphabetically by their name.
-    sort_order = 500
-
-    def __init__(self, core, datastore):
-        """ Initialize the plugin.
-
-        :param core: The Bcfg2.Server.Core initializing the plugin
-        :type core: Bcfg2.Server.Core
-        :param datastore: The path to the Bcfg2 repository on the
-                          filesystem
-        :type datastore: string
-        :raises: Bcfg2.Server.Plugin.PluginInitError
-        """
-        object.__init__(self)
-        self.Entries = {}
-        self.core = core
-        self.data = os.path.join(datastore, self.name)
-        self.running = True
-        Debuggable.__init__(self, name=self.name)
-
-    @classmethod
-    def init_repo(cls, repo):
-        """ Perform any tasks necessary to create an initial Bcfg2
-        repository.
-
-        :param repo: The path to the Bcfg2 repository on the filesystem
-        :type repo: string
-        :returns: None
-        """
-        os.makedirs(os.path.join(repo, cls.name))
-
-    def shutdown(self):
-        """ Perform shutdown tasks for the plugin
-
-        :returns: None """
-        self.running = False
-
-    def __str__(self):
-        return "%s Plugin" % self.__class__.__name__
-
-
 class DatabaseBacked(Plugin):
     @property
     def _use_db(self):
@@ -204,551 +78,6 @@ class PluginDatabaseModel(object):
         app_label = "Server"
 
 
-class Generator(object):
-    """ Generator plugins contribute to literal client configurations.
-    That is, they generate entry contents.
-
-    An entry is generated in one of two ways:
-
-    #. The Bcfg2 core looks in the ``Entries`` dict attribute of the
-       plugin object.  ``Entries`` is expected to be a dict whose keys
-       are entry tags (e.g., ``"Path"``, ``"Service"``, etc.) and
-       whose values are dicts; those dicts should map the ``name``
-       attribute of an entry to a callable that will be called to
-       generate the content.  The callable will receive two arguments:
-       the abstract entry (as an lxml.etree._Element object), and the
-       client metadata object the entry is being generated for.
-
-    #. If the entry is not listed in ``Entries``, the Bcfg2 core calls
-       :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry`; if that
-       returns True, then it calls
-       :func:`Bcfg2.Server.Plugin.Generator.HandleEntry`.
-    """
-
-    def HandlesEntry(self, entry, metadata):
-        """ HandlesEntry is the slow path method for routing
-        configuration binding requests.  It is called if the
-        ``Entries`` dict does not contain a method for binding the
-        entry.
-
-        :param entry: The entry to bind
-        :type entry: lxml.etree._Element
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :return: bool - Whether or not this plugin can handle the entry
-        :raises: Bcfg2.Server.Plugin.PluginExecutionError
-        """
-        return False
-
-    def HandleEntry(self, entry, metadata):
-        """ HandlesEntry is the slow path method for binding
-        configuration binding requests.  It is called if the
-        ``Entries`` dict does not contain a method for binding the
-        entry, and :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry`
-        returns True.
-
-        :param entry: The entry to bind
-        :type entry: lxml.etree._Element
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :return: lxml.etree._Element - The fully bound entry
-        :raises: Bcfg2.Server.Plugin.PluginExecutionError
-        """
-        return entry
-
-
-class Structure(object):
-    """ Structure Plugins contribute to abstract client
-    configurations.  That is, they produce lists of entries that will
-    be generated for a client. """
-
-    def BuildStructures(self, metadata):
-        """ Build a list of lxml.etree._Element objects that will be
-        added to the top-level ``<Configuration>`` tag of the client
-        configuration.  Consequently, each object in the list returned
-        by ``BuildStructures()`` must consist of a container tag
-        (e.g., ``<Bundle>`` or ``<Independent>``) which contains the
-        entry tags.  It must not return a list of entry tags.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :return: list of lxml.etree._Element objects
-        """
-        raise NotImplementedError
-
-
-class Metadata(object):
-    """Signal metadata capabilities for this plugin"""
-    def viz(self, hosts, bundles, key, only_client, colors):
-        """ Return a string containing a graphviz document that maps
-        out the Metadata for :ref:`bcfg2-admin viz <server-admin-viz>`
-
-        :param hosts: Include hosts in the graph
-        :type hosts: bool
-        :param bundles: Include bundles in the graph
-        :type bundles: bool
-        :param key: Include a key in the graph
-        :type key: bool
-        :param only_client: Only include data for the specified client
-        :type only_client: string
-        :param colors: Use the specified graphviz colors
-        :type colors: list of strings
-        :return: string
-        """
-        return ''
-
-    def set_version(self, client, version):
-        """ Set the version for the named client to the specified
-        version string.
-
-        :param client: Hostname of the client
-        :type client: string
-        :param profile: Client Bcfg2 version
-        :type profile: string
-        :return: None
-        :raises: Bcfg2.Server.Plugin.MetadataRuntimeError,
-                 Bcfg2.Server.Plugin.MetadataConsistencyError
-        """
-        pass
-
-    def set_profile(self, client, profile, address):
-        """ Set the profile for the named client to the named profile
-        group.
-
-        :param client: Hostname of the client
-        :type client: string
-        :param profile: Name of the profile group
-        :type profile: string
-        :param address: Address pair of ``(<ip address>, <hostname>)``
-        :type address: tuple
-        :return: None
-        :raises: Bcfg2.Server.Plugin.MetadataRuntimeError,
-                 Bcfg2.Server.Plugin.MetadataConsistencyError
-        """
-        pass
-
-    def resolve_client(self, address, cleanup_cache=False):
-        """ Resolve the canonical name of this client.  If this method
-        is not implemented, the hostname claimed by the client is
-        used.  (This may be a security risk; it's highly recommended
-        that you implement ``resolve_client`` if you are writing a
-        Metadata plugin.)
-
-        :param address: Address pair of ``(<ip address>, <hostname>)``
-        :type address: tuple
-        :param cleanup_cache: Whether or not to remove expire the
-                              entire client hostname resolution class
-        :type cleanup_cache: bool
-        :return: string - canonical client hostname
-        :raises: Bcfg2.Server.Plugin.MetadataRuntimeError,
-                 Bcfg2.Server.Plugin.MetadataConsistencyError
-        """
-        return address[1]
-
-    def AuthenticateConnection(self, cert, user, password, address):
-        """ Authenticate the given client.
-
-        :param cert: an x509 certificate
-        :type cert: dict
-        :param user: The username of the user trying to authenticate
-        :type user: string
-        :param password: The password supplied by the client
-        :type password: string
-        :param addresspair: An address pair of ``(<ip address>,
-                            <hostname>)``
-        :type addresspair: tuple
-        :return: bool - True if the authenticate succeeds, False otherwise
-        """
-        raise NotImplementedError
-
-    def get_initial_metadata(self, client_name):
-        """ Return a
-        :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` object
-        that fully describes everything the Metadata plugin knows
-        about the named client.
-
-        :param client_name: The hostname of the client
-        :type client_name: string
-        :return: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        """
-        raise NotImplementedError
-
-    def merge_additional_data(self, imd, source, data):
-        """ Add arbitrary data from a
-        :class:`Bcfg2.Server.Plugin.Connector` plugin to the given
-        metadata object.
-
-        :param imd: An initial metadata object
-        :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param source: The name of the plugin providing this data
-        :type source: string
-        :param data: The data to add
-        :type data: any
-        :return: None
-        """
-        raise NotImplementedError
-
-    def merge_additional_groups(self, imd, groups):
-        """ Add groups from a
-        :class:`Bcfg2.Server.Plugin.Connector` plugin to the given
-        metadata object.
-
-        :param imd: An initial metadata object
-        :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param groups: The groups to add
-        :type groups: list of strings
-        :return: None
-        """
-        raise NotImplementedError
-
-
-class Connector(object):
-    """ Connector plugins augment client metadata instances with
-    additional data, additional groups, or both. """
-
-    def get_additional_groups(self, metadata):
-        """ Return a list of additional groups for the given client.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :return: list of strings
-        """
-        return list()
-
-    def get_additional_data(self, metadata):
-        """ Return arbitrary additional data for the given
-        ClientMetadata object.  By convention this is usually a dict
-        object, but doesn't need to be.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :return: list of strings
-        """
-        return dict()
-
-
-class Probing(object):
-    """ Probing plugins can collect data from clients and process it.
-    """
-
-    def GetProbes(self, metadata):
-        """ Return a list of probes for the given client.  Each probe
-        should be an lxml.etree._Element object that adheres to
-        the following specification.  Each probe must the following
-        attributes:
-
-        * ``name``: The unique name of the probe.
-        * ``source``: The origin of the probe; probably the name of
-          the plugin that supplies the probe.
-        * ``interpreter``: The command that will be run on the client
-          to interpret the probe script.  Compiled (i.e.,
-          non-interpreted) probes are not supported.
-
-        The text of the XML tag should be the contents of the probe,
-        i.e., the code that will be run on the client.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :return: list of lxml.etree._Element objects
-        """
-        raise NotImplementedError
-
-    def ReceiveData(self, metadata, datalist):
-        """ Process data returned from the probes for the given
-        client.  ``datalist`` is a list of lxml.etree._Element
-        objects, each of which is a single tag; the ``name`` attribute
-        holds the unique name of the probe that was run, and the text
-        contents of the tag hold the results of the probe.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param datalist: The probe data
-        :type datalist: list of lxml.etree._Element objects
-        :return: None
-        """
-        raise NotImplementedError
-
-
-class Statistics(Plugin):
-    """ Statistics plugins handle statistics for clients.  In general,
-    you should avoid using Statistics and use
-    :class:`Bcfg2.Server.Plugin.ThreadedStatistics` instead."""
-
-    def process_statistics(self, client, xdata):
-        """ Process the given XML statistics data for the specified
-        client.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param data: The statistics data
-        :type data: lxml.etree._Element
-        :return: None
-        """
-        raise NotImplementedError
-
-
-class ThreadedStatistics(Statistics, threading.Thread):
-    """ ThreadedStatistics plugins process client statistics in a
-    separate thread. """
-
-    def __init__(self, core, datastore):
-        Statistics.__init__(self, core, datastore)
-        threading.Thread.__init__(self)
-        # Event from the core signaling an exit
-        self.terminate = core.terminate
-        self.work_queue = Queue(100000)
-        self.pending_file = os.path.join(datastore, "etc",
-                                         "%s.pending" % self.name)
-        self.daemon = False
-        self.start()
-
-    def _save(self):
-        """Save any pending data to a file."""
-        pending_data = []
-        try:
-            while not self.work_queue.empty():
-                (metadata, data) = self.work_queue.get_nowait()
-                try:
-                    pending_data.append((metadata.hostname,
-                                         lxml.etree.tostring(data,
-                                                             xml_declaration=False).decode("UTF-8")))
-                except:
-                    err = sys.exc_info()[1]
-                    self.logger.warning("Dropping interaction for %s: %s" %
-                                        (metadata.hostname, err))
-        except Empty:
-            pass
-
-        try:
-            savefile = open(self.pending_file, 'w')
-            cPickle.dump(pending_data, savefile)
-            savefile.close()
-            self.logger.info("Saved pending %s data" % self.name)
-        except:
-            err = sys.exc_info()[1]
-            self.logger.warning("Failed to save pending data: %s" % err)
-
-    def _load(self):
-        """Load any pending data from a file."""
-        if not os.path.exists(self.pending_file):
-            return True
-        pending_data = []
-        try:
-            savefile = open(self.pending_file, 'r')
-            pending_data = cPickle.load(savefile)
-            savefile.close()
-        except Exception:
-            e = sys.exc_info()[1]
-            self.logger.warning("Failed to load pending data: %s" % e)
-            return False
-        for (pmetadata, pdata) in pending_data:
-            # check that shutdown wasnt called early
-            if self.terminate.isSet():
-                return False
-
-            try:
-                while True:
-                    try:
-                        metadata = self.core.build_metadata(pmetadata)
-                        break
-                    except MetadataRuntimeError:
-                        pass
-
-                    self.terminate.wait(5)
-                    if self.terminate.isSet():
-                        return False
-
-                self.work_queue.put_nowait((metadata,
-                                            lxml.etree.XML(pdata,
-                                                           parser=Bcfg2.Server.XMLParser)))
-            except Full:
-                self.logger.warning("Queue.Full: Failed to load queue data")
-                break
-            except lxml.etree.LxmlError:
-                lxml_error = sys.exc_info()[1]
-                self.logger.error("Unable to load saved interaction: %s" %
-                                  lxml_error)
-            except MetadataConsistencyError:
-                self.logger.error("Unable to load metadata for save "
-                                  "interaction: %s" % pmetadata)
-        try:
-            os.unlink(self.pending_file)
-        except:
-            self.logger.error("Failed to unlink save file: %s" %
-                              self.pending_file)
-        self.logger.info("Loaded pending %s data" % self.name)
-        return True
-
-    def run(self):
-        if not self._load():
-            return
-        while not self.terminate.isSet() and self.work_queue != None:
-            try:
-                (client, xdata) = self.work_queue.get(block=True, timeout=2)
-            except Empty:
-                continue
-            except Exception:
-                e = sys.exc_info()[1]
-                self.logger.error("ThreadedStatistics: %s" % e)
-                continue
-            self.handle_statistic(client, xdata)
-        if self.work_queue != None and not self.work_queue.empty():
-            self._save()
-
-    def process_statistics(self, metadata, data):
-        try:
-            self.work_queue.put_nowait((metadata, copy.copy(data)))
-        except Full:
-            self.logger.warning("%s: Queue is full.  Dropping interactions." %
-                                self.name)
-
-    def handle_statistic(self, metadata, data):
-        """ Process the given XML statistics data for the specified
-        client object.  This differs from the
-        :func:`Bcfg2.Server.Plugin.Statistics.process_statistics`
-        method only in that ThreadedStatistics first adds the data to
-        a queue, and then processes them in a separate thread.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param data: The statistics data
-        :type data: lxml.etree._Element
-        :return: None
-        """
-        raise NotImplementedError
-
-
-class PullSource(object):
-    def GetExtra(self, client):
-        return []
-
-    def GetCurrentEntry(self, client, e_type, e_name):
-        raise NotImplementedError
-
-
-class PullTarget(object):
-    def AcceptChoices(self, entry, metadata):
-        raise NotImplementedError
-
-    def AcceptPullData(self, specific, new_entry, verbose):
-        raise NotImplementedError
-
-
-class Decision(object):
-    """ Decision plugins produce decision lists for affecting which
-    entries are actually installed on clients. """
-
-    def GetDecisions(self, metadata, mode):
-        """ Return a list of tuples of ``(<entry type>, <entry
-        name>)`` to be used as the decision list for the given
-        client in the specified mode.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param mode: The decision mode ("whitelist" or "blacklist")
-        :type mode: string
-        :return: list of tuples
-        """
-        raise NotImplementedError
-
-
-class ValidationError(Exception):
-    """ Exception raised by
-    :class:`Bcfg2.Server.Plugin.StructureValidator` and
-    :class:`Bcfg2.Server.Plugin.GoalValidator` objects """
-
-
-class StructureValidator(object):
-    """ StructureValidator plugins can modify the list of structures
-    after it has been created but before the entries have been
-    concretely bound. """
-
-    def validate_structures(self, metadata, structures):
-        """ Given a list of structures (i.e., of tags that contain
-        entry tags), modify that list or the structures in it
-        in-place.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param config: A list of lxml.etree._Element objects
-                       describing the structures for this client
-        :type config: list
-        :returns: None
-        :raises: Bcfg2.Server.Plugin.ValidationError
-        """
-        raise NotImplementedError
-
-
-class GoalValidator(object):
-    """ GoalValidator plugins can modify the concretely-bound configuration of
-    a client as a last stage before the configuration is sent to the
-    client. """
-
-    def validate_goals(self, metadata, config):
-        """ Given a monolithic XML document of the full configuration,
-        modify the document in-place.
-
-        :param metadata: The client metadata
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :param config: The full configuration for the client
-        :type config: lxml.etree._Element
-        :returns: None
-        :raises: Bcfg2.Server.Plugin.ValidationError
-        """
-        raise NotImplementedError
-
-
-class Version(object):
-    """ Version plugins interact with various version control systems. """
-
-    def get_revision(self):
-        """ Return the current revision of the Bcfg2 specification.
-        This will be included in the ``revision`` attribute of the
-        top-level tag of the XML configuration sent to the client.
-
-        :returns: string - the current version
-        """
-        raise NotImplementedError
-
-
-class ClientRunHooks(object):
-    """ ClientRunHooks can hook into various parts of a client run to
-    perform actions at various times without needing to pretend to be
-    a different plugin type. """
-
-    def start_client_run(self, metadata):
-        """ Invoked at the start of a client run, after all probe data
-        has been received and decision lists have been queried (if
-        applicable), but before the configuration is generated.
-
-        :param metadata: The client metadata object
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :returns: None
-        """
-        pass
-
-    def end_client_run(self, metadata):
-        """ Invoked at the end of a client run, immediately after
-        :class:`Bcfg2.Server.Plugin.GoalValidator` plugins have been run
-        and just before the configuration is returned to the client.
-
-        :param metadata: The client metadata object
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :returns: None
-        """        
-        pass
-
-    def end_statistics(self, metadata):
-        """ Invoked after statistics are processed for a client.
-
-        :param metadata: The client metadata object
-        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :returns: None
-        """
-        pass
-
-# the rest of the file contains classes for coherent file caching
-
 class FileBacked(object):
     """This object caches file data in memory.
     HandleEvent is called whenever fam registers an event.
@@ -1321,12 +650,6 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked):
                      if not key.startswith('__')])
 
 
-# new unified EntrySet backend
-class SpecificityError(Exception):
-    """Thrown in case of filename parse failure."""
-    pass
-
-
 class Specificity(CmpMixin):
     def __init__(self, all=False, group=False, hostname=False, prio=0,
                  delta=False):
diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py
new file mode 100644
index 000000000..a6543e9b9
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py
@@ -0,0 +1,548 @@
+""" Interface definitions for Bcfg2 server plugins """
+
+import os
+import sys
+import copy
+import threading
+import lxml.etree
+import Bcfg2.Server
+from Bcfg2.Compat import Queue, Empty, Full, cPickle
+from exceptions import *
+from base import Plugin
+
+class Generator(object):
+    """ Generator plugins contribute to literal client configurations.
+    That is, they generate entry contents.
+
+    An entry is generated in one of two ways:
+
+    #. The Bcfg2 core looks in the ``Entries`` dict attribute of the
+       plugin object.  ``Entries`` is expected to be a dict whose keys
+       are entry tags (e.g., ``"Path"``, ``"Service"``, etc.) and
+       whose values are dicts; those dicts should map the ``name``
+       attribute of an entry to a callable that will be called to
+       generate the content.  The callable will receive two arguments:
+       the abstract entry (as an lxml.etree._Element object), and the
+       client metadata object the entry is being generated for.
+
+    #. If the entry is not listed in ``Entries``, the Bcfg2 core calls
+       :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry`; if that
+       returns True, then it calls
+       :func:`Bcfg2.Server.Plugin.Generator.HandleEntry`.
+    """
+
+    def HandlesEntry(self, entry, metadata):
+        """ HandlesEntry is the slow path method for routing
+        configuration binding requests.  It is called if the
+        ``Entries`` dict does not contain a method for binding the
+        entry.
+
+        :param entry: The entry to bind
+        :type entry: lxml.etree._Element
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :return: bool - Whether or not this plugin can handle the entry
+        :raises: Bcfg2.Server.Plugin.PluginExecutionError
+        """
+        return False
+
+    def HandleEntry(self, entry, metadata):
+        """ HandlesEntry is the slow path method for binding
+        configuration binding requests.  It is called if the
+        ``Entries`` dict does not contain a method for binding the
+        entry, and :func:`Bcfg2.Server.Plugin.Generator.HandlesEntry`
+        returns True.
+
+        :param entry: The entry to bind
+        :type entry: lxml.etree._Element
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :return: lxml.etree._Element - The fully bound entry
+        :raises: Bcfg2.Server.Plugin.PluginExecutionError
+        """
+        return entry
+
+
+class Structure(object):
+    """ Structure Plugins contribute to abstract client
+    configurations.  That is, they produce lists of entries that will
+    be generated for a client. """
+
+    def BuildStructures(self, metadata):
+        """ Build a list of lxml.etree._Element objects that will be
+        added to the top-level ``<Configuration>`` tag of the client
+        configuration.  Consequently, each object in the list returned
+        by ``BuildStructures()`` must consist of a container tag
+        (e.g., ``<Bundle>`` or ``<Independent>``) which contains the
+        entry tags.  It must not return a list of entry tags.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :return: list of lxml.etree._Element objects
+        """
+        raise NotImplementedError
+
+
+class Metadata(object):
+    """Signal metadata capabilities for this plugin"""
+    def viz(self, hosts, bundles, key, only_client, colors):
+        """ Return a string containing a graphviz document that maps
+        out the Metadata for :ref:`bcfg2-admin viz <server-admin-viz>`
+
+        :param hosts: Include hosts in the graph
+        :type hosts: bool
+        :param bundles: Include bundles in the graph
+        :type bundles: bool
+        :param key: Include a key in the graph
+        :type key: bool
+        :param only_client: Only include data for the specified client
+        :type only_client: string
+        :param colors: Use the specified graphviz colors
+        :type colors: list of strings
+        :return: string
+        """
+        return ''
+
+    def set_version(self, client, version):
+        """ Set the version for the named client to the specified
+        version string.
+
+        :param client: Hostname of the client
+        :type client: string
+        :param profile: Client Bcfg2 version
+        :type profile: string
+        :return: None
+        :raises: Bcfg2.Server.Plugin.MetadataRuntimeError,
+                 Bcfg2.Server.Plugin.MetadataConsistencyError
+        """
+        pass
+
+    def set_profile(self, client, profile, address):
+        """ Set the profile for the named client to the named profile
+        group.
+
+        :param client: Hostname of the client
+        :type client: string
+        :param profile: Name of the profile group
+        :type profile: string
+        :param address: Address pair of ``(<ip address>, <hostname>)``
+        :type address: tuple
+        :return: None
+        :raises: Bcfg2.Server.Plugin.MetadataRuntimeError,
+                 Bcfg2.Server.Plugin.MetadataConsistencyError
+        """
+        pass
+
+    def resolve_client(self, address, cleanup_cache=False):
+        """ Resolve the canonical name of this client.  If this method
+        is not implemented, the hostname claimed by the client is
+        used.  (This may be a security risk; it's highly recommended
+        that you implement ``resolve_client`` if you are writing a
+        Metadata plugin.)
+
+        :param address: Address pair of ``(<ip address>, <hostname>)``
+        :type address: tuple
+        :param cleanup_cache: Whether or not to remove expire the
+                              entire client hostname resolution class
+        :type cleanup_cache: bool
+        :return: string - canonical client hostname
+        :raises: Bcfg2.Server.Plugin.MetadataRuntimeError,
+                 Bcfg2.Server.Plugin.MetadataConsistencyError
+        """
+        return address[1]
+
+    def AuthenticateConnection(self, cert, user, password, address):
+        """ Authenticate the given client.
+
+        :param cert: an x509 certificate
+        :type cert: dict
+        :param user: The username of the user trying to authenticate
+        :type user: string
+        :param password: The password supplied by the client
+        :type password: string
+        :param addresspair: An address pair of ``(<ip address>,
+                            <hostname>)``
+        :type addresspair: tuple
+        :return: bool - True if the authenticate succeeds, False otherwise
+        """
+        raise NotImplementedError
+
+    def get_initial_metadata(self, client_name):
+        """ Return a
+        :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` object
+        that fully describes everything the Metadata plugin knows
+        about the named client.
+
+        :param client_name: The hostname of the client
+        :type client_name: string
+        :return: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        """
+        raise NotImplementedError
+
+    def merge_additional_data(self, imd, source, data):
+        """ Add arbitrary data from a
+        :class:`Bcfg2.Server.Plugin.Connector` plugin to the given
+        metadata object.
+
+        :param imd: An initial metadata object
+        :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param source: The name of the plugin providing this data
+        :type source: string
+        :param data: The data to add
+        :type data: any
+        :return: None
+        """
+        raise NotImplementedError
+
+    def merge_additional_groups(self, imd, groups):
+        """ Add groups from a
+        :class:`Bcfg2.Server.Plugin.Connector` plugin to the given
+        metadata object.
+
+        :param imd: An initial metadata object
+        :type imd: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param groups: The groups to add
+        :type groups: list of strings
+        :return: None
+        """
+        raise NotImplementedError
+
+
+class Connector(object):
+    """ Connector plugins augment client metadata instances with
+    additional data, additional groups, or both. """
+
+    def get_additional_groups(self, metadata):
+        """ Return a list of additional groups for the given client.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :return: list of strings
+        """
+        return list()
+
+    def get_additional_data(self, metadata):
+        """ Return arbitrary additional data for the given
+        ClientMetadata object.  By convention this is usually a dict
+        object, but doesn't need to be.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :return: list of strings
+        """
+        return dict()
+
+
+class Probing(object):
+    """ Probing plugins can collect data from clients and process it.
+    """
+
+    def GetProbes(self, metadata):
+        """ Return a list of probes for the given client.  Each probe
+        should be an lxml.etree._Element object that adheres to
+        the following specification.  Each probe must the following
+        attributes:
+
+        * ``name``: The unique name of the probe.
+        * ``source``: The origin of the probe; probably the name of
+          the plugin that supplies the probe.
+        * ``interpreter``: The command that will be run on the client
+          to interpret the probe script.  Compiled (i.e.,
+          non-interpreted) probes are not supported.
+
+        The text of the XML tag should be the contents of the probe,
+        i.e., the code that will be run on the client.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :return: list of lxml.etree._Element objects
+        """
+        raise NotImplementedError
+
+    def ReceiveData(self, metadata, datalist):
+        """ Process data returned from the probes for the given
+        client.  ``datalist`` is a list of lxml.etree._Element
+        objects, each of which is a single tag; the ``name`` attribute
+        holds the unique name of the probe that was run, and the text
+        contents of the tag hold the results of the probe.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param datalist: The probe data
+        :type datalist: list of lxml.etree._Element objects
+        :return: None
+        """
+        raise NotImplementedError
+
+
+class Statistics(Plugin):
+    """ Statistics plugins handle statistics for clients.  In general,
+    you should avoid using Statistics and use
+    :class:`Bcfg2.Server.Plugin.ThreadedStatistics` instead."""
+
+    def process_statistics(self, client, xdata):
+        """ Process the given XML statistics data for the specified
+        client.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param data: The statistics data
+        :type data: lxml.etree._Element
+        :return: None
+        """
+        raise NotImplementedError
+
+
+class ThreadedStatistics(Statistics, threading.Thread):
+    """ ThreadedStatistics plugins process client statistics in a
+    separate thread. """
+
+    def __init__(self, core, datastore):
+        Statistics.__init__(self, core, datastore)
+        threading.Thread.__init__(self)
+        # Event from the core signaling an exit
+        self.terminate = core.terminate
+        self.work_queue = Queue(100000)
+        self.pending_file = os.path.join(datastore, "etc",
+                                         "%s.pending" % self.name)
+        self.daemon = False
+        self.start()
+
+    def _save(self):
+        """Save any pending data to a file."""
+        pending_data = []
+        try:
+            while not self.work_queue.empty():
+                (metadata, data) = self.work_queue.get_nowait()
+                try:
+                    pending_data.append((metadata.hostname,
+                                         lxml.etree.tostring(data,
+                                                             xml_declaration=False).decode("UTF-8")))
+                except:
+                    err = sys.exc_info()[1]
+                    self.logger.warning("Dropping interaction for %s: %s" %
+                                        (metadata.hostname, err))
+        except Empty:
+            pass
+
+        try:
+            savefile = open(self.pending_file, 'w')
+            cPickle.dump(pending_data, savefile)
+            savefile.close()
+            self.logger.info("Saved pending %s data" % self.name)
+        except:
+            err = sys.exc_info()[1]
+            self.logger.warning("Failed to save pending data: %s" % err)
+
+    def _load(self):
+        """Load any pending data from a file."""
+        if not os.path.exists(self.pending_file):
+            return True
+        pending_data = []
+        try:
+            savefile = open(self.pending_file, 'r')
+            pending_data = cPickle.load(savefile)
+            savefile.close()
+        except Exception:
+            e = sys.exc_info()[1]
+            self.logger.warning("Failed to load pending data: %s" % e)
+            return False
+        for (pmetadata, pdata) in pending_data:
+            # check that shutdown wasnt called early
+            if self.terminate.isSet():
+                return False
+
+            try:
+                while True:
+                    try:
+                        metadata = self.core.build_metadata(pmetadata)
+                        break
+                    except MetadataRuntimeError:
+                        pass
+
+                    self.terminate.wait(5)
+                    if self.terminate.isSet():
+                        return False
+
+                self.work_queue.put_nowait((metadata,
+                                            lxml.etree.XML(pdata,
+                                                           parser=Bcfg2.Server.XMLParser)))
+            except Full:
+                self.logger.warning("Queue.Full: Failed to load queue data")
+                break
+            except lxml.etree.LxmlError:
+                lxml_error = sys.exc_info()[1]
+                self.logger.error("Unable to load saved interaction: %s" %
+                                  lxml_error)
+            except MetadataConsistencyError:
+                self.logger.error("Unable to load metadata for save "
+                                  "interaction: %s" % pmetadata)
+        try:
+            os.unlink(self.pending_file)
+        except:
+            self.logger.error("Failed to unlink save file: %s" %
+                              self.pending_file)
+        self.logger.info("Loaded pending %s data" % self.name)
+        return True
+
+    def run(self):
+        if not self._load():
+            return
+        while not self.terminate.isSet() and self.work_queue != None:
+            try:
+                (client, xdata) = self.work_queue.get(block=True, timeout=2)
+            except Empty:
+                continue
+            except Exception:
+                e = sys.exc_info()[1]
+                self.logger.error("ThreadedStatistics: %s" % e)
+                continue
+            self.handle_statistic(client, xdata)
+        if self.work_queue != None and not self.work_queue.empty():
+            self._save()
+
+    def process_statistics(self, metadata, data):
+        try:
+            self.work_queue.put_nowait((metadata, copy.copy(data)))
+        except Full:
+            self.logger.warning("%s: Queue is full.  Dropping interactions." %
+                                self.name)
+
+    def handle_statistic(self, metadata, data):
+        """ Process the given XML statistics data for the specified
+        client object.  This differs from the
+        :func:`Bcfg2.Server.Plugin.Statistics.process_statistics`
+        method only in that ThreadedStatistics first adds the data to
+        a queue, and then processes them in a separate thread.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param data: The statistics data
+        :type data: lxml.etree._Element
+        :return: None
+        """
+        raise NotImplementedError
+
+
+class PullSource(object):
+    def GetExtra(self, client):
+        return []
+
+    def GetCurrentEntry(self, client, e_type, e_name):
+        raise NotImplementedError
+
+
+class PullTarget(object):
+    def AcceptChoices(self, entry, metadata):
+        raise NotImplementedError
+
+    def AcceptPullData(self, specific, new_entry, verbose):
+        raise NotImplementedError
+
+
+class Decision(object):
+    """ Decision plugins produce decision lists for affecting which
+    entries are actually installed on clients. """
+
+    def GetDecisions(self, metadata, mode):
+        """ Return a list of tuples of ``(<entry type>, <entry
+        name>)`` to be used as the decision list for the given
+        client in the specified mode.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param mode: The decision mode ("whitelist" or "blacklist")
+        :type mode: string
+        :return: list of tuples
+        """
+        raise NotImplementedError
+
+
+class StructureValidator(object):
+    """ StructureValidator plugins can modify the list of structures
+    after it has been created but before the entries have been
+    concretely bound. """
+
+    def validate_structures(self, metadata, structures):
+        """ Given a list of structures (i.e., of tags that contain
+        entry tags), modify that list or the structures in it
+        in-place.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param config: A list of lxml.etree._Element objects
+                       describing the structures for this client
+        :type config: list
+        :returns: None
+        :raises: Bcfg2.Server.Plugin.ValidationError
+        """
+        raise NotImplementedError
+
+
+class GoalValidator(object):
+    """ GoalValidator plugins can modify the concretely-bound configuration of
+    a client as a last stage before the configuration is sent to the
+    client. """
+
+    def validate_goals(self, metadata, config):
+        """ Given a monolithic XML document of the full configuration,
+        modify the document in-place.
+
+        :param metadata: The client metadata
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :param config: The full configuration for the client
+        :type config: lxml.etree._Element
+        :returns: None
+        :raises: Bcfg2.Server.Plugin.ValidationError
+        """
+        raise NotImplementedError
+
+
+class Version(object):
+    """ Version plugins interact with various version control systems. """
+
+    def get_revision(self):
+        """ Return the current revision of the Bcfg2 specification.
+        This will be included in the ``revision`` attribute of the
+        top-level tag of the XML configuration sent to the client.
+
+        :returns: string - the current version
+        """
+        raise NotImplementedError
+
+
+class ClientRunHooks(object):
+    """ ClientRunHooks can hook into various parts of a client run to
+    perform actions at various times without needing to pretend to be
+    a different plugin type. """
+
+    def start_client_run(self, metadata):
+        """ Invoked at the start of a client run, after all probe data
+        has been received and decision lists have been queried (if
+        applicable), but before the configuration is generated.
+
+        :param metadata: The client metadata object
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :returns: None
+        """
+        pass
+
+    def end_client_run(self, metadata):
+        """ Invoked at the end of a client run, immediately after
+        :class:`Bcfg2.Server.Plugin.GoalValidator` plugins have been run
+        and just before the configuration is returned to the client.
+
+        :param metadata: The client metadata object
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :returns: None
+        """        
+        pass
+
+    def end_statistics(self, metadata):
+        """ Invoked after statistics are processed for a client.
+
+        :param metadata: The client metadata object
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :returns: None
+        """
+        pass