Plugin: added new Caching interface

This gives a single unified interface for expiring caches, no matter
the plugin.  This will be particularly useful with the
MultiprocessingCore, as certain calls must be dispatched to child
processes to expire their caches.

8 files changed, 58 insertions, 15 deletions

diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 1b56099df..b577f65e1 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -263,6 +263,20 @@ class BaseCore(object):
         #: metadata
         self.metadata_cache = Cache()
 
+    def expire_caches_by_type(self, base_cls, key=None):
+        """ Expire caches for all
+        :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that
+        are instances of ``base_cls``.
+
+        :param base_cls: The base plugin interface class to match (see
+                         :mod:`Bcfg2.Server.Plugin.interfaces`)
+        :type base_cls: type
+        :param key: The cache key to expire
+        """
+        for plugin in self.plugins_by_type(base_cls):
+            if isinstance(plugin, Bcfg2.Server.Plugin.Caching):
+                plugin.expire_cache(key)
+
     def plugins_by_type(self, base_cls):
         """ Return a list of loaded plugins that match the passed type.
 
@@ -728,7 +742,7 @@ class BaseCore(object):
         if event.code2str() == 'deleted':
             return
         self.setup.reparse()
-        self.metadata_cache.expire()
+        self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata)
 
     def block_for_fam_events(self, handle_events=False):
         """ Block until all fam events have been handleed, optionally
@@ -1084,7 +1098,7 @@ class BaseCore(object):
             # that's created for RecvProbeData doesn't get cached.
             # I.e., the next metadata object that's built, after probe
             # data is processed, is cached.
-            self.metadata_cache.expire(client)
+            self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata)
         try:
             xpdata = lxml.etree.XML(probedata.encode('utf-8'),
                                     parser=Bcfg2.Server.XMLParser)
diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py
index b690242f8..c185a5893 100644
--- a/src/lib/Bcfg2/Server/MultiprocessingCore.py
+++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py
@@ -15,16 +15,16 @@ import time
 import threading
 import lxml.etree
 import multiprocessing
+import Bcfg2.Server.Plugin
 from itertools import cycle
 from Bcfg2.Cache import Cache
 from Bcfg2.Compat import Queue, Empty
-from Bcfg2.Server.Plugin import Debuggable
 from Bcfg2.Server.Core import BaseCore, exposed
 from Bcfg2.Server.BuiltinCore import Core as BuiltinCore
 from multiprocessing.connection import Listener, Client
 
 
-class DispatchingCache(Cache, Debuggable):
+class DispatchingCache(Cache, Bcfg2.Server.Plugin.Debuggable):
     """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates
     cache expiration events to child nodes. """
 
@@ -33,7 +33,7 @@ class DispatchingCache(Cache, Debuggable):
 
     def __init__(self, *args, **kwargs):
         self.rpc_q = kwargs.pop("queue")
-        Debuggable.__init__(self)
+        Bcfg2.Server.Plugin.Debuggable.__init__(self)
         Cache.__init__(self, *args, **kwargs)
 
     def expire(self, key=None):
@@ -41,7 +41,7 @@ class DispatchingCache(Cache, Debuggable):
         Cache.expire(self, key=key)
 
 
-class RPCQueue(Debuggable):
+class RPCQueue(Bcfg2.Server.Plugin.Debuggable):
     """ An implementation of a :class:`multiprocessing.Queue` designed
     for several additional use patterns:
 
@@ -54,7 +54,7 @@ class RPCQueue(Debuggable):
     poll_wait = 3.0
 
     def __init__(self):
-        Debuggable.__init__(self)
+        Bcfg2.Server.Plugin.Debuggable.__init__(self)
         self._terminate = threading.Event()
         self._queues = dict()
         self._available_listeners = Queue()
@@ -293,9 +293,9 @@ class ChildCore(BaseCore):
     @exposed
     def RecvProbeData(self, address, _):
         """ Expire the probe cache for a client """
-        if 'Probes' in self.plugins:
-            client = self.resolve_client(address, metadata=False)[0]
-            self.plugins['Probes'].load_data(client)
+        self.expire_caches_by_type(Bcfg2.Server.Plugin.Probing,
+                                   key=self.resolve_client(address,
+                                                           metadata=False)[0])
 
     @exposed
     def GetConfig(self, client):
diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py
index 222b94fe3..2dbf75f42 100644
--- a/src/lib/Bcfg2/Server/Plugin/interfaces.py
+++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py
@@ -598,3 +598,11 @@ class ClientRunHooks(object):
         :returns: None
         """
         pass
+
+
+class Caching(object):
+    """ A plugin that caches more than just the data received from the
+    FAM.  This presents a unified interface to clear the cache. """
+
+    def expire_cache(self, key=None):
+        raise NotImplementedError
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index cc0456334..03323d64b 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -487,6 +487,7 @@ class MetadataGroup(tuple):  # pylint: disable=E0012,R0924
 
 
 class Metadata(Bcfg2.Server.Plugin.Metadata,
+               Bcfg2.Server.Plugin.Caching,
                Bcfg2.Server.Plugin.ClientRunHooks,
                Bcfg2.Server.Plugin.DatabaseBacked):
     """This class contains data for bcfg2 server metadata."""
@@ -495,6 +496,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
 
     def __init__(self, core, datastore, watch_clients=True):
         Bcfg2.Server.Plugin.Metadata.__init__(self)
+        Bcfg2.Server.Plugin.Caching.__init__(self)
         Bcfg2.Server.Plugin.ClientRunHooks.__init__(self)
         Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore)
         self.watch_clients = watch_clients
@@ -934,13 +936,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                     self.groups[gname]
         self.states['groups.xml'] = True
 
+    def expire_cache(self, key=None):
+        self.core.metadata_cache.expire(key)
+
     def HandleEvent(self, event):
         """Handle update events for data files."""
         for handles, event_handler in self.handlers.items():
             if handles(event):
                 # clear the entire cache when we get an event for any
                 # metadata file
-                self.core.metadata_cache.expire()
+                self.expire_cache()
                 event_handler(event)
 
         if False not in list(self.states.values()) and self.debug_flag:
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 470a52bbc..3e4fb33ec 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -70,6 +70,7 @@ class OnDemandDict(MutableMapping):
 
 
 class Packages(Bcfg2.Server.Plugin.Plugin,
+               Bcfg2.Server.Plugin.Caching,
                Bcfg2.Server.Plugin.StructureValidator,
                Bcfg2.Server.Plugin.Generator,
                Bcfg2.Server.Plugin.Connector,
@@ -94,6 +95,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
 
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+        Bcfg2.Server.Plugin.Caching.__init__(self)
         Bcfg2.Server.Plugin.StructureValidator.__init__(self)
         Bcfg2.Server.Plugin.Generator.__init__(self)
         Bcfg2.Server.Plugin.Connector.__init__(self)
@@ -458,6 +460,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         self._load_config()
         return True
 
+    def expire_cache(self, _=None):
+        self.Reload()
+
     def _load_config(self, force_update=False):
         """
         Load the configuration data and setup sources
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index bf59809f7..b9f93052a 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -181,14 +181,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
 
 
 class Probes(Bcfg2.Server.Plugin.Probing,
+             Bcfg2.Server.Plugin.Caching,
              Bcfg2.Server.Plugin.Connector,
              Bcfg2.Server.Plugin.DatabaseBacked):
     """ A plugin to gather information from a client machine """
     __author__ = 'bcfg-dev@mcs.anl.gov'
 
     def __init__(self, core, datastore):
-        Bcfg2.Server.Plugin.Connector.__init__(self)
         Bcfg2.Server.Plugin.Probing.__init__(self)
+        Bcfg2.Server.Plugin.Caching.__init__(self)
+        Bcfg2.Server.Plugin.Connector.__init__(self)
         Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore)
 
         try:
@@ -266,6 +268,9 @@ class Probes(Bcfg2.Server.Plugin.Probing,
             hostname=client.hostname).exclude(
             group__in=self.cgroups[client.hostname]).delete()
 
+    def expire_cache(self, key=None):
+        self.load_data(client=key)
+
     def load_data(self, client=None):
         """ Load probe data from the appropriate backend (probed.xml
         or the database) """
@@ -299,7 +304,7 @@ class Probes(Bcfg2.Server.Plugin.Probing,
                     self.cgroups[client.get('name')].append(pdata.get('name'))
 
         if self.core.metadata_cache_mode in ['cautious', 'aggressive']:
-            self.core.metadata_cache.expire()
+            self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata)
 
     def _load_data_db(self, client=None):
         """ Load probe data from the database """
@@ -325,7 +330,8 @@ class Probes(Bcfg2.Server.Plugin.Probing,
             self.cgroups[pgroup.hostname].append(pgroup.group)
 
         if self.core.metadata_cache_mode in ['cautious', 'aggressive']:
-            self.core.metadata_cache.expire(client)
+            self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata,
+                                            key=client)
 
     @Bcfg2.Server.Plugin.track_statistics()
     def GetProbes(self, meta):
diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py
index 801e7006d..072f3f7e7 100644
--- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py
+++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py
@@ -127,7 +127,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin,
             self.logger.warning("PuppetENC is incompatible with aggressive "
                                 "client metadata caching, try 'cautious' or "
                                 "'initial' instead")
-            self.core.cache.expire()
+            self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata)
 
     def end_statistics(self, metadata):
         self.end_client_run(self, metadata)
diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
index d8b3104b7..2deea5f07 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
@@ -92,6 +92,7 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet):
 
 
 class SSHbase(Bcfg2.Server.Plugin.Plugin,
+              Bcfg2.Server.Plugin.Caching,
               Bcfg2.Server.Plugin.Generator,
               Bcfg2.Server.Plugin.PullTarget):
     """
@@ -125,6 +126,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
 
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+        Bcfg2.Server.Plugin.Caching.__init__(self)
         Bcfg2.Server.Plugin.Generator.__init__(self)
         Bcfg2.Server.Plugin.PullTarget.__init__(self)
         self.ipcache = {}
@@ -149,6 +151,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
                 HostKeyEntrySet(keypattern, self.data)
             self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk
 
+    def expire_cache(self, key=None):
+        self.__skn = False
+
     def get_skn(self):
         """Build memory cache of the ssh known hosts file."""
         if not self.__skn: