From 7e9787c947e99b68317f5420951a296cea858daa Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 7 Aug 2013 11:37:38 -0400 Subject: Plugin: added new Caching interface This gives a single unified interface for expiring caches, no matter the plugin. This will be particularly useful with the MultiprocessingCore, as certain calls must be dispatched to child processes to expire their caches. --- src/lib/Bcfg2/Server/Core.py | 18 ++++++++++++++++-- src/lib/Bcfg2/Server/MultiprocessingCore.py | 16 ++++++++-------- src/lib/Bcfg2/Server/Plugin/interfaces.py | 8 ++++++++ src/lib/Bcfg2/Server/Plugins/Metadata.py | 7 ++++++- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 5 +++++ src/lib/Bcfg2/Server/Plugins/Probes.py | 12 +++++++++--- src/lib/Bcfg2/Server/Plugins/PuppetENC.py | 2 +- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 5 +++++ src/sbin/bcfg2-info | 5 +++-- 9 files changed, 61 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 1b56099df..b577f65e1 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -263,6 +263,20 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + def expire_caches_by_type(self, base_cls, key=None): + """ Expire caches for all + :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that + are instances of ``base_cls``. + + :param base_cls: The base plugin interface class to match (see + :mod:`Bcfg2.Server.Plugin.interfaces`) + :type base_cls: type + :param key: The cache key to expire + """ + for plugin in self.plugins_by_type(base_cls): + if isinstance(plugin, Bcfg2.Server.Plugin.Caching): + plugin.expire_cache(key) + def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -728,7 +742,7 @@ class BaseCore(object): if event.code2str() == 'deleted': return self.setup.reparse() - self.metadata_cache.expire() + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def block_for_fam_events(self, handle_events=False): """ Block until all fam events have been handleed, optionally @@ -1084,7 +1098,7 @@ class BaseCore(object): # that's created for RecvProbeData doesn't get cached. # I.e., the next metadata object that's built, after probe # data is processed, is cached. - self.metadata_cache.expire(client) + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) try: xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index b690242f8..c185a5893 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -15,16 +15,16 @@ import time import threading import lxml.etree import multiprocessing +import Bcfg2.Server.Plugin from itertools import cycle from Bcfg2.Cache import Cache from Bcfg2.Compat import Queue, Empty -from Bcfg2.Server.Plugin import Debuggable from Bcfg2.Server.Core import BaseCore, exposed from Bcfg2.Server.BuiltinCore import Core as BuiltinCore from multiprocessing.connection import Listener, Client -class DispatchingCache(Cache, Debuggable): +class DispatchingCache(Cache, Bcfg2.Server.Plugin.Debuggable): """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates cache expiration events to child nodes. """ @@ -33,7 +33,7 @@ class DispatchingCache(Cache, Debuggable): def __init__(self, *args, **kwargs): self.rpc_q = kwargs.pop("queue") - Debuggable.__init__(self) + Bcfg2.Server.Plugin.Debuggable.__init__(self) Cache.__init__(self, *args, **kwargs) def expire(self, key=None): @@ -41,7 +41,7 @@ class DispatchingCache(Cache, Debuggable): Cache.expire(self, key=key) -class RPCQueue(Debuggable): +class RPCQueue(Bcfg2.Server.Plugin.Debuggable): """ An implementation of a :class:`multiprocessing.Queue` designed for several additional use patterns: @@ -54,7 +54,7 @@ class RPCQueue(Debuggable): poll_wait = 3.0 def __init__(self): - Debuggable.__init__(self) + Bcfg2.Server.Plugin.Debuggable.__init__(self) self._terminate = threading.Event() self._queues = dict() self._available_listeners = Queue() @@ -293,9 +293,9 @@ class ChildCore(BaseCore): @exposed def RecvProbeData(self, address, _): """ Expire the probe cache for a client """ - if 'Probes' in self.plugins: - client = self.resolve_client(address, metadata=False)[0] - self.plugins['Probes'].load_data(client) + self.expire_caches_by_type(Bcfg2.Server.Plugin.Probing, + key=self.resolve_client(address, + metadata=False)[0]) @exposed def GetConfig(self, client): diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 222b94fe3..2dbf75f42 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -598,3 +598,11 @@ class ClientRunHooks(object): :returns: None """ pass + + +class Caching(object): + """ A plugin that caches more than just the data received from the + FAM. This presents a unified interface to clear the cache. """ + + def expire_cache(self, key=None): + raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index cc0456334..03323d64b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -487,6 +487,7 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" @@ -495,6 +496,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) self.watch_clients = watch_clients @@ -934,13 +936,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[gname] self.states['groups.xml'] = True + def expire_cache(self, key=None): + self.core.metadata_cache.expire(key) + def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.core.metadata_cache.expire() + self.expire_cache() event_handler(event) if False not in list(self.states.values()) and self.debug_flag: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 470a52bbc..3e4fb33ec 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -70,6 +70,7 @@ class OnDemandDict(MutableMapping): class Packages(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -94,6 +95,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -458,6 +460,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True + def expire_cache(self, _=None): + self.Reload() + def _load_config(self, force_update=False): """ Load the configuration data and setup sources diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index bf59809f7..b9f93052a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -181,14 +181,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: @@ -266,6 +268,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, hostname=client.hostname).exclude( group__in=self.cgroups[client.hostname]).delete() + def expire_cache(self, key=None): + self.load_data(client=key) + def load_data(self, client=None): """ Load probe data from the appropriate backend (probed.xml or the database) """ @@ -299,7 +304,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[client.get('name')].append(pdata.get('name')) if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.metadata_cache.expire() + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def _load_data_db(self, client=None): """ Load probe data from the database """ @@ -325,7 +330,8 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[pgroup.hostname].append(pgroup.group) if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.metadata_cache.expire(client) + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) @Bcfg2.Server.Plugin.track_statistics() def GetProbes(self, meta): diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 801e7006d..072f3f7e7 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -127,7 +127,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.warning("PuppetENC is incompatible with aggressive " "client metadata caching, try 'cautious' or " "'initial' instead") - self.core.cache.expire() + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def end_statistics(self, metadata): self.end_client_run(self, metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index d8b3104b7..2deea5f07 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -92,6 +92,7 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -125,6 +126,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -149,6 +151,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, HostKeyEntrySet(keypattern, self.data) self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk + def expire_cache(self, key=None): + self.__skn = False + def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 451d8e49c..6008f8896 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -488,9 +488,10 @@ Bcfg2 client itself.""") alist = args.split() if len(alist): for client in self._get_client_list(alist): - self.metadata_cache.expire(client) + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) else: - self.metadata_cache.expire() + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def do_probes(self, args): """ probes [-p] - Get probe list for the given -- cgit v1.2.3-1-g7c22