From 66c272c383c52343b5a201ab59ca2e0e1ee8ee2c Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 12 Jul 2016 02:51:18 +0200 Subject: Server/Plugins/Ldap: Cache the results of the Ldap queries Using the OnDemandDict removes the results of Ldap queries from the client_metadata cache. We add a new cache per hostname cache for the single ldap queries and add a new configuration option to enable caching until the cache is expired manually via XML-RPC. --- doc/development/caching.txt | 3 ++ doc/server/plugins/grouping/ldap.txt | 26 ++++++++++---- src/lib/Bcfg2/Server/Plugins/Ldap.py | 69 +++++++++++++++++++++++++++--------- 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/doc/development/caching.txt b/doc/development/caching.txt index 83ec0290f..c8b7aba14 100644 --- a/doc/development/caching.txt +++ b/doc/development/caching.txt @@ -67,6 +67,9 @@ Currently known caches are: | pkg_sets | `, | | for clients | | | hash of the initial package selection | | | +-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Ldap, | Hostname, ```` | :func:`processed result of the query | Cached results from the Ldap queries | +| results, | | `| | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ These are enumerated so that they can be expired as needed by other plugins or other code points. diff --git a/doc/server/plugins/grouping/ldap.txt b/doc/server/plugins/grouping/ldap.txt index af18680d2..311bab9f5 100644 --- a/doc/server/plugins/grouping/ldap.txt +++ b/doc/server/plugins/grouping/ldap.txt @@ -87,6 +87,26 @@ If you wish, you could customize these values in your ``bcfg2.conf``:: retries = 3 retry_delay = 3.0 +Caching ++++++++ + +This module could not know, if a value changed on the LDAP server. So it does not cache +the results of the LDAP queries by default. + +You could enable the cache of the results in your ``bcfg2.conf``: + + [ldap] + cache = on + +If you enable the caching, you have to expire it manually. This module provides a XML-RPC +method for this purpose: :func:`Ldap.expire_cache +`. + +Even without enabling caching, the results of the LDAP queries are cached, but are +discarded before each client run. If you access the Ldap results of different client, you +may get cached results of the last run of this client. If you do not want this behaviour, +you can disable the caching completely by setting it to ``off``. + Class reference --------------- @@ -251,9 +271,3 @@ Known Issues ------------ * At this point there is no support for SSL/TLS. -* This module could not know, if a value changed on the LDAP server. So it could not - expire the client metadata cache sanely. - If you are using aggressive caching mode, this plugin will expire the metadata cache - for a single client at the start of a client run. If you are using LDAP data from - another client in a template, you will probably get the cached values from the last - client run of that other client. diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 4e66ace5e..f342fba35 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -8,6 +8,7 @@ import traceback from functools import partial import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin from Bcfg2.Logger import Debuggable from Bcfg2.Utils import ClassName, safe_module_name @@ -22,9 +23,10 @@ except ImportError: class ConfigFile(Bcfg2.Server.Plugin.FileBacked): """ Config file for the Ldap plugin """ - def __init__(self, name, core): + def __init__(self, name, core, plugin): Bcfg2.Server.Plugin.FileBacked.__init__(self, name) self.core = core + self.plugin = plugin self.queries = list() self.fam.AddMonitor(name, self) @@ -55,12 +57,15 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked): if self.core.metadata_cache_mode in ['cautious', 'aggressive']: self.core.metadata_cache.expire() + self.plugin.expire_cache() + class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.Connector): """ The Ldap plugin allows adding data from an LDAP server to your metadata. """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] experimental = True @@ -73,7 +78,11 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Options.Option( cf=('ldap', 'retry_delay'), type=float, default=5.0, dest='ldap_retry_delay', - help='The time in seconds betreen retries')] + help='The time in seconds betreen retries'), + Bcfg2.Options.BooleanOption( + cf=('ldap', 'cache'), default=None, dest='ldap_cache', + help='Cache the results of the LDAP Queries until they ' + 'are expired using the XML-RPC RMI')] def __init__(self, core): Bcfg2.Server.Plugin.Plugin.__init__(self, core) @@ -85,21 +94,37 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, raise Bcfg2.Server.Plugin.PluginInitError(msg) self.config = ConfigFile(os.path.join(self.data, 'config.py'), - core) + core, self) + self._hosts = dict() + + def _cache(self, query_name): + """ Return the :class:`Cache ` for the + given query name. """ + return Bcfg2.Server.Cache.Cache('Ldap', 'results', query_name) def _execute_query(self, query, metadata): - try: - self.debug_log("Processing query '%s'" % query.name) - return query.get_result(metadata) - except: # pylint: disable=W0702 - if hasattr(query, "name"): + """ Return the cached result of the given query for this host or + execute the given query and cache the result. """ + result = None + + if Bcfg2.Options.setup.ldap_cache is not False: + cache = self._cache(query.name) + result = cache.get(metadata.hostname, None) + + if result is None: + try: + self.debug_log("Processing query '%s'" % query.name) + result = query.get_result(metadata) + if Bcfg2.Options.setup.ldap_cache is not False: + cache[metadata.hostname] = result + except: # pylint: disable=W0702 self.logger.error( "Exception during processing of query named '%s', query " "results will be empty and may cause bind failures" % query.name) - for line in traceback.format_exc().split('\n'): - self.logger.error(line) - return None + for line in traceback.format_exc().split('\n'): + self.logger.error(line) + return result def get_additional_data(self, metadata): data = {} @@ -114,7 +139,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, else: self.debug_log("query '%s' not applicable to host '%s'" % (query.name, metadata.hostname)) - except: + except: # pylint: disable=W0702 self.logger.error( "Exception during preparation of query named '%s'. " "Query will be ignored." % query_class.__name__) @@ -124,11 +149,21 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, return Bcfg2.Server.Plugin.CallableDict(**data) def start_client_run(self, metadata): - if self.core.metadata_cache_mode == 'aggressive': - self.logger.warning("Ldap is incompatible with aggressive " - "client metadata caching, try 'cautious' " - "or 'initial'") - self.core.metadata_cache.expire(metadata.hostname) + if Bcfg2.Options.setup.ldap_cache is None: + self.expire_cache(hostname=metadata.hostname) + + def expire_cache(self, query=None, hostname=None): + """ Expire the cache. You can select the items to purge + per query and/or per host, or you can purge all cached + data. This is exposed as an XML-RPC RMI. """ + + tags = ['Ldap', 'results'] + if query: + tags.append(query) + if hostname: + tags.append(hostname) + + return Bcfg2.Server.Cache.expire(*tags) class LdapConnection(Debuggable): -- cgit v1.2.3-1-g7c22