Server/Plugins/Ldap: Cache the results of the Ldap queries

Using the OnDemandDict removes the results of Ldap queries from the
client_metadata cache. We add a new cache per hostname cache for the
single ldap queries and add a new configuration option to enable
caching until the cache is expired manually via XML-RPC.

3 files changed, 75 insertions, 23 deletions

diff --git a/doc/development/caching.txt b/doc/development/caching.txt
index 83ec0290f..c8b7aba14 100644
--- a/doc/development/caching.txt
+++ b/doc/development/caching.txt
@@ -67,6 +67,9 @@ Currently known caches are:
 | pkg_sets    | <Collection.cachekey>`,               |                                                 | for clients                                          |
 |             | hash of the initial package selection |                                                 |                                                      |
 +-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+
+| Ldap,       | Hostname, ``<query name>``            | :func:`processed result of the query            | Cached results from the Ldap queries                 |
+| results,    |                                       | <Bcfg2.Server.Plugins.LdapQuery.process_result>`|                                                      |
++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+
 
 These are enumerated so that they can be expired as needed by other
 plugins or other code points.
diff --git a/doc/server/plugins/grouping/ldap.txt b/doc/server/plugins/grouping/ldap.txt
index af18680d2..311bab9f5 100644
--- a/doc/server/plugins/grouping/ldap.txt
+++ b/doc/server/plugins/grouping/ldap.txt
@@ -87,6 +87,26 @@ If you wish, you could customize these values in your ``bcfg2.conf``::
     retries = 3
     retry_delay = 3.0
 
+Caching
++++++++
+
+This module could not know, if a value changed on the LDAP server. So it does not cache
+the results of the LDAP queries by default.
+
+You could enable the cache of the results in your ``bcfg2.conf``:
+
+    [ldap]
+    cache = on
+
+If you enable the caching, you have to expire it manually. This module provides a XML-RPC
+method for this purpose: :func:`Ldap.expire_cache
+<Bcfg2.Server.Plugins.Ldap.expire_cache>`.
+
+Even without enabling caching, the results of the LDAP queries are cached, but are
+discarded before each client run. If you access the Ldap results of different client, you
+may get cached results of the last run of this client. If you do not want this behaviour,
+you can disable the caching completely by setting it to ``off``.
+
 Class reference
 ---------------
 
@@ -251,9 +271,3 @@ Known Issues
 ------------
 
 * At this point there is no support for SSL/TLS.
-* This module could not know, if a value changed on the LDAP server. So it could not
-  expire the client metadata cache sanely.
-  If you are using aggressive caching mode, this plugin will expire the metadata cache
-  for a single client at the start of a client run. If you are using LDAP data from
-  another client in a template, you will probably get the cached values from the last
-  client run of that other client.
diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
index 4e66ace5e..f342fba35 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ldap.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -8,6 +8,7 @@ import traceback
 from functools import partial
 
 import Bcfg2.Options
+import Bcfg2.Server.Cache
 import Bcfg2.Server.Plugin
 from Bcfg2.Logger import Debuggable
 from Bcfg2.Utils import ClassName, safe_module_name
@@ -22,9 +23,10 @@ except ImportError:
 class ConfigFile(Bcfg2.Server.Plugin.FileBacked):
     """ Config file for the Ldap plugin  """
 
-    def __init__(self, name, core):
+    def __init__(self, name, core, plugin):
         Bcfg2.Server.Plugin.FileBacked.__init__(self, name)
         self.core = core
+        self.plugin = plugin
         self.queries = list()
         self.fam.AddMonitor(name, self)
 
@@ -55,12 +57,15 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked):
         if self.core.metadata_cache_mode in ['cautious', 'aggressive']:
             self.core.metadata_cache.expire()
 
+        self.plugin.expire_cache()
+
 
 class Ldap(Bcfg2.Server.Plugin.Plugin,
            Bcfg2.Server.Plugin.ClientRunHooks,
            Bcfg2.Server.Plugin.Connector):
     """ The Ldap plugin allows adding data from an LDAP server
     to your metadata. """
+    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache']
 
     experimental = True
 
@@ -73,7 +78,11 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
         Bcfg2.Options.Option(
             cf=('ldap', 'retry_delay'), type=float, default=5.0,
             dest='ldap_retry_delay',
-            help='The time in seconds betreen retries')]
+            help='The time in seconds betreen retries'),
+        Bcfg2.Options.BooleanOption(
+            cf=('ldap', 'cache'), default=None, dest='ldap_cache',
+            help='Cache the results of the LDAP Queries until they '
+                 'are expired using the XML-RPC RMI')]
 
     def __init__(self, core):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core)
@@ -85,21 +94,37 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
             raise Bcfg2.Server.Plugin.PluginInitError(msg)
 
         self.config = ConfigFile(os.path.join(self.data, 'config.py'),
-                                 core)
+                                 core, self)
+        self._hosts = dict()
+
+    def _cache(self, query_name):
+        """ Return the :class:`Cache <Bcfg2.Server.Cache>` for the
+        given query name. """
+        return Bcfg2.Server.Cache.Cache('Ldap', 'results', query_name)
 
     def _execute_query(self, query, metadata):
-        try:
-            self.debug_log("Processing query '%s'" % query.name)
-            return query.get_result(metadata)
-        except:  # pylint: disable=W0702
-            if hasattr(query, "name"):
+        """ Return the cached result of the given query for this host or
+        execute the given query and cache the result. """
+        result = None
+
+        if Bcfg2.Options.setup.ldap_cache is not False:
+            cache = self._cache(query.name)
+            result = cache.get(metadata.hostname, None)
+
+        if result is None:
+            try:
+                self.debug_log("Processing query '%s'" % query.name)
+                result = query.get_result(metadata)
+                if Bcfg2.Options.setup.ldap_cache is not False:
+                    cache[metadata.hostname] = result
+            except:  # pylint: disable=W0702
                 self.logger.error(
                     "Exception during processing of query named '%s', query "
                     "results will be empty and may cause bind failures" %
                     query.name)
-            for line in traceback.format_exc().split('\n'):
-                self.logger.error(line)
-        return None
+                for line in traceback.format_exc().split('\n'):
+                    self.logger.error(line)
+        return result
 
     def get_additional_data(self, metadata):
         data = {}
@@ -114,7 +139,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
                 else:
                     self.debug_log("query '%s' not applicable to host '%s'" %
                                    (query.name, metadata.hostname))
-            except:
+            except:  # pylint: disable=W0702
                 self.logger.error(
                     "Exception during preparation of query named '%s'. "
                     "Query will be ignored." % query_class.__name__)
@@ -124,11 +149,21 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
         return Bcfg2.Server.Plugin.CallableDict(**data)
 
     def start_client_run(self, metadata):
-        if self.core.metadata_cache_mode == 'aggressive':
-            self.logger.warning("Ldap is incompatible with aggressive "
-                                "client metadata caching, try 'cautious' "
-                                "or 'initial'")
-        self.core.metadata_cache.expire(metadata.hostname)
+        if Bcfg2.Options.setup.ldap_cache is None:
+            self.expire_cache(hostname=metadata.hostname)
+
+    def expire_cache(self, query=None, hostname=None):
+        """ Expire the cache. You can select the items to purge
+        per query and/or per host, or you can purge all cached
+        data. This is exposed as an XML-RPC RMI. """
+
+        tags = ['Ldap', 'results']
+        if query:
+            tags.append(query)
+        if hostname:
+            tags.append(hostname)
+
+        return Bcfg2.Server.Cache.expire(*tags)
 
 
 class LdapConnection(Debuggable):