From fb7d6c605c46aa6373e8b3cf121527acf011b980 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Tue, 12 Jul 2016 00:19:37 +0200
Subject: Server/Plugin: Move OnDemandDict to helpers

The OnDemandDict could be used by different plugins.
---
 src/lib/Bcfg2/Server/Plugin/helpers.py            | 48 ++++++++++++++++++++-
 src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 52 ++---------------------
 2 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 6b521dfd6..17363a675 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -13,7 +13,7 @@ import Bcfg2.Server
 import Bcfg2.Options
 import Bcfg2.Server.FileMonitor
 from Bcfg2.Logger import Debuggable
-from Bcfg2.Compat import CmpMixin, wraps
+from Bcfg2.Compat import CmpMixin, MutableMapping, wraps
 from Bcfg2.Server.Plugin.base import Plugin
 from Bcfg2.Server.Plugin.interfaces import Generator, TemplateDataProvider
 from Bcfg2.Server.Plugin.exceptions import SpecificityError, \
@@ -1698,3 +1698,49 @@ class GroupSpool(Plugin, Generator):
                 return
             reqid = self.fam.AddMonitor(name, self)
             self.handles[reqid] = relative
+
+
+class OnDemandDict(MutableMapping):
+    """ This maps a set of keys to a set of value-getting functions;
+    the values are populated on-the-fly by the functions as the values
+    are needed (and not before).  This is for example used by
+    :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`;
+    see the docstring for that function for details on why.
+
+    Unlike a dict, you should not specify values for the righthand
+    side of this mapping, but functions that get values.  E.g.:
+
+    .. code-block:: python
+
+        d = OnDemandDict(foo=load_foo,
+                         bar=lambda: "bar");
+    """
+
+    def __init__(self, **getters):
+        self._values = dict()
+        self._getters = dict(**getters)
+
+    def __getitem__(self, key):
+        if key not in self._values:
+            self._values[key] = self._getters[key]()
+        return self._values[key]
+
+    def __setitem__(self, key, getter):
+        self._getters[key] = getter
+
+    def __delitem__(self, key):
+        del self._values[key]
+        del self._getters[key]
+
+    def __len__(self):
+        return len(self._getters)
+
+    def __iter__(self):
+        return iter(self._getters.keys())
+
+    def __repr__(self):
+        rv = dict(self._values)
+        for key in self._getters.keys():
+            if key not in rv:
+                rv[key] = 'unknown'
+        return str(rv)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 3aa5c415f..23ccd7b8e 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -10,7 +10,7 @@ import lxml.etree
 import Bcfg2.Options
 import Bcfg2.Server.Cache
 import Bcfg2.Server.Plugin
-from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping
+from Bcfg2.Compat import urlopen, HTTPError, URLError
 from Bcfg2.Server.Plugins.Packages.Collection import Collection, \
     get_collection_class
 from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
@@ -36,52 +36,6 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction):
     fail_silently = True
 
 
-class OnDemandDict(MutableMapping):
-    """ This maps a set of keys to a set of value-getting functions;
-    the values are populated on-the-fly by the functions as the values
-    are needed (and not before).  This is used by
-    :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`;
-    see the docstring for that function for details on why.
-
-    Unlike a dict, you should not specify values for for the righthand
-    side of this mapping, but functions that get values.  E.g.:
-
-    .. code-block:: python
-
-        d = OnDemandDict(foo=load_foo,
-                         bar=lambda: "bar");
-    """
-
-    def __init__(self, **getters):
-        self._values = dict()
-        self._getters = dict(**getters)
-
-    def __getitem__(self, key):
-        if key not in self._values:
-            self._values[key] = self._getters[key]()
-        return self._values[key]
-
-    def __setitem__(self, key, getter):
-        self._getters[key] = getter
-
-    def __delitem__(self, key):
-        del self._values[key]
-        del self._getters[key]
-
-    def __len__(self):
-        return len(self._getters)
-
-    def __iter__(self):
-        return iter(self._getters.keys())
-
-    def __repr__(self):
-        rv = dict(self._values)
-        for key in self._getters.keys():
-            if key not in rv:
-                rv[key] = 'unknown'
-        return str(rv)
-
-
 class Packages(Bcfg2.Server.Plugin.Plugin,
                Bcfg2.Server.Plugin.StructureValidator,
                Bcfg2.Server.Plugin.Generator,
@@ -578,7 +532,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
 
     def get_additional_data(self, metadata):
         """ Return additional data for the given client.  This will be
-        an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict`
+        an :class:`Bcfg2.Server.Plugin.OnDemandDict`
         containing two keys:
 
         * ``sources``, whose value is a list of data returned from
@@ -610,7 +564,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             get_collection() until it's absolutely necessary. """
             return self.get_collection(metadata).get_additional_data()
 
-        return OnDemandDict(
+        return Bcfg2.Server.Plugin.OnDemandDict(
             sources=get_sources,
             get_config=lambda: self.get_config)
 
-- 
cgit v1.2.3-1-g7c22


From 257ba8d92eda8be0a347f3d68b174a2354782578 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Tue, 12 Jul 2016 00:24:23 +0200
Subject: Server/Plugin: Support functions and values for OnDemandDict

Now you can also specify simple values for the OnDemandDict.
---
 src/lib/Bcfg2/Server/Plugin/helpers.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 17363a675..9ef4a2527 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -1707,13 +1707,14 @@ class OnDemandDict(MutableMapping):
     :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`;
     see the docstring for that function for details on why.
 
-    Unlike a dict, you should not specify values for the righthand
-    side of this mapping, but functions that get values.  E.g.:
+    Unlike a dict, you can specify values or functions for the
+    righthand side of this mapping. If you specify a function, it will
+    be evaluated, when you access the value for the first time. E.g.:
 
     .. code-block:: python
 
         d = OnDemandDict(foo=load_foo,
-                         bar=lambda: "bar");
+                         bar="bar")
     """
 
     def __init__(self, **getters):
@@ -1722,7 +1723,11 @@ class OnDemandDict(MutableMapping):
 
     def __getitem__(self, key):
         if key not in self._values:
-            self._values[key] = self._getters[key]()
+            if callable(self._getters[key]):
+                self._values[key] = self._getters[key]()
+            else:
+                self._values[key] = self._getters[key]
+
         return self._values[key]
 
     def __setitem__(self, key, getter):
-- 
cgit v1.2.3-1-g7c22


From 9af8a7ea4b1de56ac6b8800277518e9159fb7dca Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Fri, 15 Jul 2016 20:27:19 +0200
Subject: Server/Plugin: CallableDict is an OnDemandDict without caching

Add a CallableDict (like OnDemandDict, but without caching the results)
to have a dict-like class, that can be cached with the client metadata
without caching the dynamic values.
---
 src/lib/Bcfg2/Server/Plugin/helpers.py | 62 +++++++++++++++++++++++++---------
 1 file changed, 46 insertions(+), 16 deletions(-)

diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 9ef4a2527..762d018eb 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -1700,7 +1700,7 @@ class GroupSpool(Plugin, Generator):
             self.handles[reqid] = relative
 
 
-class OnDemandDict(MutableMapping):
+class CallableDict(MutableMapping):
     """ This maps a set of keys to a set of value-getting functions;
     the values are populated on-the-fly by the functions as the values
     are needed (and not before).  This is for example used by
@@ -1709,32 +1709,27 @@ class OnDemandDict(MutableMapping):
 
     Unlike a dict, you can specify values or functions for the
     righthand side of this mapping. If you specify a function, it will
-    be evaluated, when you access the value for the first time. E.g.:
+    be evaluated everytime you access the value. E.g.:
 
     .. code-block:: python
 
-        d = OnDemandDict(foo=load_foo,
+        d = CallableDict(foo=load_foo,
                          bar="bar")
     """
 
     def __init__(self, **getters):
-        self._values = dict()
         self._getters = dict(**getters)
 
     def __getitem__(self, key):
-        if key not in self._values:
-            if callable(self._getters[key]):
-                self._values[key] = self._getters[key]()
-            else:
-                self._values[key] = self._getters[key]
-
-        return self._values[key]
+        if callable(self._getters[key]):
+            return self._getters[key]()
+        else:
+            return self._getters[key]
 
     def __setitem__(self, key, getter):
         self._getters[key] = getter
 
     def __delitem__(self, key):
-        del self._values[key]
         del self._getters[key]
 
     def __len__(self):
@@ -1743,9 +1738,44 @@ class OnDemandDict(MutableMapping):
     def __iter__(self):
         return iter(self._getters.keys())
 
-    def __repr__(self):
-        rv = dict(self._values)
+    def _current_data(self):
+        """ Return a dict with the current available static data
+        and ``unknown`` for all callable values.
+        """
+        rv = dict()
         for key in self._getters.keys():
-            if key not in rv:
+            if callable(self._getters[key]):
                 rv[key] = 'unknown'
-        return str(rv)
+            else:
+                rv[key] = self._getters[key]
+        return rv
+
+    def __repr__(self):
+        return str(self._current_data())
+
+
+class OnDemandDict(CallableDict):
+    """ This is like a :class:`CallableDict` but it will cache
+    the results of the callable getters, so that it is only evaluated
+    once when you first access it.
+    """
+
+    def __init__(self, **getters):
+        CallableDict.__init__(self, **getters)
+        self._values = dict()
+
+    def __getitem__(self, key):
+        if key not in self._values:
+            self._values[key] = super(OnDemandDict, self).__getitem__(key)
+        return self._values[key]
+
+    def __delitem__(self, key):
+        super(OnDemandDict, self).__delitem__(key)
+        del self._values[key]
+
+    def _current_data(self):
+        rv = super(OnDemandDict, self)._current_data()
+        for (key, value) in rv.items():
+            if key in self._values:
+                rv[key] = value
+        return rv
-- 
cgit v1.2.3-1-g7c22


From 99f1110d43e00654eb06c35495871da253c21e5b Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Tue, 12 Jul 2016 00:51:18 +0200
Subject: Server/Plugins/Ldap: Use CallableDict

With the CallableDict the LdapQueries will only be executed, if the values
are used.
---
 src/lib/Bcfg2/Server/Plugins/Ldap.py | 42 ++++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
index 757150300..81077508c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ldap.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -5,6 +5,8 @@ import os
 import sys
 import time
 import traceback
+from functools import partial
+
 import Bcfg2.Options
 import Bcfg2.Server.Plugin
 from Bcfg2.Logger import Debuggable
@@ -84,20 +86,10 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
 
         self.config = ConfigFile(os.path.join(self.data, 'config.py'))
 
-    def get_additional_data(self, metadata):
-        query = None
+    def _execute_query(self, query, metadata):
         try:
-            data = {}
-            self.debug_log("Found queries %s" % self.config.queries)
-            for query_class in self.config.queries:
-                query = query_class()
-                if query.is_applicable(metadata):
-                    self.debug_log("Processing query '%s'" % query.name)
-                    data[query.name] = query.get_result(metadata)
-                else:
-                    self.debug_log("query '%s' not applicable to host '%s'" %
-                                   (query.name, metadata.hostname))
-            return data
+            self.debug_log("Processing query '%s'" % query.name)
+            return query.get_result(metadata)
         except:  # pylint: disable=W0702
             if hasattr(query, "name"):
                 self.logger.error(
@@ -106,7 +98,29 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
                     query.name)
             for line in traceback.format_exc().split('\n'):
                 self.logger.error(line)
-            return {}
+        return None
+
+    def get_additional_data(self, metadata):
+        data = {}
+        self.debug_log("Found queries %s" % self.config.queries)
+        for query_class in self.config.queries:
+            try:
+                query = query_class()
+                if query.is_applicable(metadata):
+                    self.debug_log("Processing query '%s'" % query.name)
+                    data[query.name] = partial(
+                        self._execute_query, query, metadata)
+                else:
+                    self.debug_log("query '%s' not applicable to host '%s'" %
+                                   (query.name, metadata.hostname))
+            except:
+                self.logger.error(
+                    "Exception during preparation of query named '%s'. "
+                    "Query will be ignored." % query_class.__name__)
+                for line in traceback.format_exc().split('\n'):
+                    self.logger.error(line)
+
+        return Bcfg2.Server.Plugin.CallableDict(**data)
 
     def start_client_run(self, metadata):
         if self.core.metadata_cache_mode == 'aggressive':
-- 
cgit v1.2.3-1-g7c22


From e28947abe9042e33586924c4daa2b1c09ffe4782 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Tue, 12 Jul 2016 02:43:23 +0200
Subject: Server/Cache: Return the number of expired items

---
 src/lib/Bcfg2/Server/Cache.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/lib/Bcfg2/Server/Cache.py b/src/lib/Bcfg2/Server/Cache.py
index d05eb0bf6..b3b906b2c 100644
--- a/src/lib/Bcfg2/Server/Cache.py
+++ b/src/lib/Bcfg2/Server/Cache.py
@@ -96,15 +96,19 @@ class _Cache(MutableMapping):
         return len(list(iter(self)))
 
     def expire(self, key=None):
-        """ expire all items, or a specific item, from the cache """
+        """ expire all items, or a specific item, from the cache
+
+        :returns: number of expired entries
+        """
+
         if key is None:
-            expire(*self._tags)
+            return expire(*self._tags)
         else:
             tags = self._tags | set([key])
             # py 2.5 doesn't support mixing *args and explicit keyword
             # args
             kwargs = dict(exact=True)
-            expire(*tags, **kwargs)
+            return expire(*tags, **kwargs)
 
     def __repr__(self):
         return repr(dict(self))
@@ -152,7 +156,10 @@ def expire(*tags, **kwargs):
     """ Expire all items, a set of items, or one specific item from
     the cache.  If ``exact`` is set to True, then if the given tag set
     doesn't match exactly one item in the cache, nothing will be
-    expired. """
+    expired.
+
+    :returns: number of expired entries
+    """
     exact = kwargs.pop("exact", False)
     count = 0
     if not tags:
@@ -170,6 +177,8 @@ def expire(*tags, **kwargs):
     for hook in _hooks:
         hook(tags, exact, count)
 
+    return count
+
 
 def add_expire_hook(func):
     """ Add a hook that will be called when an item is expired from
-- 
cgit v1.2.3-1-g7c22


From 3909b0e2897f56b1b50d66c0bde76fcaa111ce25 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Tue, 12 Jul 2016 03:58:44 +0200
Subject: Server/Plugins/Ldap: Add missing argument

---
 src/lib/Bcfg2/Server/Plugins/Ldap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
index 81077508c..4e66ace5e 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ldap.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -84,7 +84,8 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
             self.logger.error(msg)
             raise Bcfg2.Server.Plugin.PluginInitError(msg)
 
-        self.config = ConfigFile(os.path.join(self.data, 'config.py'))
+        self.config = ConfigFile(os.path.join(self.data, 'config.py'),
+                                 core)
 
     def _execute_query(self, query, metadata):
         try:
-- 
cgit v1.2.3-1-g7c22


From 66c272c383c52343b5a201ab59ca2e0e1ee8ee2c Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Tue, 12 Jul 2016 02:51:18 +0200
Subject: Server/Plugins/Ldap: Cache the results of the Ldap queries

Using the OnDemandDict removes the results of Ldap queries from the
client_metadata cache. We add a new cache per hostname cache for the
single ldap queries and add a new configuration option to enable
caching until the cache is expired manually via XML-RPC.
---
 doc/development/caching.txt          |  3 ++
 doc/server/plugins/grouping/ldap.txt | 26 ++++++++++----
 src/lib/Bcfg2/Server/Plugins/Ldap.py | 69 +++++++++++++++++++++++++++---------
 3 files changed, 75 insertions(+), 23 deletions(-)

diff --git a/doc/development/caching.txt b/doc/development/caching.txt
index 83ec0290f..c8b7aba14 100644
--- a/doc/development/caching.txt
+++ b/doc/development/caching.txt
@@ -67,6 +67,9 @@ Currently known caches are:
 | pkg_sets    | <Collection.cachekey>`,               |                                                 | for clients                                          |
 |             | hash of the initial package selection |                                                 |                                                      |
 +-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+
+| Ldap,       | Hostname, ``<query name>``            | :func:`processed result of the query            | Cached results from the Ldap queries                 |
+| results,    |                                       | <Bcfg2.Server.Plugins.LdapQuery.process_result>`|                                                      |
++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+
 
 These are enumerated so that they can be expired as needed by other
 plugins or other code points.
diff --git a/doc/server/plugins/grouping/ldap.txt b/doc/server/plugins/grouping/ldap.txt
index af18680d2..311bab9f5 100644
--- a/doc/server/plugins/grouping/ldap.txt
+++ b/doc/server/plugins/grouping/ldap.txt
@@ -87,6 +87,26 @@ If you wish, you could customize these values in your ``bcfg2.conf``::
     retries = 3
     retry_delay = 3.0
 
+Caching
++++++++
+
+This module could not know, if a value changed on the LDAP server. So it does not cache
+the results of the LDAP queries by default.
+
+You could enable the cache of the results in your ``bcfg2.conf``:
+
+    [ldap]
+    cache = on
+
+If you enable the caching, you have to expire it manually. This module provides a XML-RPC
+method for this purpose: :func:`Ldap.expire_cache
+<Bcfg2.Server.Plugins.Ldap.expire_cache>`.
+
+Even without enabling caching, the results of the LDAP queries are cached, but are
+discarded before each client run. If you access the Ldap results of different client, you
+may get cached results of the last run of this client. If you do not want this behaviour,
+you can disable the caching completely by setting it to ``off``.
+
 Class reference
 ---------------
 
@@ -251,9 +271,3 @@ Known Issues
 ------------
 
 * At this point there is no support for SSL/TLS.
-* This module could not know, if a value changed on the LDAP server. So it could not
-  expire the client metadata cache sanely.
-  If you are using aggressive caching mode, this plugin will expire the metadata cache
-  for a single client at the start of a client run. If you are using LDAP data from
-  another client in a template, you will probably get the cached values from the last
-  client run of that other client.
diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
index 4e66ace5e..f342fba35 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ldap.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -8,6 +8,7 @@ import traceback
 from functools import partial
 
 import Bcfg2.Options
+import Bcfg2.Server.Cache
 import Bcfg2.Server.Plugin
 from Bcfg2.Logger import Debuggable
 from Bcfg2.Utils import ClassName, safe_module_name
@@ -22,9 +23,10 @@ except ImportError:
 class ConfigFile(Bcfg2.Server.Plugin.FileBacked):
     """ Config file for the Ldap plugin  """
 
-    def __init__(self, name, core):
+    def __init__(self, name, core, plugin):
         Bcfg2.Server.Plugin.FileBacked.__init__(self, name)
         self.core = core
+        self.plugin = plugin
         self.queries = list()
         self.fam.AddMonitor(name, self)
 
@@ -55,12 +57,15 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked):
         if self.core.metadata_cache_mode in ['cautious', 'aggressive']:
             self.core.metadata_cache.expire()
 
+        self.plugin.expire_cache()
+
 
 class Ldap(Bcfg2.Server.Plugin.Plugin,
            Bcfg2.Server.Plugin.ClientRunHooks,
            Bcfg2.Server.Plugin.Connector):
     """ The Ldap plugin allows adding data from an LDAP server
     to your metadata. """
+    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache']
 
     experimental = True
 
@@ -73,7 +78,11 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
         Bcfg2.Options.Option(
             cf=('ldap', 'retry_delay'), type=float, default=5.0,
             dest='ldap_retry_delay',
-            help='The time in seconds betreen retries')]
+            help='The time in seconds betreen retries'),
+        Bcfg2.Options.BooleanOption(
+            cf=('ldap', 'cache'), default=None, dest='ldap_cache',
+            help='Cache the results of the LDAP Queries until they '
+                 'are expired using the XML-RPC RMI')]
 
     def __init__(self, core):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core)
@@ -85,21 +94,37 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
             raise Bcfg2.Server.Plugin.PluginInitError(msg)
 
         self.config = ConfigFile(os.path.join(self.data, 'config.py'),
-                                 core)
+                                 core, self)
+        self._hosts = dict()
+
+    def _cache(self, query_name):
+        """ Return the :class:`Cache <Bcfg2.Server.Cache>` for the
+        given query name. """
+        return Bcfg2.Server.Cache.Cache('Ldap', 'results', query_name)
 
     def _execute_query(self, query, metadata):
-        try:
-            self.debug_log("Processing query '%s'" % query.name)
-            return query.get_result(metadata)
-        except:  # pylint: disable=W0702
-            if hasattr(query, "name"):
+        """ Return the cached result of the given query for this host or
+        execute the given query and cache the result. """
+        result = None
+
+        if Bcfg2.Options.setup.ldap_cache is not False:
+            cache = self._cache(query.name)
+            result = cache.get(metadata.hostname, None)
+
+        if result is None:
+            try:
+                self.debug_log("Processing query '%s'" % query.name)
+                result = query.get_result(metadata)
+                if Bcfg2.Options.setup.ldap_cache is not False:
+                    cache[metadata.hostname] = result
+            except:  # pylint: disable=W0702
                 self.logger.error(
                     "Exception during processing of query named '%s', query "
                     "results will be empty and may cause bind failures" %
                     query.name)
-            for line in traceback.format_exc().split('\n'):
-                self.logger.error(line)
-        return None
+                for line in traceback.format_exc().split('\n'):
+                    self.logger.error(line)
+        return result
 
     def get_additional_data(self, metadata):
         data = {}
@@ -114,7 +139,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
                 else:
                     self.debug_log("query '%s' not applicable to host '%s'" %
                                    (query.name, metadata.hostname))
-            except:
+            except:  # pylint: disable=W0702
                 self.logger.error(
                     "Exception during preparation of query named '%s'. "
                     "Query will be ignored." % query_class.__name__)
@@ -124,11 +149,21 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
         return Bcfg2.Server.Plugin.CallableDict(**data)
 
     def start_client_run(self, metadata):
-        if self.core.metadata_cache_mode == 'aggressive':
-            self.logger.warning("Ldap is incompatible with aggressive "
-                                "client metadata caching, try 'cautious' "
-                                "or 'initial'")
-        self.core.metadata_cache.expire(metadata.hostname)
+        if Bcfg2.Options.setup.ldap_cache is None:
+            self.expire_cache(hostname=metadata.hostname)
+
+    def expire_cache(self, query=None, hostname=None):
+        """ Expire the cache. You can select the items to purge
+        per query and/or per host, or you can purge all cached
+        data. This is exposed as an XML-RPC RMI. """
+
+        tags = ['Ldap', 'results']
+        if query:
+            tags.append(query)
+        if hostname:
+            tags.append(hostname)
+
+        return Bcfg2.Server.Cache.expire(*tags)
 
 
 class LdapConnection(Debuggable):
-- 
cgit v1.2.3-1-g7c22


From 5f2daf138aab3a993c182797dc3ca2049f6bd7af Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Fri, 15 Jul 2016 17:26:54 +0200
Subject: Server/Plugins/Ldap: Support specifying the ldap uri

You can now specify the server to connect by either host (and optionally
port) or by specifying the full ldap uri. If you specify host and port
the connection will use the plain (unencrypted) ldap protocol by default.
Only if you specify the port "636", it will use ldaps now.
---
 doc/server/plugins/grouping/ldap.txt | 19 ++++++++++---------
 src/lib/Bcfg2/Server/Plugins/Ldap.py | 18 ++++++++++++------
 2 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/doc/server/plugins/grouping/ldap.txt b/doc/server/plugins/grouping/ldap.txt
index 311bab9f5..f9c25dee9 100644
--- a/doc/server/plugins/grouping/ldap.txt
+++ b/doc/server/plugins/grouping/ldap.txt
@@ -7,7 +7,7 @@ Ldap
 ====
 
 .. warning::
-    This plugin is considered experimental and has known issues (see below).
+    This plugin is considered experimental.
 
 Purpose
 -------
@@ -115,8 +115,8 @@ LdapConnection
 
 .. class:: LdapConnection
 
-   This class represents an LDAP connection. Every query must be associated with exactly
-   one connection.
+   This class represents an LDAP connection. Every query must be associated
+   with exactly one connection.
 
 .. attribute:: LdapConnection.binddn
 
@@ -132,7 +132,13 @@ LdapConnection
 
 .. attribute:: LdapConnection.port
 
-   Port where LDAP server is listening (defaults to 389).
+   Port where LDAP server is listening (defaults to 389). If you use
+   port 636 this module will use ldaps to connect to the server.
+
+.. attribute:: LdapConnection.uri
+
+   LDAP URI of the LDAP server to connect to. This is prefered over
+   :attr:`LdapConnection.host` and :attr:`LdapConnection.port`.
 
 You may pass any of these attributes as keyword arguments when creating the connection object.
 
@@ -266,8 +272,3 @@ search below that DN.
 You do not need to add all LdapQueries to the ``__queries__`` list. Only add those to
 that list, that should be called automatically and whose results should be added to the
 client metadata.
-
-Known Issues
-------------
-
-* At this point there is no support for SSL/TLS.
diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
index f342fba35..0b66f7777 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ldap.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -169,7 +169,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
 class LdapConnection(Debuggable):
     """ Connection to an LDAP server. """
 
-    def __init__(self, host="localhost", port=389, binddn=None,
+    def __init__(self, host="localhost", port=389, uri=None, binddn=None,
                  bindpw=None):
         Debuggable.__init__(self)
 
@@ -180,6 +180,7 @@ class LdapConnection(Debuggable):
 
         self.host = host
         self.port = port
+        self.uri = uri
         self.binddn = binddn
         self.bindpw = bindpw
         self.conn = None
@@ -204,7 +205,8 @@ class LdapConnection(Debuggable):
         """ Open a connection to the configured LDAP server, and do a simple
         bind ff both binddn and bindpw are set. """
         self.disconnect()
-        self.conn = ldap.initialize(self.url)
+        self.conn = ldap.initialize(self.get_uri())
+
         if self.binddn is not None and self.bindpw is not None:
             self.conn.simple_bind_s(self.binddn, self.bindpw)
 
@@ -228,16 +230,20 @@ class LdapConnection(Debuggable):
                 self.conn = None
                 self.logger.error(
                     "LdapConnection: Server %s down. Retry %d/%d in %.2fs." %
-                    (self.url, attempt + 1, Bcfg2.Options.setup.ldap_retries,
+                    (self.get_uri(), attempt + 1,
+                     Bcfg2.Options.setup.ldap_retries,
                      Bcfg2.Options.setup.ldap_retry_delay))
                 time.sleep(Bcfg2.Options.setup.ldap_retry_delay)
 
         return None
 
-    @property
-    def url(self):
+    def get_uri(self):
         """ The URL of the LDAP server. """
-        return "ldap://%s:%d" % (self.host, self.port)
+        if self.uri is None:
+            if self.port == 636:
+                return "ldaps://%s" % self.host
+            return "ldap://%s:%d" % (self.host, self.port)
+        return self.uri
 
 
 class LdapQuery(object):
-- 
cgit v1.2.3-1-g7c22


From b914052d7c33cc45012f693763189aa7db7a78a2 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Fri, 15 Jul 2016 17:31:23 +0200
Subject: Server/Plugins/Ldap: Support arbitrary ldap options

You can now set arbitrary ldap option for the connection by specifying
a dict with the key and the value. You should use the constants from
python-ldap.
---
 doc/server/plugins/grouping/ldap.txt | 11 +++++++++++
 src/lib/Bcfg2/Server/Plugins/Ldap.py |  9 +++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/doc/server/plugins/grouping/ldap.txt b/doc/server/plugins/grouping/ldap.txt
index f9c25dee9..abbd5e005 100644
--- a/doc/server/plugins/grouping/ldap.txt
+++ b/doc/server/plugins/grouping/ldap.txt
@@ -140,6 +140,17 @@ LdapConnection
    LDAP URI of the LDAP server to connect to. This is prefered over
    :attr:`LdapConnection.host` and :attr:`LdapConnection.port`.
 
+   .. note::
+
+      If you are using ldaps you may have to specify additional options
+      for enabling the certificate validation or setting the path for
+      the trusted certificates with :attr:`LdapConnection.options`.
+
+.. attribute:: LdapConnection.options
+
+   Arbitrary options for the LDAP connection. You should specify it
+   as a dict and use the ``OPT_*`` constants from ``python-ldap``.
+
 You may pass any of these attributes as keyword arguments when creating the connection object.
 
 LdapQuery
diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
index 0b66f7777..a51f47dae 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ldap.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -169,8 +169,8 @@ class Ldap(Bcfg2.Server.Plugin.Plugin,
 class LdapConnection(Debuggable):
     """ Connection to an LDAP server. """
 
-    def __init__(self, host="localhost", port=389, uri=None, binddn=None,
-                 bindpw=None):
+    def __init__(self, host="localhost", port=389, uri=None, options=None,
+                 binddn=None, bindpw=None):
         Debuggable.__init__(self)
 
         if HAS_LDAP:
@@ -181,6 +181,7 @@ class LdapConnection(Debuggable):
         self.host = host
         self.port = port
         self.uri = uri
+        self.options = options
         self.binddn = binddn
         self.bindpw = bindpw
         self.conn = None
@@ -207,6 +208,10 @@ class LdapConnection(Debuggable):
         self.disconnect()
         self.conn = ldap.initialize(self.get_uri())
 
+        if self.options is not None:
+            for (option, value) in self.options.items():
+                self.conn.set_option(option, value)
+
         if self.binddn is not None and self.bindpw is not None:
             self.conn.simple_bind_s(self.binddn, self.bindpw)
 
-- 
cgit v1.2.3-1-g7c22


From 0985c2aed06c14d8b79805d21449f2f1d31dd20c Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander.sulfrian@fu-berlin.de>
Date: Wed, 17 Aug 2016 03:28:20 +0200
Subject: Server/Plugins/Ldap: Fix module name

If the module name contains slashes, python will issue an warning:

> Ldap/config.py:1: RuntimeWarning: Parent module '__Ldap_/root/repo/Ldap/config' not found while handling absolute import
>   from Bcfg2.Server.Plugins.Ldap import LdapConnection, LdapQuery

So we simply use the basename without the file extension for the module name.
---
 src/lib/Bcfg2/Server/Plugins/Ldap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py
index a51f47dae..770419ba5 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ldap.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py
@@ -33,7 +33,8 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked):
     def Index(self):
         """ Get the queries from the config file """
         try:
-            module = imp.load_source(safe_module_name('Ldap', self.name),
+            module_name = os.path.splitext(os.path.basename(self.name))[0]
+            module = imp.load_source(safe_module_name('Ldap', module_name),
                                      self.name)
         except:  # pylint: disable=W0702
             err = sys.exc_info()[1]
-- 
cgit v1.2.3-1-g7c22