From 99f1110d43e00654eb06c35495871da253c21e5b Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 12 Jul 2016 00:51:18 +0200 Subject: Server/Plugins/Ldap: Use CallableDict With the CallableDict the LdapQueries will only be executed, if the values are used. --- src/lib/Bcfg2/Server/Plugins/Ldap.py | 42 ++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Ldap.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 757150300..81077508c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -5,6 +5,8 @@ import os import sys import time import traceback +from functools import partial + import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Logger import Debuggable @@ -84,20 +86,10 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, self.config = ConfigFile(os.path.join(self.data, 'config.py')) - def get_additional_data(self, metadata): - query = None + def _execute_query(self, query, metadata): try: - data = {} - self.debug_log("Found queries %s" % self.config.queries) - for query_class in self.config.queries: - query = query_class() - if query.is_applicable(metadata): - self.debug_log("Processing query '%s'" % query.name) - data[query.name] = query.get_result(metadata) - else: - self.debug_log("query '%s' not applicable to host '%s'" % - (query.name, metadata.hostname)) - return data + self.debug_log("Processing query '%s'" % query.name) + return query.get_result(metadata) except: # pylint: disable=W0702 if hasattr(query, "name"): self.logger.error( @@ -106,7 +98,29 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, query.name) for line in traceback.format_exc().split('\n'): self.logger.error(line) - return {} + return None + + def get_additional_data(self, metadata): + data = {} + self.debug_log("Found queries %s" % self.config.queries) + for query_class in self.config.queries: + try: + query = query_class() + if query.is_applicable(metadata): + self.debug_log("Processing query '%s'" % query.name) + data[query.name] = partial( + self._execute_query, query, metadata) + else: + self.debug_log("query '%s' not applicable to host '%s'" % + (query.name, metadata.hostname)) + except: + self.logger.error( + "Exception during preparation of query named '%s'. " + "Query will be ignored." % query_class.__name__) + for line in traceback.format_exc().split('\n'): + self.logger.error(line) + + return Bcfg2.Server.Plugin.CallableDict(**data) def start_client_run(self, metadata): if self.core.metadata_cache_mode == 'aggressive': -- cgit v1.2.3-1-g7c22 From 3909b0e2897f56b1b50d66c0bde76fcaa111ce25 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 12 Jul 2016 03:58:44 +0200 Subject: Server/Plugins/Ldap: Add missing argument --- src/lib/Bcfg2/Server/Plugins/Ldap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Ldap.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 81077508c..4e66ace5e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -84,7 +84,8 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) - self.config = ConfigFile(os.path.join(self.data, 'config.py')) + self.config = ConfigFile(os.path.join(self.data, 'config.py'), + core) def _execute_query(self, query, metadata): try: -- cgit v1.2.3-1-g7c22 From 66c272c383c52343b5a201ab59ca2e0e1ee8ee2c Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 12 Jul 2016 02:51:18 +0200 Subject: Server/Plugins/Ldap: Cache the results of the Ldap queries Using the OnDemandDict removes the results of Ldap queries from the client_metadata cache. We add a new cache per hostname cache for the single ldap queries and add a new configuration option to enable caching until the cache is expired manually via XML-RPC. --- src/lib/Bcfg2/Server/Plugins/Ldap.py | 69 +++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 17 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Ldap.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 4e66ace5e..f342fba35 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -8,6 +8,7 @@ import traceback from functools import partial import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin from Bcfg2.Logger import Debuggable from Bcfg2.Utils import ClassName, safe_module_name @@ -22,9 +23,10 @@ except ImportError: class ConfigFile(Bcfg2.Server.Plugin.FileBacked): """ Config file for the Ldap plugin """ - def __init__(self, name, core): + def __init__(self, name, core, plugin): Bcfg2.Server.Plugin.FileBacked.__init__(self, name) self.core = core + self.plugin = plugin self.queries = list() self.fam.AddMonitor(name, self) @@ -55,12 +57,15 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked): if self.core.metadata_cache_mode in ['cautious', 'aggressive']: self.core.metadata_cache.expire() + self.plugin.expire_cache() + class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.Connector): """ The Ldap plugin allows adding data from an LDAP server to your metadata. """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] experimental = True @@ -73,7 +78,11 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Options.Option( cf=('ldap', 'retry_delay'), type=float, default=5.0, dest='ldap_retry_delay', - help='The time in seconds betreen retries')] + help='The time in seconds betreen retries'), + Bcfg2.Options.BooleanOption( + cf=('ldap', 'cache'), default=None, dest='ldap_cache', + help='Cache the results of the LDAP Queries until they ' + 'are expired using the XML-RPC RMI')] def __init__(self, core): Bcfg2.Server.Plugin.Plugin.__init__(self, core) @@ -85,21 +94,37 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, raise Bcfg2.Server.Plugin.PluginInitError(msg) self.config = ConfigFile(os.path.join(self.data, 'config.py'), - core) + core, self) + self._hosts = dict() + + def _cache(self, query_name): + """ Return the :class:`Cache ` for the + given query name. """ + return Bcfg2.Server.Cache.Cache('Ldap', 'results', query_name) def _execute_query(self, query, metadata): - try: - self.debug_log("Processing query '%s'" % query.name) - return query.get_result(metadata) - except: # pylint: disable=W0702 - if hasattr(query, "name"): + """ Return the cached result of the given query for this host or + execute the given query and cache the result. """ + result = None + + if Bcfg2.Options.setup.ldap_cache is not False: + cache = self._cache(query.name) + result = cache.get(metadata.hostname, None) + + if result is None: + try: + self.debug_log("Processing query '%s'" % query.name) + result = query.get_result(metadata) + if Bcfg2.Options.setup.ldap_cache is not False: + cache[metadata.hostname] = result + except: # pylint: disable=W0702 self.logger.error( "Exception during processing of query named '%s', query " "results will be empty and may cause bind failures" % query.name) - for line in traceback.format_exc().split('\n'): - self.logger.error(line) - return None + for line in traceback.format_exc().split('\n'): + self.logger.error(line) + return result def get_additional_data(self, metadata): data = {} @@ -114,7 +139,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, else: self.debug_log("query '%s' not applicable to host '%s'" % (query.name, metadata.hostname)) - except: + except: # pylint: disable=W0702 self.logger.error( "Exception during preparation of query named '%s'. " "Query will be ignored." % query_class.__name__) @@ -124,11 +149,21 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, return Bcfg2.Server.Plugin.CallableDict(**data) def start_client_run(self, metadata): - if self.core.metadata_cache_mode == 'aggressive': - self.logger.warning("Ldap is incompatible with aggressive " - "client metadata caching, try 'cautious' " - "or 'initial'") - self.core.metadata_cache.expire(metadata.hostname) + if Bcfg2.Options.setup.ldap_cache is None: + self.expire_cache(hostname=metadata.hostname) + + def expire_cache(self, query=None, hostname=None): + """ Expire the cache. You can select the items to purge + per query and/or per host, or you can purge all cached + data. This is exposed as an XML-RPC RMI. """ + + tags = ['Ldap', 'results'] + if query: + tags.append(query) + if hostname: + tags.append(hostname) + + return Bcfg2.Server.Cache.expire(*tags) class LdapConnection(Debuggable): -- cgit v1.2.3-1-g7c22 From 5f2daf138aab3a993c182797dc3ca2049f6bd7af Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 15 Jul 2016 17:26:54 +0200 Subject: Server/Plugins/Ldap: Support specifying the ldap uri You can now specify the server to connect by either host (and optionally port) or by specifying the full ldap uri. If you specify host and port the connection will use the plain (unencrypted) ldap protocol by default. Only if you specify the port "636", it will use ldaps now. --- src/lib/Bcfg2/Server/Plugins/Ldap.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Ldap.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index f342fba35..0b66f7777 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -169,7 +169,7 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, class LdapConnection(Debuggable): """ Connection to an LDAP server. """ - def __init__(self, host="localhost", port=389, binddn=None, + def __init__(self, host="localhost", port=389, uri=None, binddn=None, bindpw=None): Debuggable.__init__(self) @@ -180,6 +180,7 @@ class LdapConnection(Debuggable): self.host = host self.port = port + self.uri = uri self.binddn = binddn self.bindpw = bindpw self.conn = None @@ -204,7 +205,8 @@ class LdapConnection(Debuggable): """ Open a connection to the configured LDAP server, and do a simple bind ff both binddn and bindpw are set. """ self.disconnect() - self.conn = ldap.initialize(self.url) + self.conn = ldap.initialize(self.get_uri()) + if self.binddn is not None and self.bindpw is not None: self.conn.simple_bind_s(self.binddn, self.bindpw) @@ -228,16 +230,20 @@ class LdapConnection(Debuggable): self.conn = None self.logger.error( "LdapConnection: Server %s down. Retry %d/%d in %.2fs." % - (self.url, attempt + 1, Bcfg2.Options.setup.ldap_retries, + (self.get_uri(), attempt + 1, + Bcfg2.Options.setup.ldap_retries, Bcfg2.Options.setup.ldap_retry_delay)) time.sleep(Bcfg2.Options.setup.ldap_retry_delay) return None - @property - def url(self): + def get_uri(self): """ The URL of the LDAP server. """ - return "ldap://%s:%d" % (self.host, self.port) + if self.uri is None: + if self.port == 636: + return "ldaps://%s" % self.host + return "ldap://%s:%d" % (self.host, self.port) + return self.uri class LdapQuery(object): -- cgit v1.2.3-1-g7c22 From b914052d7c33cc45012f693763189aa7db7a78a2 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 15 Jul 2016 17:31:23 +0200 Subject: Server/Plugins/Ldap: Support arbitrary ldap options You can now set arbitrary ldap option for the connection by specifying a dict with the key and the value. You should use the constants from python-ldap. --- src/lib/Bcfg2/Server/Plugins/Ldap.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Ldap.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 0b66f7777..a51f47dae 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -169,8 +169,8 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, class LdapConnection(Debuggable): """ Connection to an LDAP server. """ - def __init__(self, host="localhost", port=389, uri=None, binddn=None, - bindpw=None): + def __init__(self, host="localhost", port=389, uri=None, options=None, + binddn=None, bindpw=None): Debuggable.__init__(self) if HAS_LDAP: @@ -181,6 +181,7 @@ class LdapConnection(Debuggable): self.host = host self.port = port self.uri = uri + self.options = options self.binddn = binddn self.bindpw = bindpw self.conn = None @@ -207,6 +208,10 @@ class LdapConnection(Debuggable): self.disconnect() self.conn = ldap.initialize(self.get_uri()) + if self.options is not None: + for (option, value) in self.options.items(): + self.conn.set_option(option, value) + if self.binddn is not None and self.bindpw is not None: self.conn.simple_bind_s(self.binddn, self.bindpw) -- cgit v1.2.3-1-g7c22 From 0985c2aed06c14d8b79805d21449f2f1d31dd20c Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 17 Aug 2016 03:28:20 +0200 Subject: Server/Plugins/Ldap: Fix module name If the module name contains slashes, python will issue an warning: > Ldap/config.py:1: RuntimeWarning: Parent module '__Ldap_/root/repo/Ldap/config' not found while handling absolute import > from Bcfg2.Server.Plugins.Ldap import LdapConnection, LdapQuery So we simply use the basename without the file extension for the module name. --- src/lib/Bcfg2/Server/Plugins/Ldap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Ldap.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index a51f47dae..770419ba5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -33,7 +33,8 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked): def Index(self): """ Get the queries from the config file """ try: - module = imp.load_source(safe_module_name('Ldap', self.name), + module_name = os.path.splitext(os.path.basename(self.name))[0] + module = imp.load_source(safe_module_name('Ldap', module_name), self.name) except: # pylint: disable=W0702 err = sys.exc_info()[1] -- cgit v1.2.3-1-g7c22