From 35b53c77c4b7edad7cf84146abf5722ea5323eba Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 13:48:43 -0400 Subject: New plugin: AWSTags AWSTags allows querying tags from EC2, and setting groups based on the tag names or values. --- src/lib/Bcfg2/Server/Plugins/AWSTags.py | 217 ++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Plugins/AWSTags.py (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py new file mode 100644 index 000000000..962b5d6a1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -0,0 +1,217 @@ +""" Query tags from AWS via boto, optionally setting group membership """ + +import os +import re +import sys +import Bcfg2.Server.Lint +import Bcfg2.Server.Plugin +from boto import connect_ec2 +from Bcfg2.Cache import Cache +from Bcfg2.Compat import ConfigParser + + +class NoInstanceFound(Exception): + """ Raised when there's no AWS instance for a given hostname """ + + +class AWSTagPattern(object): + """ Handler for a single Tag entry """ + + def __init__(self, name, value, groups): + self.name = re.compile(name) + if value is not None: + self.value = re.compile(value) + else: + self.value = value + self.groups = groups + + def get_groups(self, tags): + """ Get groups that apply to the given tag set """ + for key, value in tags.items(): + name_match = self.name.search(key) + if name_match: + if self.value is not None: + value_match = self.value.search(value) + if value_match: + return self._munge_groups(value_match) + else: + return self._munge_groups(name_match) + break + return [] + + def _munge_groups(self, match): + """ Replace backreferences (``$1``, ``$2``) in Group tags with + their values in the regex. """ + rv = [] + sub = match.groups() + for group in self.groups: + newg = group + for idx in range(len(sub)): + newg = newg.replace('$%s' % (idx + 1), sub[idx]) + rv.append(newg) + return rv + + def __str__(self): + if self.value: + return "%s: %s=%s: %s" % (self.__class__.__name__, self.name, + self.value, self.groups) + else: + return "%s: %s: %s" % (self.__class__.__name__, self.name, + self.groups) + + +class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of AWSTags config.xml """ + __identifier__ = None + create = 'AWSTags' + + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + should_monitor=True) + self.core = core + self.tags = [] + + def Index(self): + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() + self.tags = [] + for entry in self.xdata.xpath('//Tag'): + try: + groups = [g.text for g in entry.findall('Group')] + self.tags.append(AWSTagPattern(entry.get("name"), + entry.get("value"), + groups)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to initialize pattern %s: " + "%s" % (entry.get("name"), + sys.exc_info()[1])) + + def get_groups(self, hostname, tags): + """ return a list of groups that should be added to the given + client based on patterns that match the hostname """ + ret = [] + for pattern in self.tags: + try: + ret.extend(pattern.get_groups(tags)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to process pattern %s for " + "%s" % (pattern, hostname), + exc_info=1) + return ret + + +class AWSTags(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ Query tags from AWS via boto, optionally setting group membership """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + key_id = self.core.setup.cfp.get("awstags", "access_key_id") + secret_key = self.core.setup.cfp.get("awstags", + "secret_access_key") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError( + "AWSTags is not configured in bcfg2.conf: %s" % err) + self.debug_log("%s: Connecting to EC2" % self.name) + self._ec2 = connect_ec2(aws_access_key_id=key_id, + aws_secret_access_key=secret_key) + self._tagcache = Cache() + try: + self._keep_cache = self.core.setup.cfp.getboolean("awstags", + "cache") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._keep_cache = True + + self.config = PatternFile(os.path.join(self.data, 'config.xml'), + core=core) + + def _load_instance(self, hostname): + """ Load an instance from EC2 whose private DNS name matches + the given hostname """ + self.debug_log("AWSTags: Loading instance with private-dns-name=%s" % + hostname) + filters = {'private-dns-name': hostname} + reservations = self._ec2.get_all_instances(filters=filters) + if reservations: + res = reservations[0] + if res.instances: + return res.instances[0] + raise NoInstanceFound( + "AWSTags: No instance found with private-dns-name=%s" % + hostname) + + def _get_tags_from_ec2(self, hostname): + """ Get tags for the given host from EC2. This does not use + the local caching layer. """ + self.debug_log("AWSTags: Getting tags for %s from AWS" % + hostname) + try: + return self._load_instance(hostname).tags + except NoInstanceFound: + self.debug_log(sys.exc_info()[1]) + return dict() + + def get_tags(self, metadata): + """ Get tags for the given host. This caches the tags locally + if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is + true. """ + if not self._keep_cache: + return self._get_tags_from_ec2(metadata) + + if metadata.hostname not in self._tagcache: + self._tagcache[metadata.hostname] = \ + self._get_tags_from_ec2(metadata.hostname) + return self._tagcache[metadata.hostname] + + def expire_cache(self, key=None): + self._tagcache.expire(key=key) + + def start_client_run(self, metadata): + self.expire_cache(self, key=metadata.hostname) + + def get_additional_data(self, metadata): + return self.get_tags(metadata) + + def get_additional_groups(self, metadata): + return self.config.get_groups(metadata.hostname, + self.get_tags(metadata)) + + +class AWSTagsLint(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + ` patterns for validity. """ + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) -- cgit v1.2.3-1-g7c22 From 0bf89e137796811478575455c6bcbb008bec71f8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 13:49:27 -0400 Subject: Metadata: better logging when updating XML data fails --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 04ba79c55..d47fd644b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -677,14 +677,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node is None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = 'Unexpected error finding %s "%s"' % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) -- cgit v1.2.3-1-g7c22 From 5eb0b282d4e5142e37018d12b4a11cae71c82e40 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 15:03:40 -0400 Subject: AWSTags: fixed cache clearing at start of client run --- src/lib/Bcfg2/Server/Plugins/AWSTags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py index 962b5d6a1..147f37fbf 100644 --- a/src/lib/Bcfg2/Server/Plugins/AWSTags.py +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -181,7 +181,7 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, self._tagcache.expire(key=key) def start_client_run(self, metadata): - self.expire_cache(self, key=metadata.hostname) + self.expire_cache(key=metadata.hostname) def get_additional_data(self, metadata): return self.get_tags(metadata) -- cgit v1.2.3-1-g7c22 From 36641e89d28aeb411cc1886a7ecd90b8bcbf0f8f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 27 Sep 2013 06:40:04 -0400 Subject: GroupLogic: fixed thread-local variable initialization --- src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index aa71d2cfe..d74c16e8b 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -47,19 +47,21 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), core.fam) self._local = local() - # building is a thread-local set that tracks which machines - # GroupLogic is getting additional groups for. If a - # get_additional_groups() is called twice for a machine before - # the first call has completed, the second call returns an - # empty list. This is for infinite recursion protection; - # without this check, it'd be impossible to use things like - # metadata.query.in_group() in GroupLogic, since that requires - # building all metadata, which requires running - # GroupLogic.get_additional_groups() for all hosts, which - # requires building all metadata... - self._local.building = set() def get_additional_groups(self, metadata): + if not hasattr(self._local, "building"): + # building is a thread-local set that tracks which + # machines GroupLogic is getting additional groups for. + # If a get_additional_groups() is called twice for a + # machine before the first call has completed, the second + # call returns an empty list. This is for infinite + # recursion protection; without this check, it'd be + # impossible to use things like metadata.query.in_group() + # in GroupLogic, since that requires building all + # metadata, which requires running + # GroupLogic.get_additional_groups() for all hosts, which + # requires building all metadata... + self._local.building = set() if metadata.hostname in self._local.building: return [] self._local.building.add(metadata.hostname) -- cgit v1.2.3-1-g7c22 From 7f6c10db41c22b3924539aae19164a9ab9a80468 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 3 Oct 2013 16:24:05 -0400 Subject: Metadata: import any() from Compat --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index d47fd644b..4a0413a55 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -16,7 +16,9 @@ import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked -from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 +# pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, any, wraps +# pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo try: -- cgit v1.2.3-1-g7c22 From 9a6a231ccb4f509c0f6fa932c97bad647d29af50 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 4 Oct 2013 15:10:36 -0400 Subject: Metadata: read in clients.xml on every write This ensures consistency between the in-memory representation of clients.xml and the representation on disk. If we don't read our writes immediately, there's a race condition when creating a new client: If it asserts its profile or version before the FAM event from the clients.xml edit is processed, then the clients doesn't appear to exist yet, and Bcfg2 complains. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 4a0413a55..047dd4f4e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -221,6 +221,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) + self.load_xml() def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" -- cgit v1.2.3-1-g7c22 From 3a5eec174af0b9907b29fdfd3eb1e4fd7677beeb Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 9 Oct 2013 11:24:07 -0400 Subject: Packages: fixed metadata.Packages["sources"] --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 3cdcdc162..479138ef1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -651,7 +651,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ getter for the 'sources' key of the OnDemandDict returned by this function. This delays calling get_collection() until it's absolutely necessary. """ - return self.get_collection(metadata).get_additional_data + return self.get_collection(metadata).get_additional_data() return OnDemandDict( sources=get_sources, -- cgit v1.2.3-1-g7c22