diff options
author | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2013-10-28 09:20:12 -0400 |
---|---|---|
committer | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2013-10-28 09:20:12 -0400 |
commit | 99c5e5ca3f834bd1d42b280eb03c73cfcf7c52d8 (patch) | |
tree | 5468ffd8e07b500c6b6732c853e7f23840a76181 /src/lib | |
parent | 48934ccb7f7bec3151c1aa24cb5bf495836e5b89 (diff) | |
parent | 1d27fdadac37eb95d56f98cf7bdd1721a339df81 (diff) | |
download | bcfg2-99c5e5ca3f834bd1d42b280eb03c73cfcf7c52d8.tar.gz bcfg2-99c5e5ca3f834bd1d42b280eb03c73cfcf7c52d8.tar.bz2 bcfg2-99c5e5ca3f834bd1d42b280eb03c73cfcf7c52d8.zip |
Merge branch 'maint'
Conflicts:
doc/development/lint.txt
misc/bcfg2.spec
src/lib/Bcfg2/Reporting/Collector.py
src/lib/Bcfg2/Server/Core.py
src/lib/Bcfg2/Server/Plugins/Metadata.py
src/lib/Bcfg2/Server/models.py
testsuite/install.sh
Diffstat (limited to 'src/lib')
-rw-r--r-- | src/lib/Bcfg2/Reporting/Collector.py | 38 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/AWSTags.py | 29 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/AWSTags.py | 190 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 24 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 16 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 |
8 files changed, 275 insertions, 28 deletions
diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index a93f1b0ae..945c1dd05 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -20,11 +20,38 @@ from Bcfg2.Reporting.Transport.DirectStore import DirectStore from Bcfg2.Reporting.Storage.base import StorageError + class ReportingError(Exception): """Generic reporting exception""" pass +class ReportingStoreThread(threading.Thread): + """Thread for calling the storage backend""" + def __init__(self, interaction, storage, group=None, target=None, + name=None, args=(), kwargs=None): + """Initialize the thread with a reference to the interaction + as well as the storage engine to use""" + threading.Thread.__init__(self, group, target, name, args, + kwargs or dict()) + self.interaction = interaction + self.storage = storage + self.logger = logging.getLogger('bcfg2-report-collector') + + def run(self): + """Call the database storage procedure (aka import)""" + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + traceback.format_exc().splitlines()[-1]) + + class ReportingCollector(object): """The collecting process for reports""" options = [Bcfg2.Options.Common.reporting_storage, @@ -99,15 +126,8 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - try: - start = time.time() - self.storage.import_interaction(interaction) - self.logger.info("Imported interaction for %s in %ss" % - (interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - raise + store_thread = ReportingStoreThread(interaction, self.storage) + store_thread.start() except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 99bf4baf5..4e7f30db5 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -595,7 +595,8 @@ class Core(object): return ret except: self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, oldname, entry.get('name'))) + (entry.tag, entry.get('realname'), + entry.get('name'))) entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) diff --git a/src/lib/Bcfg2/Server/Lint/AWSTags.py b/src/lib/Bcfg2/Server/Lint/AWSTags.py new file mode 100644 index 000000000..a6af63dd6 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/AWSTags.py @@ -0,0 +1,29 @@ +import re +import sys +import Bcfg2.Server.Lint + + +class AWSTags(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + <server-plugins-connectors-awstags>` patterns for validity. """ + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 2042382e7..e38619355 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -54,6 +54,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Decisions/*.xml": "decisions.xsd", "Packages/sources.xml": "packages.xsd", "GroupPatterns/config.xml": "grouppatterns.xsd", + "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "GroupLogic/groups.xml": "grouplogic.xsd" diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py new file mode 100644 index 000000000..4b81a1275 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -0,0 +1,190 @@ +""" Query tags from AWS via boto, optionally setting group membership """ + +import os +import re +import sys +import Bcfg2.Server.Plugin +from boto import connect_ec2 +from Bcfg2.Server.Cache import Cache +from Bcfg2.Compat import ConfigParser + + +class NoInstanceFound(Exception): + """ Raised when there's no AWS instance for a given hostname """ + + +class AWSTagPattern(object): + """ Handler for a single Tag entry """ + + def __init__(self, name, value, groups): + self.name = re.compile(name) + if value is not None: + self.value = re.compile(value) + else: + self.value = value + self.groups = groups + + def get_groups(self, tags): + """ Get groups that apply to the given tag set """ + for key, value in tags.items(): + name_match = self.name.search(key) + if name_match: + if self.value is not None: + value_match = self.value.search(value) + if value_match: + return self._munge_groups(value_match) + else: + return self._munge_groups(name_match) + break + return [] + + def _munge_groups(self, match): + """ Replace backreferences (``$1``, ``$2``) in Group tags with + their values in the regex. """ + rv = [] + sub = match.groups() + for group in self.groups: + newg = group + for idx in range(len(sub)): + newg = newg.replace('$%s' % (idx + 1), sub[idx]) + rv.append(newg) + return rv + + def __str__(self): + if self.value: + return "%s: %s=%s: %s" % (self.__class__.__name__, self.name, + self.value, self.groups) + else: + return "%s: %s: %s" % (self.__class__.__name__, self.name, + self.groups) + + +class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of AWSTags config.xml """ + __identifier__ = None + create = 'AWSTags' + + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + should_monitor=True) + self.core = core + self.tags = [] + + def Index(self): + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() + self.tags = [] + for entry in self.xdata.xpath('//Tag'): + try: + groups = [g.text for g in entry.findall('Group')] + self.tags.append(AWSTagPattern(entry.get("name"), + entry.get("value"), + groups)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to initialize pattern %s: " + "%s" % (entry.get("name"), + sys.exc_info()[1])) + + def get_groups(self, hostname, tags): + """ return a list of groups that should be added to the given + client based on patterns that match the hostname """ + ret = [] + for pattern in self.tags: + try: + ret.extend(pattern.get_groups(tags)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to process pattern %s for " + "%s" % (pattern, hostname), + exc_info=1) + return ret + + +class AWSTags(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ Query tags from AWS via boto, optionally setting group membership """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + key_id = self.core.setup.cfp.get("awstags", "access_key_id") + secret_key = self.core.setup.cfp.get("awstags", + "secret_access_key") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError( + "AWSTags is not configured in bcfg2.conf: %s" % err) + self.debug_log("%s: Connecting to EC2" % self.name) + self._ec2 = connect_ec2(aws_access_key_id=key_id, + aws_secret_access_key=secret_key) + self._tagcache = Cache() + try: + self._keep_cache = self.core.setup.cfp.getboolean("awstags", + "cache") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._keep_cache = True + + self.config = PatternFile(os.path.join(self.data, 'config.xml'), + core=core) + + def _load_instance(self, hostname): + """ Load an instance from EC2 whose private DNS name matches + the given hostname """ + self.debug_log("AWSTags: Loading instance with private-dns-name=%s" % + hostname) + filters = {'private-dns-name': hostname} + reservations = self._ec2.get_all_instances(filters=filters) + if reservations: + res = reservations[0] + if res.instances: + return res.instances[0] + raise NoInstanceFound( + "AWSTags: No instance found with private-dns-name=%s" % + hostname) + + def _get_tags_from_ec2(self, hostname): + """ Get tags for the given host from EC2. This does not use + the local caching layer. """ + self.debug_log("AWSTags: Getting tags for %s from AWS" % + hostname) + try: + return self._load_instance(hostname).tags + except NoInstanceFound: + self.debug_log(sys.exc_info()[1]) + return dict() + + def get_tags(self, metadata): + """ Get tags for the given host. This caches the tags locally + if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is + true. """ + if not self._keep_cache: + return self._get_tags_from_ec2(metadata) + + if metadata.hostname not in self._tagcache: + self._tagcache[metadata.hostname] = \ + self._get_tags_from_ec2(metadata.hostname) + return self._tagcache[metadata.hostname] + + def expire_cache(self, key=None): + self._tagcache.expire(key=key) + + def start_client_run(self, metadata): + self.expire_cache(key=metadata.hostname) + + def get_additional_data(self, metadata): + return self.get_tags(metadata) + + def get_additional_groups(self, metadata): + return self.config.get_groups(metadata.hostname, + self.get_tags(metadata)) diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index ebcab1a6b..06e797ec3 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -41,19 +41,21 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), should_monitor=True) self._local = local() - # building is a thread-local set that tracks which machines - # GroupLogic is getting additional groups for. If a - # get_additional_groups() is called twice for a machine before - # the first call has completed, the second call returns an - # empty list. This is for infinite recursion protection; - # without this check, it'd be impossible to use things like - # metadata.query.in_group() in GroupLogic, since that requires - # building all metadata, which requires running - # GroupLogic.get_additional_groups() for all hosts, which - # requires building all metadata... - self._local.building = set() def get_additional_groups(self, metadata): + if not hasattr(self._local, "building"): + # building is a thread-local set that tracks which + # machines GroupLogic is getting additional groups for. + # If a get_additional_groups() is called twice for a + # machine before the first call has completed, the second + # call returns an empty list. This is for infinite + # recursion protection; without this check, it'd be + # impossible to use things like metadata.query.in_group() + # in GroupLogic, since that requires building all + # metadata, which requires running + # GroupLogic.get_additional_groups() for all hosts, which + # requires building all metadata... + self._local.building = set() if metadata.hostname in self._local.building: return [] self._local.building.add(metadata.hostname) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 073424c90..03f8af719 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -17,7 +17,9 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 +# pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, any, wraps +# pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo # pylint: disable=C0103 @@ -224,6 +226,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) + self.load_xml() def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" @@ -696,14 +699,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node is None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = 'Unexpected error finding %s "%s"' % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4c685d427..195765c69 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -605,7 +605,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ getter for the 'sources' key of the OnDemandDict returned by this function. This delays calling get_collection() until it's absolutely necessary. """ - return self.get_collection(metadata).get_additional_data + return self.get_collection(metadata).get_additional_data() return OnDemandDict( sources=get_sources, |