Merge branch 'maint'

Conflicts:
	doc/development/lint.txt
	misc/bcfg2.spec
	src/lib/Bcfg2/Reporting/Collector.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/Plugins/Metadata.py
	src/lib/Bcfg2/Server/models.py
	testsuite/install.sh

8 files changed, 275 insertions, 28 deletions

diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py
index a93f1b0ae..945c1dd05 100644
--- a/src/lib/Bcfg2/Reporting/Collector.py
+++ b/src/lib/Bcfg2/Reporting/Collector.py
@@ -20,11 +20,38 @@ from Bcfg2.Reporting.Transport.DirectStore import DirectStore
 from Bcfg2.Reporting.Storage.base import StorageError
 
 
+
 class ReportingError(Exception):
     """Generic reporting exception"""
     pass
 
 
+class ReportingStoreThread(threading.Thread):
+    """Thread for calling the storage backend"""
+    def __init__(self, interaction, storage, group=None, target=None,
+                 name=None, args=(), kwargs=None):
+        """Initialize the thread with a reference to the interaction
+        as well as the storage engine to use"""
+        threading.Thread.__init__(self, group, target, name, args,
+                                  kwargs or dict())
+        self.interaction = interaction
+        self.storage = storage
+        self.logger = logging.getLogger('bcfg2-report-collector')
+
+    def run(self):
+        """Call the database storage procedure (aka import)"""
+        try:
+            start = time.time()
+            self.storage.import_interaction(self.interaction)
+            self.logger.info("Imported interaction for %s in %ss" %
+                             (self.interaction.get('hostname', '<unknown>'),
+                              time.time() - start))
+        except:
+            #TODO requeue?
+            self.logger.error("Unhandled exception in import thread %s" %
+                              traceback.format_exc().splitlines()[-1])
+
+
 class ReportingCollector(object):
     """The collecting process for reports"""
     options = [Bcfg2.Options.Common.reporting_storage,
@@ -99,15 +126,8 @@ class ReportingCollector(object):
                 interaction = self.transport.fetch()
                 if not interaction:
                     continue
-                try:
-                    start = time.time()
-                    self.storage.import_interaction(interaction)
-                    self.logger.info("Imported interaction for %s in %ss" %
-                                     (interaction.get('hostname', '<unknown>'),
-                                      time.time() - start))
-                except:
-                    #TODO requeue?
-                    raise
+                store_thread = ReportingStoreThread(interaction, self.storage)
+                store_thread.start()
             except (SystemExit, KeyboardInterrupt):
                 self.logger.info("Shutting down")
                 self.shutdown()
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 99bf4baf5..4e7f30db5 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -595,7 +595,8 @@ class Core(object):
                 return ret
             except:
                 self.logger.error("Failed binding entry %s:%s with altsrc %s" %
-                                  (entry.tag, oldname, entry.get('name')))
+                                  (entry.tag, entry.get('realname'),
+                                   entry.get('name')))
                 entry.set('name', oldname)
                 self.logger.error("Falling back to %s:%s" %
                                   (entry.tag, entry.get('name')))
diff --git a/src/lib/Bcfg2/Server/Lint/AWSTags.py b/src/lib/Bcfg2/Server/Lint/AWSTags.py
new file mode 100644
index 000000000..a6af63dd6
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Lint/AWSTags.py
@@ -0,0 +1,29 @@
+import re
+import sys
+import Bcfg2.Server.Lint
+
+
+class AWSTags(Bcfg2.Server.Lint.ServerPlugin):
+    """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags
+    <server-plugins-connectors-awstags>` patterns for validity. """
+
+    def Run(self):
+        cfg = self.core.plugins['AWSTags'].config
+        for entry in cfg.xdata.xpath('//Tag'):
+            self.check(entry, "name")
+            if entry.get("value"):
+                self.check(entry, "value")
+
+    @classmethod
+    def Errors(cls):
+        return {"pattern-fails-to-initialize": "error"}
+
+    def check(self, entry, attr):
+        """ Check a single attribute (``name`` or ``value``) of a
+        single entry for validity. """
+        try:
+            re.compile(entry.get(attr))
+        except re.error:
+            self.LintError("pattern-fails-to-initialize",
+                           "'%s' regex could not be compiled: %s\n    %s" %
+                           (attr, sys.exc_info()[1], entry.get("name")))
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 2042382e7..e38619355 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -54,6 +54,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
              "Decisions/*.xml": "decisions.xsd",
              "Packages/sources.xml": "packages.xsd",
              "GroupPatterns/config.xml": "grouppatterns.xsd",
+             "AWSTags/config.xml": "awstags.xsd",
              "NagiosGen/config.xml": "nagiosgen.xsd",
              "FileProbes/config.xml": "fileprobes.xsd",
              "GroupLogic/groups.xml": "grouplogic.xsd"
diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py
new file mode 100644
index 000000000..4b81a1275
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py
@@ -0,0 +1,190 @@
+""" Query tags from AWS via boto, optionally setting group membership """
+
+import os
+import re
+import sys
+import Bcfg2.Server.Plugin
+from boto import connect_ec2
+from Bcfg2.Server.Cache import Cache
+from Bcfg2.Compat import ConfigParser
+
+
+class NoInstanceFound(Exception):
+    """ Raised when there's no AWS instance for a given hostname """
+
+
+class AWSTagPattern(object):
+    """ Handler for a single Tag entry """
+
+    def __init__(self, name, value, groups):
+        self.name = re.compile(name)
+        if value is not None:
+            self.value = re.compile(value)
+        else:
+            self.value = value
+        self.groups = groups
+
+    def get_groups(self, tags):
+        """ Get groups that apply to the given tag set """
+        for key, value in tags.items():
+            name_match = self.name.search(key)
+            if name_match:
+                if self.value is not None:
+                    value_match = self.value.search(value)
+                    if value_match:
+                        return self._munge_groups(value_match)
+                else:
+                    return self._munge_groups(name_match)
+                break
+        return []
+
+    def _munge_groups(self, match):
+        """ Replace backreferences (``$1``, ``$2``) in Group tags with
+        their values in the regex. """
+        rv = []
+        sub = match.groups()
+        for group in self.groups:
+            newg = group
+            for idx in range(len(sub)):
+                newg = newg.replace('$%s' % (idx + 1), sub[idx])
+            rv.append(newg)
+        return rv
+
+    def __str__(self):
+        if self.value:
+            return "%s: %s=%s: %s" % (self.__class__.__name__, self.name,
+                                      self.value, self.groups)
+        else:
+            return "%s: %s: %s" % (self.__class__.__name__, self.name,
+                                   self.groups)
+
+
+class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked):
+    """ representation of AWSTags config.xml """
+    __identifier__ = None
+    create = 'AWSTags'
+
+    def __init__(self, filename, core=None):
+        try:
+            fam = core.fam
+        except AttributeError:
+            fam = None
+        Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam,
+                                                   should_monitor=True)
+        self.core = core
+        self.tags = []
+
+    def Index(self):
+        Bcfg2.Server.Plugin.XMLFileBacked.Index(self)
+        if (self.core and
+            self.core.metadata_cache_mode in ['cautious', 'aggressive']):
+            self.core.metadata_cache.expire()
+        self.tags = []
+        for entry in self.xdata.xpath('//Tag'):
+            try:
+                groups = [g.text for g in entry.findall('Group')]
+                self.tags.append(AWSTagPattern(entry.get("name"),
+                                               entry.get("value"),
+                                               groups))
+            except:  # pylint: disable=W0702
+                self.logger.error("AWSTags: Failed to initialize pattern %s: "
+                                  "%s" % (entry.get("name"),
+                                          sys.exc_info()[1]))
+
+    def get_groups(self, hostname, tags):
+        """ return a list of groups that should be added to the given
+        client based on patterns that match the hostname """
+        ret = []
+        for pattern in self.tags:
+            try:
+                ret.extend(pattern.get_groups(tags))
+            except:  # pylint: disable=W0702
+                self.logger.error("AWSTags: Failed to process pattern %s for "
+                                  "%s" % (pattern, hostname),
+                                  exc_info=1)
+        return ret
+
+
+class AWSTags(Bcfg2.Server.Plugin.Plugin,
+              Bcfg2.Server.Plugin.Caching,
+              Bcfg2.Server.Plugin.ClientRunHooks,
+              Bcfg2.Server.Plugin.Connector):
+    """ Query tags from AWS via boto, optionally setting group membership """
+    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache']
+
+    def __init__(self, core, datastore):
+        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+        Bcfg2.Server.Plugin.Caching.__init__(self)
+        Bcfg2.Server.Plugin.ClientRunHooks.__init__(self)
+        Bcfg2.Server.Plugin.Connector.__init__(self)
+        try:
+            key_id = self.core.setup.cfp.get("awstags", "access_key_id")
+            secret_key = self.core.setup.cfp.get("awstags",
+                                                 "secret_access_key")
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            err = sys.exc_info()[1]
+            raise Bcfg2.Server.Plugin.PluginInitError(
+                "AWSTags is not configured in bcfg2.conf: %s" % err)
+        self.debug_log("%s: Connecting to EC2" % self.name)
+        self._ec2 = connect_ec2(aws_access_key_id=key_id,
+                                aws_secret_access_key=secret_key)
+        self._tagcache = Cache()
+        try:
+            self._keep_cache = self.core.setup.cfp.getboolean("awstags",
+                                                              "cache")
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            self._keep_cache = True
+
+        self.config = PatternFile(os.path.join(self.data, 'config.xml'),
+                                  core=core)
+
+    def _load_instance(self, hostname):
+        """ Load an instance from EC2 whose private DNS name matches
+        the given hostname """
+        self.debug_log("AWSTags: Loading instance with private-dns-name=%s" %
+                       hostname)
+        filters = {'private-dns-name': hostname}
+        reservations = self._ec2.get_all_instances(filters=filters)
+        if reservations:
+            res = reservations[0]
+            if res.instances:
+                return res.instances[0]
+        raise NoInstanceFound(
+            "AWSTags: No instance found with private-dns-name=%s" %
+            hostname)
+
+    def _get_tags_from_ec2(self, hostname):
+        """ Get tags for the given host from EC2. This does not use
+        the local caching layer. """
+        self.debug_log("AWSTags: Getting tags for %s from AWS" %
+                       hostname)
+        try:
+            return self._load_instance(hostname).tags
+        except NoInstanceFound:
+            self.debug_log(sys.exc_info()[1])
+            return dict()
+
+    def get_tags(self, metadata):
+        """ Get tags for the given host.  This caches the tags locally
+        if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is
+        true. """
+        if not self._keep_cache:
+            return self._get_tags_from_ec2(metadata)
+
+        if metadata.hostname not in self._tagcache:
+            self._tagcache[metadata.hostname] = \
+                self._get_tags_from_ec2(metadata.hostname)
+        return self._tagcache[metadata.hostname]
+
+    def expire_cache(self, key=None):
+        self._tagcache.expire(key=key)
+
+    def start_client_run(self, metadata):
+        self.expire_cache(key=metadata.hostname)
+
+    def get_additional_data(self, metadata):
+        return self.get_tags(metadata)
+
+    def get_additional_groups(self, metadata):
+        return self.config.get_groups(metadata.hostname,
+                                      self.get_tags(metadata))
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
index ebcab1a6b..06e797ec3 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
@@ -41,19 +41,21 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin,
         self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"),
                                        should_monitor=True)
         self._local = local()
-        # building is a thread-local set that tracks which machines
-        # GroupLogic is getting additional groups for.  If a
-        # get_additional_groups() is called twice for a machine before
-        # the first call has completed, the second call returns an
-        # empty list.  This is for infinite recursion protection;
-        # without this check, it'd be impossible to use things like
-        # metadata.query.in_group() in GroupLogic, since that requires
-        # building all metadata, which requires running
-        # GroupLogic.get_additional_groups() for all hosts, which
-        # requires building all metadata...
-        self._local.building = set()
 
     def get_additional_groups(self, metadata):
+        if not hasattr(self._local, "building"):
+            # building is a thread-local set that tracks which
+            # machines GroupLogic is getting additional groups for.
+            # If a get_additional_groups() is called twice for a
+            # machine before the first call has completed, the second
+            # call returns an empty list.  This is for infinite
+            # recursion protection; without this check, it'd be
+            # impossible to use things like metadata.query.in_group()
+            # in GroupLogic, since that requires building all
+            # metadata, which requires running
+            # GroupLogic.get_additional_groups() for all hosts, which
+            # requires building all metadata...
+            self._local.building = set()
         if metadata.hostname in self._local.building:
             return []
         self._local.building.add(metadata.hostname)
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 073424c90..03f8af719 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -17,7 +17,9 @@ import Bcfg2.Server.Plugin
 import Bcfg2.Server.FileMonitor
 from Bcfg2.Utils import locked
 from Bcfg2.Server.Cache import Cache
-from Bcfg2.Compat import MutableMapping, all, wraps  # pylint: disable=W0622
+# pylint: disable=W0622
+from Bcfg2.Compat import MutableMapping, all, any, wraps
+# pylint: enable=W0622
 from Bcfg2.version import Bcfg2VersionInfo
 
 # pylint: disable=C0103
@@ -224,6 +226,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked):
                                                          sys.exc_info()[1])
             self.logger.error(msg)
             raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg)
+        self.load_xml()
 
     def find_xml_for_xpath(self, xpath):
         """Find and load xml file containing the xpath query"""
@@ -696,14 +699,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         """ Generic method to modify XML data (group, client, etc.) """
         node = self._search_xdata(tag, name, config.xdata, alias=alias)
         if node is None:
-            self.logger.error("%s \"%s\" does not exist" % (tag, name))
-            raise Bcfg2.Server.Plugin.MetadataConsistencyError
+            msg = "%s \"%s\" does not exist" % (tag, name)
+            self.logger.error(msg)
+            raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
         xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
                                           (tag, node.get('name')))
         if not xdict:
-            self.logger.error("Unexpected error finding %s \"%s\"" %
-                              (tag, name))
-            raise Bcfg2.Server.Plugin.MetadataConsistencyError
+            msg = 'Unexpected error finding %s "%s"' % (tag, name)
+            self.logger.error(msg)
+            raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
         for key, val in list(attribs.items()):
             xdict['xquery'][0].set(key, val)
         config.write_xml(xdict['filename'], xdict['xmltree'])
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 4c685d427..195765c69 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -605,7 +605,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             """ getter for the 'sources' key of the OnDemandDict
             returned by this function.  This delays calling
             get_collection() until it's absolutely necessary. """
-            return self.get_collection(metadata).get_additional_data
+            return self.get_collection(metadata).get_additional_data()
 
         return OnDemandDict(
             sources=get_sources,