summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--debian/control1
-rw-r--r--doc/development/lint.txt5
-rw-r--r--doc/server/plugins/connectors/awstags.txt124
-rw-r--r--doc/server/plugins/connectors/properties.txt4
-rw-r--r--schemas/awstags.xsd73
-rw-r--r--src/lib/Bcfg2/Reporting/Collector.py38
-rw-r--r--src/lib/Bcfg2/Server/Core.py3
-rw-r--r--src/lib/Bcfg2/Server/Lint/AWSTags.py29
-rw-r--r--src/lib/Bcfg2/Server/Lint/Validate.py1
-rw-r--r--src/lib/Bcfg2/Server/Plugins/AWSTags.py190
-rw-r--r--src/lib/Bcfg2/Server/Plugins/GroupLogic.py24
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py16
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py2
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py140
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py7
-rwxr-xr-xtestsuite/install.sh8
16 files changed, 633 insertions, 32 deletions
diff --git a/debian/control b/debian/control
index f99e5888e..1b1ca8f61 100644
--- a/debian/control
+++ b/debian/control
@@ -9,6 +9,7 @@ Build-Depends: debhelper (>= 7.0.50~),
python-sphinx (>= 1.0.7+dfsg) | python3-sphinx,
python-lxml,
python-daemon,
+ python-boto,
python-cherrypy3,
python-gamin,
python-genshi,
diff --git a/doc/development/lint.txt b/doc/development/lint.txt
index 685823ab1..56a3d8a66 100644
--- a/doc/development/lint.txt
+++ b/doc/development/lint.txt
@@ -106,6 +106,11 @@ Basics
Existing ``bcfg2-lint`` Plugins
===============================
+AWSTags
+-------
+
+.. automodule:: Bcfg2.Server.Lint.AWSTags
+
Bundler
-------
diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt
new file mode 100644
index 000000000..b884ca065
--- /dev/null
+++ b/doc/server/plugins/connectors/awstags.txt
@@ -0,0 +1,124 @@
+.. -*- mode: rst -*-
+
+.. _server-plugins-connectors-awstags:
+
+=========
+ AWSTags
+=========
+
+The AWSTags plugin is a connector that retrieves tags from instances
+in EC2, and can assign optionally assign
+group membership pased on patterns in the tags. See `Using Tags
+<http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html>`_
+for details on using tags in EC2.
+
+AWSTags queries EC2 for instances whose ``private-dns-name`` property
+matches the hostname of the client.
+
+Setup
+=====
+
+#. Add ``AWSTags`` to the ``plugins`` option in ``/etc/bcfg2.conf``
+#. Configure AWS credentials in ``/etc/bcfg2.conf`` (See
+ `Configuration`_ below for details.)
+#. Optionally, create ``AWSTags/config.xml`` (See `Assigning Groups`_
+ below for details.)
+#. Restart the Bcfg2 server.
+
+Using Tag Data
+==============
+
+AWSTags exposes the data in templates as a dict available as
+``metadata.AWSTags``. E.g., in a :ref:`Genshi template
+<server-plugins-generators-cfg-genshi>`, you could do:
+
+.. code-block:: genshitext
+
+ Known tags on ${metadata.hostname}:
+ {% for key, val in metadata.AWSTags.items() %}\
+ ${key} ${val}
+ {% end %}\
+
+This would produce something like::
+
+ Known tags on foo.example.com:
+ Name foo.example.com
+ some random tag the value
+
+Assigning Groups
+================
+
+AWSTags can assign groups based on the tag data. This functionality
+is configured in ``AWSTags/config.xml``.
+
+Example
+-------
+
+.. code-block:: xml
+
+ <AWSTags>
+ <Tag name="^foo$">
+ <Group>foo</Group>
+ </Tag>
+ <Tag name="^bar$" value="^bar$">
+ <Group>bar</Group>
+ </Tag>
+ <Tag name="^bcfg2 group$" value="(.*)">
+ <Group>$1</Group>
+ </Tag>
+ </AWSTags>
+
+In this example, any machine with a tag named ``foo`` would be added
+to the ``foo`` group. Any machine with a tag named ``bar`` whose
+value was also ``bar`` would be added to the ``bar`` group. Finally,
+any machine with a tag named ``bcfg2 group`` would be added to the
+group named in the value of that tag.
+
+Note that both the ``name`` and ``value`` attributes are *always*
+regular expressions.
+
+If a ``<Tag/>`` element has only a ``name`` attribute, then it only
+checks for existence of a matching tag. If it has both ``name`` and
+``value``, then it checks for a matching tag with a matching value.
+
+You can use backreferences (``$1``, ``$2``, etc.) in the group names.
+If only ``name`` is specified, then the backreferences will refer to
+groups in the ``name`` regex. If ``name`` and ``value`` are both
+specified, then backreferences will refer to groups in the ``value``
+regex. If you specify both ``name`` and ``value``, it is not possible
+to refer to groups in the ``name`` regex.
+
+Schema Reference
+----------------
+
+.. xml:schema:: awstags.xsd
+
+Configuration
+=============
+
+AWSTags recognizes several options in ``/etc/bcfg2.conf``; at a
+minimum, you must configure an AWS access key ID and secret key. All
+of the following options are in the ``[awstags]`` section:
+
++-----------------------+-----------------------------------------------------+
+| Option | Description |
++=======================+=====================================================+
+| ``access_key_id`` | The AWS access key ID |
++-----------------------+-----------------------------------------------------+
+| ``secret_access_key`` | The AWS secret access key |
++-----------------------+-----------------------------------------------------+
+| ``cache`` | Whether or not to cache tag lookups. See `Caching`_ |
+| | for details. Default is to cache. |
++-----------------------+-----------------------------------------------------+
+
+Caching
+=======
+
+Since the AWS API isn't always very quick to respond, AWSTags caches
+its results by default. The cache is fairly short-lived: the cache
+for each host is expired when it starts a client run, so it will start
+the run with fresh data.
+
+If you frequently update tags on your instances, you may wish to
+disable caching. That's probably a bad idea, and would tend to
+suggest that updating tags frequently is perhaps the Wrong Thing.
diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt
index 6061e9451..a6f6af741 100644
--- a/doc/server/plugins/connectors/properties.txt
+++ b/doc/server/plugins/connectors/properties.txt
@@ -183,6 +183,8 @@ XML tag should be ``<Properties>``.
JSON Property Files
-------------------
+.. versionadded:: 1.3.0
+
The data in a JSON property file can be accessed with the ``json``
attribute, which is the loaded JSON data. The JSON properties
interface does not provide any additional functionality beyond the
@@ -191,6 +193,8 @@ interface does not provide any additional functionality beyond the
YAML Property Files
-------------------
+.. versionadded:: 1.3.0
+
The data in a YAML property file can be accessed with the ``yaml``
attribute, which is the loaded YAML data. Only a single YAML document
may be included in a file.
diff --git a/schemas/awstags.xsd b/schemas/awstags.xsd
new file mode 100644
index 000000000..72be0366f
--- /dev/null
+++ b/schemas/awstags.xsd
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en">
+
+ <xsd:annotation>
+ <xsd:documentation>
+ :ref:`AWSTags &lt;server-plugins-connectors-awstags&gt;` config
+ schema for bcfg2
+ </xsd:documentation>
+ </xsd:annotation>
+
+ <xsd:import namespace="http://www.w3.org/XML/1998/namespace"
+ schemaLocation="xml.xsd"/>
+
+ <xsd:complexType name="TagType">
+ <xsd:choice minOccurs="1" maxOccurs="unbounded">
+ <xsd:element name="Group" type="xsd:string" minOccurs="1"
+ maxOccurs="unbounded">
+ <xsd:annotation>
+ <xsd:documentation>
+ The group to assign to machines with tags that match the
+ enclosing Tag expression. More than one group can be
+ specified.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:element>
+ </xsd:choice>
+ <xsd:attribute name="name" type="xsd:string" use="required">
+ <xsd:annotation>
+ <xsd:documentation>
+ The name pattern to match against. This is a regular
+ expression. It is not anchored.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ <xsd:attribute name="value" type="xsd:string">
+ <xsd:annotation>
+ <xsd:documentation>
+ The value pattern to match against. This is a regular
+ expression. It is not anchored.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ </xsd:complexType>
+
+ <xsd:complexType name="AWSTagsType">
+ <xsd:annotation>
+ <xsd:documentation>
+ Top-level tag for ``AWSTags/config.xml``.
+ </xsd:documentation>
+ </xsd:annotation>
+ <xsd:choice minOccurs="1" maxOccurs="unbounded">
+ <xsd:element name="Tag" type="TagType">
+ <xsd:annotation>
+ <xsd:documentation>
+ Representation of a pattern that matches AWS tags. Tags can be
+ matched in one of two ways:
+
+ * If only :xml:attribute:`TagType:name` is specified, then
+ AWSTags will only look for a tag with a matching name, and
+ the value of tags is ignored.
+ * If both :xml:attribute:`TagType:name` and
+ :xml:attribute:`TagType:value` are specified, a tag must
+ have a matching name *and* a matching value.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:element>
+ <xsd:element name="AWSTags" type="AWSTagsType"/>
+ </xsd:choice>
+ <xsd:attribute ref="xml:base"/>
+ </xsd:complexType>
+
+ <xsd:element name="AWSTags" type="AWSTagsType"/>
+</xsd:schema>
diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py
index a93f1b0ae..945c1dd05 100644
--- a/src/lib/Bcfg2/Reporting/Collector.py
+++ b/src/lib/Bcfg2/Reporting/Collector.py
@@ -20,11 +20,38 @@ from Bcfg2.Reporting.Transport.DirectStore import DirectStore
from Bcfg2.Reporting.Storage.base import StorageError
+
class ReportingError(Exception):
"""Generic reporting exception"""
pass
+class ReportingStoreThread(threading.Thread):
+ """Thread for calling the storage backend"""
+ def __init__(self, interaction, storage, group=None, target=None,
+ name=None, args=(), kwargs=None):
+ """Initialize the thread with a reference to the interaction
+ as well as the storage engine to use"""
+ threading.Thread.__init__(self, group, target, name, args,
+ kwargs or dict())
+ self.interaction = interaction
+ self.storage = storage
+ self.logger = logging.getLogger('bcfg2-report-collector')
+
+ def run(self):
+ """Call the database storage procedure (aka import)"""
+ try:
+ start = time.time()
+ self.storage.import_interaction(self.interaction)
+ self.logger.info("Imported interaction for %s in %ss" %
+ (self.interaction.get('hostname', '<unknown>'),
+ time.time() - start))
+ except:
+ #TODO requeue?
+ self.logger.error("Unhandled exception in import thread %s" %
+ traceback.format_exc().splitlines()[-1])
+
+
class ReportingCollector(object):
"""The collecting process for reports"""
options = [Bcfg2.Options.Common.reporting_storage,
@@ -99,15 +126,8 @@ class ReportingCollector(object):
interaction = self.transport.fetch()
if not interaction:
continue
- try:
- start = time.time()
- self.storage.import_interaction(interaction)
- self.logger.info("Imported interaction for %s in %ss" %
- (interaction.get('hostname', '<unknown>'),
- time.time() - start))
- except:
- #TODO requeue?
- raise
+ store_thread = ReportingStoreThread(interaction, self.storage)
+ store_thread.start()
except (SystemExit, KeyboardInterrupt):
self.logger.info("Shutting down")
self.shutdown()
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 99bf4baf5..4e7f30db5 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -595,7 +595,8 @@ class Core(object):
return ret
except:
self.logger.error("Failed binding entry %s:%s with altsrc %s" %
- (entry.tag, oldname, entry.get('name')))
+ (entry.tag, entry.get('realname'),
+ entry.get('name')))
entry.set('name', oldname)
self.logger.error("Falling back to %s:%s" %
(entry.tag, entry.get('name')))
diff --git a/src/lib/Bcfg2/Server/Lint/AWSTags.py b/src/lib/Bcfg2/Server/Lint/AWSTags.py
new file mode 100644
index 000000000..a6af63dd6
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Lint/AWSTags.py
@@ -0,0 +1,29 @@
+import re
+import sys
+import Bcfg2.Server.Lint
+
+
+class AWSTags(Bcfg2.Server.Lint.ServerPlugin):
+ """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags
+ <server-plugins-connectors-awstags>` patterns for validity. """
+
+ def Run(self):
+ cfg = self.core.plugins['AWSTags'].config
+ for entry in cfg.xdata.xpath('//Tag'):
+ self.check(entry, "name")
+ if entry.get("value"):
+ self.check(entry, "value")
+
+ @classmethod
+ def Errors(cls):
+ return {"pattern-fails-to-initialize": "error"}
+
+ def check(self, entry, attr):
+ """ Check a single attribute (``name`` or ``value``) of a
+ single entry for validity. """
+ try:
+ re.compile(entry.get(attr))
+ except re.error:
+ self.LintError("pattern-fails-to-initialize",
+ "'%s' regex could not be compiled: %s\n %s" %
+ (attr, sys.exc_info()[1], entry.get("name")))
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 2042382e7..e38619355 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -54,6 +54,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"Decisions/*.xml": "decisions.xsd",
"Packages/sources.xml": "packages.xsd",
"GroupPatterns/config.xml": "grouppatterns.xsd",
+ "AWSTags/config.xml": "awstags.xsd",
"NagiosGen/config.xml": "nagiosgen.xsd",
"FileProbes/config.xml": "fileprobes.xsd",
"GroupLogic/groups.xml": "grouplogic.xsd"
diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py
new file mode 100644
index 000000000..4b81a1275
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py
@@ -0,0 +1,190 @@
+""" Query tags from AWS via boto, optionally setting group membership """
+
+import os
+import re
+import sys
+import Bcfg2.Server.Plugin
+from boto import connect_ec2
+from Bcfg2.Server.Cache import Cache
+from Bcfg2.Compat import ConfigParser
+
+
+class NoInstanceFound(Exception):
+ """ Raised when there's no AWS instance for a given hostname """
+
+
+class AWSTagPattern(object):
+ """ Handler for a single Tag entry """
+
+ def __init__(self, name, value, groups):
+ self.name = re.compile(name)
+ if value is not None:
+ self.value = re.compile(value)
+ else:
+ self.value = value
+ self.groups = groups
+
+ def get_groups(self, tags):
+ """ Get groups that apply to the given tag set """
+ for key, value in tags.items():
+ name_match = self.name.search(key)
+ if name_match:
+ if self.value is not None:
+ value_match = self.value.search(value)
+ if value_match:
+ return self._munge_groups(value_match)
+ else:
+ return self._munge_groups(name_match)
+ break
+ return []
+
+ def _munge_groups(self, match):
+ """ Replace backreferences (``$1``, ``$2``) in Group tags with
+ their values in the regex. """
+ rv = []
+ sub = match.groups()
+ for group in self.groups:
+ newg = group
+ for idx in range(len(sub)):
+ newg = newg.replace('$%s' % (idx + 1), sub[idx])
+ rv.append(newg)
+ return rv
+
+ def __str__(self):
+ if self.value:
+ return "%s: %s=%s: %s" % (self.__class__.__name__, self.name,
+ self.value, self.groups)
+ else:
+ return "%s: %s: %s" % (self.__class__.__name__, self.name,
+ self.groups)
+
+
+class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked):
+ """ representation of AWSTags config.xml """
+ __identifier__ = None
+ create = 'AWSTags'
+
+ def __init__(self, filename, core=None):
+ try:
+ fam = core.fam
+ except AttributeError:
+ fam = None
+ Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam,
+ should_monitor=True)
+ self.core = core
+ self.tags = []
+
+ def Index(self):
+ Bcfg2.Server.Plugin.XMLFileBacked.Index(self)
+ if (self.core and
+ self.core.metadata_cache_mode in ['cautious', 'aggressive']):
+ self.core.metadata_cache.expire()
+ self.tags = []
+ for entry in self.xdata.xpath('//Tag'):
+ try:
+ groups = [g.text for g in entry.findall('Group')]
+ self.tags.append(AWSTagPattern(entry.get("name"),
+ entry.get("value"),
+ groups))
+ except: # pylint: disable=W0702
+ self.logger.error("AWSTags: Failed to initialize pattern %s: "
+ "%s" % (entry.get("name"),
+ sys.exc_info()[1]))
+
+ def get_groups(self, hostname, tags):
+ """ return a list of groups that should be added to the given
+ client based on patterns that match the hostname """
+ ret = []
+ for pattern in self.tags:
+ try:
+ ret.extend(pattern.get_groups(tags))
+ except: # pylint: disable=W0702
+ self.logger.error("AWSTags: Failed to process pattern %s for "
+ "%s" % (pattern, hostname),
+ exc_info=1)
+ return ret
+
+
+class AWSTags(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Caching,
+ Bcfg2.Server.Plugin.ClientRunHooks,
+ Bcfg2.Server.Plugin.Connector):
+ """ Query tags from AWS via boto, optionally setting group membership """
+ __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache']
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Caching.__init__(self)
+ Bcfg2.Server.Plugin.ClientRunHooks.__init__(self)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ try:
+ key_id = self.core.setup.cfp.get("awstags", "access_key_id")
+ secret_key = self.core.setup.cfp.get("awstags",
+ "secret_access_key")
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ err = sys.exc_info()[1]
+ raise Bcfg2.Server.Plugin.PluginInitError(
+ "AWSTags is not configured in bcfg2.conf: %s" % err)
+ self.debug_log("%s: Connecting to EC2" % self.name)
+ self._ec2 = connect_ec2(aws_access_key_id=key_id,
+ aws_secret_access_key=secret_key)
+ self._tagcache = Cache()
+ try:
+ self._keep_cache = self.core.setup.cfp.getboolean("awstags",
+ "cache")
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+ self._keep_cache = True
+
+ self.config = PatternFile(os.path.join(self.data, 'config.xml'),
+ core=core)
+
+ def _load_instance(self, hostname):
+ """ Load an instance from EC2 whose private DNS name matches
+ the given hostname """
+ self.debug_log("AWSTags: Loading instance with private-dns-name=%s" %
+ hostname)
+ filters = {'private-dns-name': hostname}
+ reservations = self._ec2.get_all_instances(filters=filters)
+ if reservations:
+ res = reservations[0]
+ if res.instances:
+ return res.instances[0]
+ raise NoInstanceFound(
+ "AWSTags: No instance found with private-dns-name=%s" %
+ hostname)
+
+ def _get_tags_from_ec2(self, hostname):
+ """ Get tags for the given host from EC2. This does not use
+ the local caching layer. """
+ self.debug_log("AWSTags: Getting tags for %s from AWS" %
+ hostname)
+ try:
+ return self._load_instance(hostname).tags
+ except NoInstanceFound:
+ self.debug_log(sys.exc_info()[1])
+ return dict()
+
+ def get_tags(self, metadata):
+ """ Get tags for the given host. This caches the tags locally
+ if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is
+ true. """
+ if not self._keep_cache:
+ return self._get_tags_from_ec2(metadata)
+
+ if metadata.hostname not in self._tagcache:
+ self._tagcache[metadata.hostname] = \
+ self._get_tags_from_ec2(metadata.hostname)
+ return self._tagcache[metadata.hostname]
+
+ def expire_cache(self, key=None):
+ self._tagcache.expire(key=key)
+
+ def start_client_run(self, metadata):
+ self.expire_cache(key=metadata.hostname)
+
+ def get_additional_data(self, metadata):
+ return self.get_tags(metadata)
+
+ def get_additional_groups(self, metadata):
+ return self.config.get_groups(metadata.hostname,
+ self.get_tags(metadata))
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
index ebcab1a6b..06e797ec3 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
@@ -41,19 +41,21 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin,
self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"),
should_monitor=True)
self._local = local()
- # building is a thread-local set that tracks which machines
- # GroupLogic is getting additional groups for. If a
- # get_additional_groups() is called twice for a machine before
- # the first call has completed, the second call returns an
- # empty list. This is for infinite recursion protection;
- # without this check, it'd be impossible to use things like
- # metadata.query.in_group() in GroupLogic, since that requires
- # building all metadata, which requires running
- # GroupLogic.get_additional_groups() for all hosts, which
- # requires building all metadata...
- self._local.building = set()
def get_additional_groups(self, metadata):
+ if not hasattr(self._local, "building"):
+ # building is a thread-local set that tracks which
+ # machines GroupLogic is getting additional groups for.
+ # If a get_additional_groups() is called twice for a
+ # machine before the first call has completed, the second
+ # call returns an empty list. This is for infinite
+ # recursion protection; without this check, it'd be
+ # impossible to use things like metadata.query.in_group()
+ # in GroupLogic, since that requires building all
+ # metadata, which requires running
+ # GroupLogic.get_additional_groups() for all hosts, which
+ # requires building all metadata...
+ self._local.building = set()
if metadata.hostname in self._local.building:
return []
self._local.building.add(metadata.hostname)
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 073424c90..03f8af719 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -17,7 +17,9 @@ import Bcfg2.Server.Plugin
import Bcfg2.Server.FileMonitor
from Bcfg2.Utils import locked
from Bcfg2.Server.Cache import Cache
-from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622
+# pylint: disable=W0622
+from Bcfg2.Compat import MutableMapping, all, any, wraps
+# pylint: enable=W0622
from Bcfg2.version import Bcfg2VersionInfo
# pylint: disable=C0103
@@ -224,6 +226,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked):
sys.exc_info()[1])
self.logger.error(msg)
raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg)
+ self.load_xml()
def find_xml_for_xpath(self, xpath):
"""Find and load xml file containing the xpath query"""
@@ -696,14 +699,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
""" Generic method to modify XML data (group, client, etc.) """
node = self._search_xdata(tag, name, config.xdata, alias=alias)
if node is None:
- self.logger.error("%s \"%s\" does not exist" % (tag, name))
- raise Bcfg2.Server.Plugin.MetadataConsistencyError
+ msg = "%s \"%s\" does not exist" % (tag, name)
+ self.logger.error(msg)
+ raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
(tag, node.get('name')))
if not xdict:
- self.logger.error("Unexpected error finding %s \"%s\"" %
- (tag, name))
- raise Bcfg2.Server.Plugin.MetadataConsistencyError
+ msg = 'Unexpected error finding %s "%s"' % (tag, name)
+ self.logger.error(msg)
+ raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
for key, val in list(attribs.items()):
xdict['xquery'][0].set(key, val)
config.write_xml(xdict['filename'], xdict['xmltree'])
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 4c685d427..195765c69 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -605,7 +605,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
""" getter for the 'sources' key of the OnDemandDict
returned by this function. This delays calling
get_collection() until it's absolutely necessary. """
- return self.get_collection(metadata).get_additional_data
+ return self.get_collection(metadata).get_additional_data()
return OnDemandDict(
sources=get_sources,
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py
new file mode 100644
index 000000000..05e0bb9a1
--- /dev/null
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py
@@ -0,0 +1,140 @@
+import os
+import sys
+import lxml.etree
+import Bcfg2.Server.Plugin
+from mock import Mock, MagicMock, patch
+try:
+ from Bcfg2.Server.Plugins.AWSTags import *
+ HAS_BOTO = True
+except ImportError:
+ HAS_BOTO = False
+
+# add all parent testsuite directories to sys.path to allow (most)
+# relative imports in python 2.4
+path = os.path.dirname(__file__)
+while path != "/":
+ if os.path.basename(path).lower().startswith("test"):
+ sys.path.append(path)
+ if os.path.basename(path) == "testsuite":
+ break
+ path = os.path.dirname(path)
+from common import *
+from TestPlugin import TestPlugin, TestConnector, TestClientRunHooks
+
+config = '''
+<AWSTags>
+ <Tag name="name-only">
+ <Group>group1</Group>
+ <Group>group2</Group>
+ </Tag>
+ <Tag name="name-and-value" value="value">
+ <Group>group3</Group>
+ </Tag>
+ <Tag name="regex-(.*)">
+ <Group>group-$1</Group>
+ </Tag>
+ <Tag name="regex-value" value="(.*)">
+ <Group>group-$1</Group>
+ </Tag>
+</AWSTags>
+'''
+
+tags = {
+ "empty.example.com": {},
+ "no-matches.example.com": {"nameonly": "foo",
+ "Name": "no-matches",
+ "foo": "bar"},
+ "foo.example.com": {"name-only": "name-only",
+ "name-and-value": "wrong",
+ "regex-name": "foo"},
+ "bar.example.com": {"name-and-value": "value",
+ "regex-value": "bar"}}
+
+groups = {
+ "empty.example.com": [],
+ "no-matches.example.com": [],
+ "foo.example.com": ["group1", "group2", "group-name"],
+ "bar.example.com": ["group3", "group-value", "group-bar"]}
+
+
+def make_instance(name):
+ rv = Mock()
+ rv.private_dns_name = name
+ rv.tags = tags[name]
+ return rv
+
+
+instances = [make_instance(n) for n in tags.keys()]
+
+
+def get_all_instances(filters=None):
+ insts = [i for i in instances
+ if i.private_dns_name == filters['private-dns-name']]
+ res = Mock()
+ res.instances = insts
+ return [res]
+
+
+if HAS_BOTO:
+ class TestAWSTags(TestPlugin, TestClientRunHooks, TestConnector):
+ test_obj = AWSTags
+
+ def get_obj(self, core=None):
+ @patchIf(not isinstance(Bcfg2.Server.Plugins.AWSTags.connect_ec2,
+ Mock),
+ "Bcfg2.Server.Plugins.AWSTags.connect_ec2", Mock())
+ @patch("lxml.etree.Element", Mock())
+ def inner():
+ obj = TestPlugin.get_obj(self, core=core)
+ obj.config.data = config
+ obj.config.Index()
+ return obj
+ return inner()
+
+ @patch("Bcfg2.Server.Plugins.AWSTags.connect_ec2")
+ def test_connect(self, mock_connect_ec2):
+ """ Test connection to EC2 """
+ key_id = "a09sdbipasdf"
+ access_key = "oiilb234ipwe9"
+
+ def cfp_get(section, option):
+ if option == "access_key_id":
+ return key_id
+ elif option == "secret_access_key":
+ return access_key
+ else:
+ return Mock()
+
+ core = Mock()
+ core.setup.cfp.get = Mock(side_effect=cfp_get)
+ awstags = self.get_obj(core=core)
+ mock_connect_ec2.assert_called_with(
+ aws_access_key_id=key_id,
+ aws_secret_access_key=access_key)
+
+ def test_get_additional_data(self):
+ """ Test AWSTags.get_additional_data() """
+ awstags = self.get_obj()
+ awstags._ec2.get_all_instances = \
+ Mock(side_effect=get_all_instances)
+
+ for hostname, expected in tags.items():
+ metadata = Mock()
+ metadata.hostname = hostname
+ self.assertItemsEqual(awstags.get_additional_data(metadata),
+ expected)
+
+ def test_get_additional_groups_caching(self):
+ """ Test AWSTags.get_additional_groups() with caching enabled """
+ awstags = self.get_obj()
+ awstags._ec2.get_all_instances = \
+ Mock(side_effect=get_all_instances)
+
+ for hostname, expected in groups.items():
+ metadata = Mock()
+ metadata.hostname = hostname
+ actual = awstags.get_additional_groups(metadata)
+ msg = """%s has incorrect groups:
+actual: %s
+expected: %s""" % (hostname, actual, expected)
+ self.assertItemsEqual(actual, expected, msg)
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
index 290edb83a..8814ae171 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
@@ -336,6 +336,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
@patch('Bcfg2.Utils.locked', Mock(return_value=False))
@patch('fcntl.lockf', Mock())
+ @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml")
@patch('os.open')
@patch('os.fdopen')
@patch('os.unlink')
@@ -343,7 +344,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
@patch('os.path.islink')
@patch('os.readlink')
def test_write_xml(self, mock_readlink, mock_islink, mock_rename,
- mock_unlink, mock_fdopen, mock_open):
+ mock_unlink, mock_fdopen, mock_open, mock_load_xml):
fname = "clients.xml"
config = self.get_obj(fname)
fpath = os.path.join(self.metadata.data, fname)
@@ -357,6 +358,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
mock_unlink.reset_mock()
mock_fdopen.reset_mock()
mock_open.reset_mock()
+ mock_load_xml.reset_mock()
mock_islink.return_value = False
@@ -368,6 +370,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
self.assertTrue(mock_fdopen.return_value.write.called)
mock_islink.assert_called_with(fpath)
mock_rename.assert_called_with(tmpfile, fpath)
+ mock_load_xml.assert_called_with()
# test: clients.xml.new is locked the first time we write it
def rv(fname, mode):
@@ -386,6 +389,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
self.assertTrue(mock_fdopen.return_value.write.called)
mock_islink.assert_called_with(fpath)
mock_rename.assert_called_with(tmpfile, fpath)
+ mock_load_xml.assert_called_with()
# test writing a symlinked clients.xml
reset()
@@ -394,6 +398,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
mock_readlink.return_value = linkdest
config.write_xml(fpath, get_clients_test_tree())
mock_rename.assert_called_with(tmpfile, linkdest)
+ mock_load_xml.assert_called_with()
# test failure of os.rename()
reset()
diff --git a/testsuite/install.sh b/testsuite/install.sh
index 6895034d5..8721c8015 100755
--- a/testsuite/install.sh
+++ b/testsuite/install.sh
@@ -11,15 +11,17 @@ if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" ]]; then
fi
if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then
- pip install --use-mirrors PyYAML pyinotify
+ pip install --use-mirrors PyYAML pyinotify boto
if [[ $PYVER == "2.5" ]]; then
# markdown 2.2+ doesn't work on py2.5
- pip install --use-mirrors simplejson 'markdown<2.2'
+ pip install --use-mirrors simplejson 'markdown<2.2' 'django<1.4.9'
+ else
+ pip install 'django<1.5'
fi
if [[ ${PYVER:0:1} == "2" ]]; then
# django supports py3k, but South doesn't, and the django bits
# in bcfg2 require South
- pip install cheetah 'django<1.5' 'South<0.8' M2Crypto
+ pip install cheetah 'South<0.8' M2Crypto
fi
else
# python < 2.6 requires M2Crypto for SSL communication, not just
generated by cgit v1.2.3-1-g7c22 (git 2.25.1) at 2024-06-02 16:00:56 +0000