diff options
-rw-r--r-- | debian/control | 1 | ||||
-rw-r--r-- | doc/development/lint.txt | 5 | ||||
-rw-r--r-- | doc/server/plugins/connectors/awstags.txt | 124 | ||||
-rw-r--r-- | doc/server/plugins/connectors/properties.txt | 4 | ||||
-rw-r--r-- | schemas/awstags.xsd | 73 | ||||
-rw-r--r-- | src/lib/Bcfg2/Reporting/Collector.py | 38 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/AWSTags.py | 29 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/AWSTags.py | 190 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 24 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 16 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 | ||||
-rw-r--r-- | testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py | 140 | ||||
-rw-r--r-- | testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py | 7 | ||||
-rwxr-xr-x | testsuite/install.sh | 8 |
16 files changed, 633 insertions, 32 deletions
diff --git a/debian/control b/debian/control index f99e5888e..1b1ca8f61 100644 --- a/debian/control +++ b/debian/control @@ -9,6 +9,7 @@ Build-Depends: debhelper (>= 7.0.50~), python-sphinx (>= 1.0.7+dfsg) | python3-sphinx, python-lxml, python-daemon, + python-boto, python-cherrypy3, python-gamin, python-genshi, diff --git a/doc/development/lint.txt b/doc/development/lint.txt index 685823ab1..56a3d8a66 100644 --- a/doc/development/lint.txt +++ b/doc/development/lint.txt @@ -106,6 +106,11 @@ Basics Existing ``bcfg2-lint`` Plugins =============================== +AWSTags +------- + +.. automodule:: Bcfg2.Server.Lint.AWSTags + Bundler ------- diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt new file mode 100644 index 000000000..b884ca065 --- /dev/null +++ b/doc/server/plugins/connectors/awstags.txt @@ -0,0 +1,124 @@ +.. -*- mode: rst -*- + +.. _server-plugins-connectors-awstags: + +========= + AWSTags +========= + +The AWSTags plugin is a connector that retrieves tags from instances +in EC2, and can assign optionally assign +group membership pased on patterns in the tags. See `Using Tags +<http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html>`_ +for details on using tags in EC2. + +AWSTags queries EC2 for instances whose ``private-dns-name`` property +matches the hostname of the client. + +Setup +===== + +#. Add ``AWSTags`` to the ``plugins`` option in ``/etc/bcfg2.conf`` +#. Configure AWS credentials in ``/etc/bcfg2.conf`` (See + `Configuration`_ below for details.) +#. Optionally, create ``AWSTags/config.xml`` (See `Assigning Groups`_ + below for details.) +#. Restart the Bcfg2 server. + +Using Tag Data +============== + +AWSTags exposes the data in templates as a dict available as +``metadata.AWSTags``. E.g., in a :ref:`Genshi template +<server-plugins-generators-cfg-genshi>`, you could do: + +.. code-block:: genshitext + + Known tags on ${metadata.hostname}: + {% for key, val in metadata.AWSTags.items() %}\ + ${key} ${val} + {% end %}\ + +This would produce something like:: + + Known tags on foo.example.com: + Name foo.example.com + some random tag the value + +Assigning Groups +================ + +AWSTags can assign groups based on the tag data. This functionality +is configured in ``AWSTags/config.xml``. + +Example +------- + +.. code-block:: xml + + <AWSTags> + <Tag name="^foo$"> + <Group>foo</Group> + </Tag> + <Tag name="^bar$" value="^bar$"> + <Group>bar</Group> + </Tag> + <Tag name="^bcfg2 group$" value="(.*)"> + <Group>$1</Group> + </Tag> + </AWSTags> + +In this example, any machine with a tag named ``foo`` would be added +to the ``foo`` group. Any machine with a tag named ``bar`` whose +value was also ``bar`` would be added to the ``bar`` group. Finally, +any machine with a tag named ``bcfg2 group`` would be added to the +group named in the value of that tag. + +Note that both the ``name`` and ``value`` attributes are *always* +regular expressions. + +If a ``<Tag/>`` element has only a ``name`` attribute, then it only +checks for existence of a matching tag. If it has both ``name`` and +``value``, then it checks for a matching tag with a matching value. + +You can use backreferences (``$1``, ``$2``, etc.) in the group names. +If only ``name`` is specified, then the backreferences will refer to +groups in the ``name`` regex. If ``name`` and ``value`` are both +specified, then backreferences will refer to groups in the ``value`` +regex. If you specify both ``name`` and ``value``, it is not possible +to refer to groups in the ``name`` regex. + +Schema Reference +---------------- + +.. xml:schema:: awstags.xsd + +Configuration +============= + +AWSTags recognizes several options in ``/etc/bcfg2.conf``; at a +minimum, you must configure an AWS access key ID and secret key. All +of the following options are in the ``[awstags]`` section: + ++-----------------------+-----------------------------------------------------+ +| Option | Description | ++=======================+=====================================================+ +| ``access_key_id`` | The AWS access key ID | ++-----------------------+-----------------------------------------------------+ +| ``secret_access_key`` | The AWS secret access key | ++-----------------------+-----------------------------------------------------+ +| ``cache`` | Whether or not to cache tag lookups. See `Caching`_ | +| | for details. Default is to cache. | ++-----------------------+-----------------------------------------------------+ + +Caching +======= + +Since the AWS API isn't always very quick to respond, AWSTags caches +its results by default. The cache is fairly short-lived: the cache +for each host is expired when it starts a client run, so it will start +the run with fresh data. + +If you frequently update tags on your instances, you may wish to +disable caching. That's probably a bad idea, and would tend to +suggest that updating tags frequently is perhaps the Wrong Thing. diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt index 6061e9451..a6f6af741 100644 --- a/doc/server/plugins/connectors/properties.txt +++ b/doc/server/plugins/connectors/properties.txt @@ -183,6 +183,8 @@ XML tag should be ``<Properties>``. JSON Property Files ------------------- +.. versionadded:: 1.3.0 + The data in a JSON property file can be accessed with the ``json`` attribute, which is the loaded JSON data. The JSON properties interface does not provide any additional functionality beyond the @@ -191,6 +193,8 @@ interface does not provide any additional functionality beyond the YAML Property Files ------------------- +.. versionadded:: 1.3.0 + The data in a YAML property file can be accessed with the ``yaml`` attribute, which is the loaded YAML data. Only a single YAML document may be included in a file. diff --git a/schemas/awstags.xsd b/schemas/awstags.xsd new file mode 100644 index 000000000..72be0366f --- /dev/null +++ b/schemas/awstags.xsd @@ -0,0 +1,73 @@ +<?xml version="1.0" encoding="utf-8"?> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> + + <xsd:annotation> + <xsd:documentation> + :ref:`AWSTags <server-plugins-connectors-awstags>` config + schema for bcfg2 + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://www.w3.org/XML/1998/namespace" + schemaLocation="xml.xsd"/> + + <xsd:complexType name="TagType"> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Group" type="xsd:string" minOccurs="1" + maxOccurs="unbounded"> + <xsd:annotation> + <xsd:documentation> + The group to assign to machines with tags that match the + enclosing Tag expression. More than one group can be + specified. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + </xsd:choice> + <xsd:attribute name="name" type="xsd:string" use="required"> + <xsd:annotation> + <xsd:documentation> + The name pattern to match against. This is a regular + expression. It is not anchored. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="value" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + The value pattern to match against. This is a regular + expression. It is not anchored. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:complexType name="AWSTagsType"> + <xsd:annotation> + <xsd:documentation> + Top-level tag for ``AWSTags/config.xml``. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Tag" type="TagType"> + <xsd:annotation> + <xsd:documentation> + Representation of a pattern that matches AWS tags. Tags can be + matched in one of two ways: + + * If only :xml:attribute:`TagType:name` is specified, then + AWSTags will only look for a tag with a matching name, and + the value of tags is ignored. + * If both :xml:attribute:`TagType:name` and + :xml:attribute:`TagType:value` are specified, a tag must + have a matching name *and* a matching value. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="AWSTags" type="AWSTagsType"/> + </xsd:choice> + <xsd:attribute ref="xml:base"/> + </xsd:complexType> + + <xsd:element name="AWSTags" type="AWSTagsType"/> +</xsd:schema> diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index a93f1b0ae..945c1dd05 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -20,11 +20,38 @@ from Bcfg2.Reporting.Transport.DirectStore import DirectStore from Bcfg2.Reporting.Storage.base import StorageError + class ReportingError(Exception): """Generic reporting exception""" pass +class ReportingStoreThread(threading.Thread): + """Thread for calling the storage backend""" + def __init__(self, interaction, storage, group=None, target=None, + name=None, args=(), kwargs=None): + """Initialize the thread with a reference to the interaction + as well as the storage engine to use""" + threading.Thread.__init__(self, group, target, name, args, + kwargs or dict()) + self.interaction = interaction + self.storage = storage + self.logger = logging.getLogger('bcfg2-report-collector') + + def run(self): + """Call the database storage procedure (aka import)""" + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + traceback.format_exc().splitlines()[-1]) + + class ReportingCollector(object): """The collecting process for reports""" options = [Bcfg2.Options.Common.reporting_storage, @@ -99,15 +126,8 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - try: - start = time.time() - self.storage.import_interaction(interaction) - self.logger.info("Imported interaction for %s in %ss" % - (interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - raise + store_thread = ReportingStoreThread(interaction, self.storage) + store_thread.start() except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 99bf4baf5..4e7f30db5 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -595,7 +595,8 @@ class Core(object): return ret except: self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, oldname, entry.get('name'))) + (entry.tag, entry.get('realname'), + entry.get('name'))) entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) diff --git a/src/lib/Bcfg2/Server/Lint/AWSTags.py b/src/lib/Bcfg2/Server/Lint/AWSTags.py new file mode 100644 index 000000000..a6af63dd6 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/AWSTags.py @@ -0,0 +1,29 @@ +import re +import sys +import Bcfg2.Server.Lint + + +class AWSTags(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + <server-plugins-connectors-awstags>` patterns for validity. """ + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 2042382e7..e38619355 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -54,6 +54,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Decisions/*.xml": "decisions.xsd", "Packages/sources.xml": "packages.xsd", "GroupPatterns/config.xml": "grouppatterns.xsd", + "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "GroupLogic/groups.xml": "grouplogic.xsd" diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py new file mode 100644 index 000000000..4b81a1275 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -0,0 +1,190 @@ +""" Query tags from AWS via boto, optionally setting group membership """ + +import os +import re +import sys +import Bcfg2.Server.Plugin +from boto import connect_ec2 +from Bcfg2.Server.Cache import Cache +from Bcfg2.Compat import ConfigParser + + +class NoInstanceFound(Exception): + """ Raised when there's no AWS instance for a given hostname """ + + +class AWSTagPattern(object): + """ Handler for a single Tag entry """ + + def __init__(self, name, value, groups): + self.name = re.compile(name) + if value is not None: + self.value = re.compile(value) + else: + self.value = value + self.groups = groups + + def get_groups(self, tags): + """ Get groups that apply to the given tag set """ + for key, value in tags.items(): + name_match = self.name.search(key) + if name_match: + if self.value is not None: + value_match = self.value.search(value) + if value_match: + return self._munge_groups(value_match) + else: + return self._munge_groups(name_match) + break + return [] + + def _munge_groups(self, match): + """ Replace backreferences (``$1``, ``$2``) in Group tags with + their values in the regex. """ + rv = [] + sub = match.groups() + for group in self.groups: + newg = group + for idx in range(len(sub)): + newg = newg.replace('$%s' % (idx + 1), sub[idx]) + rv.append(newg) + return rv + + def __str__(self): + if self.value: + return "%s: %s=%s: %s" % (self.__class__.__name__, self.name, + self.value, self.groups) + else: + return "%s: %s: %s" % (self.__class__.__name__, self.name, + self.groups) + + +class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of AWSTags config.xml """ + __identifier__ = None + create = 'AWSTags' + + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + should_monitor=True) + self.core = core + self.tags = [] + + def Index(self): + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() + self.tags = [] + for entry in self.xdata.xpath('//Tag'): + try: + groups = [g.text for g in entry.findall('Group')] + self.tags.append(AWSTagPattern(entry.get("name"), + entry.get("value"), + groups)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to initialize pattern %s: " + "%s" % (entry.get("name"), + sys.exc_info()[1])) + + def get_groups(self, hostname, tags): + """ return a list of groups that should be added to the given + client based on patterns that match the hostname """ + ret = [] + for pattern in self.tags: + try: + ret.extend(pattern.get_groups(tags)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to process pattern %s for " + "%s" % (pattern, hostname), + exc_info=1) + return ret + + +class AWSTags(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ Query tags from AWS via boto, optionally setting group membership """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + key_id = self.core.setup.cfp.get("awstags", "access_key_id") + secret_key = self.core.setup.cfp.get("awstags", + "secret_access_key") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError( + "AWSTags is not configured in bcfg2.conf: %s" % err) + self.debug_log("%s: Connecting to EC2" % self.name) + self._ec2 = connect_ec2(aws_access_key_id=key_id, + aws_secret_access_key=secret_key) + self._tagcache = Cache() + try: + self._keep_cache = self.core.setup.cfp.getboolean("awstags", + "cache") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._keep_cache = True + + self.config = PatternFile(os.path.join(self.data, 'config.xml'), + core=core) + + def _load_instance(self, hostname): + """ Load an instance from EC2 whose private DNS name matches + the given hostname """ + self.debug_log("AWSTags: Loading instance with private-dns-name=%s" % + hostname) + filters = {'private-dns-name': hostname} + reservations = self._ec2.get_all_instances(filters=filters) + if reservations: + res = reservations[0] + if res.instances: + return res.instances[0] + raise NoInstanceFound( + "AWSTags: No instance found with private-dns-name=%s" % + hostname) + + def _get_tags_from_ec2(self, hostname): + """ Get tags for the given host from EC2. This does not use + the local caching layer. """ + self.debug_log("AWSTags: Getting tags for %s from AWS" % + hostname) + try: + return self._load_instance(hostname).tags + except NoInstanceFound: + self.debug_log(sys.exc_info()[1]) + return dict() + + def get_tags(self, metadata): + """ Get tags for the given host. This caches the tags locally + if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is + true. """ + if not self._keep_cache: + return self._get_tags_from_ec2(metadata) + + if metadata.hostname not in self._tagcache: + self._tagcache[metadata.hostname] = \ + self._get_tags_from_ec2(metadata.hostname) + return self._tagcache[metadata.hostname] + + def expire_cache(self, key=None): + self._tagcache.expire(key=key) + + def start_client_run(self, metadata): + self.expire_cache(key=metadata.hostname) + + def get_additional_data(self, metadata): + return self.get_tags(metadata) + + def get_additional_groups(self, metadata): + return self.config.get_groups(metadata.hostname, + self.get_tags(metadata)) diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index ebcab1a6b..06e797ec3 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -41,19 +41,21 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), should_monitor=True) self._local = local() - # building is a thread-local set that tracks which machines - # GroupLogic is getting additional groups for. If a - # get_additional_groups() is called twice for a machine before - # the first call has completed, the second call returns an - # empty list. This is for infinite recursion protection; - # without this check, it'd be impossible to use things like - # metadata.query.in_group() in GroupLogic, since that requires - # building all metadata, which requires running - # GroupLogic.get_additional_groups() for all hosts, which - # requires building all metadata... - self._local.building = set() def get_additional_groups(self, metadata): + if not hasattr(self._local, "building"): + # building is a thread-local set that tracks which + # machines GroupLogic is getting additional groups for. + # If a get_additional_groups() is called twice for a + # machine before the first call has completed, the second + # call returns an empty list. This is for infinite + # recursion protection; without this check, it'd be + # impossible to use things like metadata.query.in_group() + # in GroupLogic, since that requires building all + # metadata, which requires running + # GroupLogic.get_additional_groups() for all hosts, which + # requires building all metadata... + self._local.building = set() if metadata.hostname in self._local.building: return [] self._local.building.add(metadata.hostname) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 073424c90..03f8af719 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -17,7 +17,9 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 +# pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, any, wraps +# pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo # pylint: disable=C0103 @@ -224,6 +226,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) + self.load_xml() def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" @@ -696,14 +699,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node is None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = 'Unexpected error finding %s "%s"' % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4c685d427..195765c69 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -605,7 +605,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ getter for the 'sources' key of the OnDemandDict returned by this function. This delays calling get_collection() until it's absolutely necessary. """ - return self.get_collection(metadata).get_additional_data + return self.get_collection(metadata).get_additional_data() return OnDemandDict( sources=get_sources, diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py new file mode 100644 index 000000000..05e0bb9a1 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py @@ -0,0 +1,140 @@ +import os +import sys +import lxml.etree +import Bcfg2.Server.Plugin +from mock import Mock, MagicMock, patch +try: + from Bcfg2.Server.Plugins.AWSTags import * + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestPlugin import TestPlugin, TestConnector, TestClientRunHooks + +config = ''' +<AWSTags> + <Tag name="name-only"> + <Group>group1</Group> + <Group>group2</Group> + </Tag> + <Tag name="name-and-value" value="value"> + <Group>group3</Group> + </Tag> + <Tag name="regex-(.*)"> + <Group>group-$1</Group> + </Tag> + <Tag name="regex-value" value="(.*)"> + <Group>group-$1</Group> + </Tag> +</AWSTags> +''' + +tags = { + "empty.example.com": {}, + "no-matches.example.com": {"nameonly": "foo", + "Name": "no-matches", + "foo": "bar"}, + "foo.example.com": {"name-only": "name-only", + "name-and-value": "wrong", + "regex-name": "foo"}, + "bar.example.com": {"name-and-value": "value", + "regex-value": "bar"}} + +groups = { + "empty.example.com": [], + "no-matches.example.com": [], + "foo.example.com": ["group1", "group2", "group-name"], + "bar.example.com": ["group3", "group-value", "group-bar"]} + + +def make_instance(name): + rv = Mock() + rv.private_dns_name = name + rv.tags = tags[name] + return rv + + +instances = [make_instance(n) for n in tags.keys()] + + +def get_all_instances(filters=None): + insts = [i for i in instances + if i.private_dns_name == filters['private-dns-name']] + res = Mock() + res.instances = insts + return [res] + + +if HAS_BOTO: + class TestAWSTags(TestPlugin, TestClientRunHooks, TestConnector): + test_obj = AWSTags + + def get_obj(self, core=None): + @patchIf(not isinstance(Bcfg2.Server.Plugins.AWSTags.connect_ec2, + Mock), + "Bcfg2.Server.Plugins.AWSTags.connect_ec2", Mock()) + @patch("lxml.etree.Element", Mock()) + def inner(): + obj = TestPlugin.get_obj(self, core=core) + obj.config.data = config + obj.config.Index() + return obj + return inner() + + @patch("Bcfg2.Server.Plugins.AWSTags.connect_ec2") + def test_connect(self, mock_connect_ec2): + """ Test connection to EC2 """ + key_id = "a09sdbipasdf" + access_key = "oiilb234ipwe9" + + def cfp_get(section, option): + if option == "access_key_id": + return key_id + elif option == "secret_access_key": + return access_key + else: + return Mock() + + core = Mock() + core.setup.cfp.get = Mock(side_effect=cfp_get) + awstags = self.get_obj(core=core) + mock_connect_ec2.assert_called_with( + aws_access_key_id=key_id, + aws_secret_access_key=access_key) + + def test_get_additional_data(self): + """ Test AWSTags.get_additional_data() """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in tags.items(): + metadata = Mock() + metadata.hostname = hostname + self.assertItemsEqual(awstags.get_additional_data(metadata), + expected) + + def test_get_additional_groups_caching(self): + """ Test AWSTags.get_additional_groups() with caching enabled """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in groups.items(): + metadata = Mock() + metadata.hostname = hostname + actual = awstags.get_additional_groups(metadata) + msg = """%s has incorrect groups: +actual: %s +expected: %s""" % (hostname, actual, expected) + self.assertItemsEqual(actual, expected, msg) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index 290edb83a..8814ae171 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -336,6 +336,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): @patch('Bcfg2.Utils.locked', Mock(return_value=False)) @patch('fcntl.lockf', Mock()) + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml") @patch('os.open') @patch('os.fdopen') @patch('os.unlink') @@ -343,7 +344,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): @patch('os.path.islink') @patch('os.readlink') def test_write_xml(self, mock_readlink, mock_islink, mock_rename, - mock_unlink, mock_fdopen, mock_open): + mock_unlink, mock_fdopen, mock_open, mock_load_xml): fname = "clients.xml" config = self.get_obj(fname) fpath = os.path.join(self.metadata.data, fname) @@ -357,6 +358,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): mock_unlink.reset_mock() mock_fdopen.reset_mock() mock_open.reset_mock() + mock_load_xml.reset_mock() mock_islink.return_value = False @@ -368,6 +370,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertTrue(mock_fdopen.return_value.write.called) mock_islink.assert_called_with(fpath) mock_rename.assert_called_with(tmpfile, fpath) + mock_load_xml.assert_called_with() # test: clients.xml.new is locked the first time we write it def rv(fname, mode): @@ -386,6 +389,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertTrue(mock_fdopen.return_value.write.called) mock_islink.assert_called_with(fpath) mock_rename.assert_called_with(tmpfile, fpath) + mock_load_xml.assert_called_with() # test writing a symlinked clients.xml reset() @@ -394,6 +398,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): mock_readlink.return_value = linkdest config.write_xml(fpath, get_clients_test_tree()) mock_rename.assert_called_with(tmpfile, linkdest) + mock_load_xml.assert_called_with() # test failure of os.rename() reset() diff --git a/testsuite/install.sh b/testsuite/install.sh index 6895034d5..8721c8015 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -11,15 +11,17 @@ if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" ]]; then fi if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - pip install --use-mirrors PyYAML pyinotify + pip install --use-mirrors PyYAML pyinotify boto if [[ $PYVER == "2.5" ]]; then # markdown 2.2+ doesn't work on py2.5 - pip install --use-mirrors simplejson 'markdown<2.2' + pip install --use-mirrors simplejson 'markdown<2.2' 'django<1.4.9' + else + pip install 'django<1.5' fi if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits # in bcfg2 require South - pip install cheetah 'django<1.5' 'South<0.8' M2Crypto + pip install cheetah 'South<0.8' M2Crypto fi else # python < 2.6 requires M2Crypto for SSL communication, not just |