diff options
| -rw-r--r-- | doc/development/lint.txt | 5 | ||||
| -rw-r--r-- | doc/server/plugins/connectors/awstags.txt | 124 | ||||
| -rw-r--r-- | misc/bcfg2.spec | 1 | ||||
| -rw-r--r-- | schemas/awstags.xsd | 73 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py | 1 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/AWSTags.py | 217 | ||||
| -rw-r--r-- | testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py | 140 |
7 files changed, 561 insertions, 0 deletions
diff --git a/doc/development/lint.txt b/doc/development/lint.txt index 6a4651f92..6c0be960d 100644 --- a/doc/development/lint.txt +++ b/doc/development/lint.txt @@ -106,6 +106,11 @@ Basics Existing ``bcfg2-lint`` Plugins =============================== +AWSTagsLint +----------- + +.. autoclass:: Bcfg2.Server.Plugins.AWSTags.AWSTagsLint + BundlerLint ----------- diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt new file mode 100644 index 000000000..b884ca065 --- /dev/null +++ b/doc/server/plugins/connectors/awstags.txt @@ -0,0 +1,124 @@ +.. -*- mode: rst -*- + +.. _server-plugins-connectors-awstags: + +========= + AWSTags +========= + +The AWSTags plugin is a connector that retrieves tags from instances +in EC2, and can assign optionally assign +group membership pased on patterns in the tags. See `Using Tags +<http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html>`_ +for details on using tags in EC2. + +AWSTags queries EC2 for instances whose ``private-dns-name`` property +matches the hostname of the client. + +Setup +===== + +#. Add ``AWSTags`` to the ``plugins`` option in ``/etc/bcfg2.conf`` +#. Configure AWS credentials in ``/etc/bcfg2.conf`` (See + `Configuration`_ below for details.) +#. Optionally, create ``AWSTags/config.xml`` (See `Assigning Groups`_ + below for details.) +#. Restart the Bcfg2 server. + +Using Tag Data +============== + +AWSTags exposes the data in templates as a dict available as +``metadata.AWSTags``. E.g., in a :ref:`Genshi template +<server-plugins-generators-cfg-genshi>`, you could do: + +.. code-block:: genshitext + + Known tags on ${metadata.hostname}: + {% for key, val in metadata.AWSTags.items() %}\ + ${key} ${val} + {% end %}\ + +This would produce something like:: + + Known tags on foo.example.com: + Name foo.example.com + some random tag the value + +Assigning Groups +================ + +AWSTags can assign groups based on the tag data. This functionality +is configured in ``AWSTags/config.xml``. + +Example +------- + +.. code-block:: xml + + <AWSTags> + <Tag name="^foo$"> + <Group>foo</Group> + </Tag> + <Tag name="^bar$" value="^bar$"> + <Group>bar</Group> + </Tag> + <Tag name="^bcfg2 group$" value="(.*)"> + <Group>$1</Group> + </Tag> + </AWSTags> + +In this example, any machine with a tag named ``foo`` would be added +to the ``foo`` group. Any machine with a tag named ``bar`` whose +value was also ``bar`` would be added to the ``bar`` group. Finally, +any machine with a tag named ``bcfg2 group`` would be added to the +group named in the value of that tag. + +Note that both the ``name`` and ``value`` attributes are *always* +regular expressions. + +If a ``<Tag/>`` element has only a ``name`` attribute, then it only +checks for existence of a matching tag. If it has both ``name`` and +``value``, then it checks for a matching tag with a matching value. + +You can use backreferences (``$1``, ``$2``, etc.) in the group names. +If only ``name`` is specified, then the backreferences will refer to +groups in the ``name`` regex. If ``name`` and ``value`` are both +specified, then backreferences will refer to groups in the ``value`` +regex. If you specify both ``name`` and ``value``, it is not possible +to refer to groups in the ``name`` regex. + +Schema Reference +---------------- + +.. xml:schema:: awstags.xsd + +Configuration +============= + +AWSTags recognizes several options in ``/etc/bcfg2.conf``; at a +minimum, you must configure an AWS access key ID and secret key. All +of the following options are in the ``[awstags]`` section: + ++-----------------------+-----------------------------------------------------+ +| Option | Description | ++=======================+=====================================================+ +| ``access_key_id`` | The AWS access key ID | ++-----------------------+-----------------------------------------------------+ +| ``secret_access_key`` | The AWS secret access key | ++-----------------------+-----------------------------------------------------+ +| ``cache`` | Whether or not to cache tag lookups. See `Caching`_ | +| | for details. Default is to cache. | ++-----------------------+-----------------------------------------------------+ + +Caching +======= + +Since the AWS API isn't always very quick to respond, AWSTags caches +its results by default. The cache is fairly short-lived: the cache +for each host is expired when it starts a client run, so it will start +the run with fresh data. + +If you frequently update tags on your instances, you may wish to +disable caching. That's probably a bad idea, and would tend to +suggest that updating tags frequently is perhaps the Wrong Thing. diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 47a1a9548..66c0c8de1 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -188,6 +188,7 @@ Group: Documentation/HTML %else Group: Documentation %endif +BuildRequires: python-boto %if 0%{?suse_version} BuildRequires: python-M2Crypto BuildRequires: python-Genshi diff --git a/schemas/awstags.xsd b/schemas/awstags.xsd new file mode 100644 index 000000000..72be0366f --- /dev/null +++ b/schemas/awstags.xsd @@ -0,0 +1,73 @@ +<?xml version="1.0" encoding="utf-8"?> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> + + <xsd:annotation> + <xsd:documentation> + :ref:`AWSTags <server-plugins-connectors-awstags>` config + schema for bcfg2 + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://www.w3.org/XML/1998/namespace" + schemaLocation="xml.xsd"/> + + <xsd:complexType name="TagType"> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Group" type="xsd:string" minOccurs="1" + maxOccurs="unbounded"> + <xsd:annotation> + <xsd:documentation> + The group to assign to machines with tags that match the + enclosing Tag expression. More than one group can be + specified. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + </xsd:choice> + <xsd:attribute name="name" type="xsd:string" use="required"> + <xsd:annotation> + <xsd:documentation> + The name pattern to match against. This is a regular + expression. It is not anchored. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="value" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + The value pattern to match against. This is a regular + expression. It is not anchored. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:complexType name="AWSTagsType"> + <xsd:annotation> + <xsd:documentation> + Top-level tag for ``AWSTags/config.xml``. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Tag" type="TagType"> + <xsd:annotation> + <xsd:documentation> + Representation of a pattern that matches AWS tags. Tags can be + matched in one of two ways: + + * If only :xml:attribute:`TagType:name` is specified, then + AWSTags will only look for a tag with a matching name, and + the value of tags is ignored. + * If both :xml:attribute:`TagType:name` and + :xml:attribute:`TagType:value` are specified, a tag must + have a matching name *and* a matching value. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="AWSTags" type="AWSTagsType"/> + </xsd:choice> + <xsd:attribute ref="xml:base"/> + </xsd:complexType> + + <xsd:element name="AWSTags" type="AWSTagsType"/> +</xsd:schema> diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 09f3f3d25..c537877a0 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -47,6 +47,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Decisions/*.xml": "decisions.xsd", "Packages/sources.xml": "packages.xsd", "GroupPatterns/config.xml": "grouppatterns.xsd", + "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py new file mode 100644 index 000000000..962b5d6a1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -0,0 +1,217 @@ +""" Query tags from AWS via boto, optionally setting group membership """ + +import os +import re +import sys +import Bcfg2.Server.Lint +import Bcfg2.Server.Plugin +from boto import connect_ec2 +from Bcfg2.Cache import Cache +from Bcfg2.Compat import ConfigParser + + +class NoInstanceFound(Exception): + """ Raised when there's no AWS instance for a given hostname """ + + +class AWSTagPattern(object): + """ Handler for a single Tag entry """ + + def __init__(self, name, value, groups): + self.name = re.compile(name) + if value is not None: + self.value = re.compile(value) + else: + self.value = value + self.groups = groups + + def get_groups(self, tags): + """ Get groups that apply to the given tag set """ + for key, value in tags.items(): + name_match = self.name.search(key) + if name_match: + if self.value is not None: + value_match = self.value.search(value) + if value_match: + return self._munge_groups(value_match) + else: + return self._munge_groups(name_match) + break + return [] + + def _munge_groups(self, match): + """ Replace backreferences (``$1``, ``$2``) in Group tags with + their values in the regex. """ + rv = [] + sub = match.groups() + for group in self.groups: + newg = group + for idx in range(len(sub)): + newg = newg.replace('$%s' % (idx + 1), sub[idx]) + rv.append(newg) + return rv + + def __str__(self): + if self.value: + return "%s: %s=%s: %s" % (self.__class__.__name__, self.name, + self.value, self.groups) + else: + return "%s: %s: %s" % (self.__class__.__name__, self.name, + self.groups) + + +class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of AWSTags config.xml """ + __identifier__ = None + create = 'AWSTags' + + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + should_monitor=True) + self.core = core + self.tags = [] + + def Index(self): + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() + self.tags = [] + for entry in self.xdata.xpath('//Tag'): + try: + groups = [g.text for g in entry.findall('Group')] + self.tags.append(AWSTagPattern(entry.get("name"), + entry.get("value"), + groups)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to initialize pattern %s: " + "%s" % (entry.get("name"), + sys.exc_info()[1])) + + def get_groups(self, hostname, tags): + """ return a list of groups that should be added to the given + client based on patterns that match the hostname """ + ret = [] + for pattern in self.tags: + try: + ret.extend(pattern.get_groups(tags)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to process pattern %s for " + "%s" % (pattern, hostname), + exc_info=1) + return ret + + +class AWSTags(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ Query tags from AWS via boto, optionally setting group membership """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + key_id = self.core.setup.cfp.get("awstags", "access_key_id") + secret_key = self.core.setup.cfp.get("awstags", + "secret_access_key") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError( + "AWSTags is not configured in bcfg2.conf: %s" % err) + self.debug_log("%s: Connecting to EC2" % self.name) + self._ec2 = connect_ec2(aws_access_key_id=key_id, + aws_secret_access_key=secret_key) + self._tagcache = Cache() + try: + self._keep_cache = self.core.setup.cfp.getboolean("awstags", + "cache") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._keep_cache = True + + self.config = PatternFile(os.path.join(self.data, 'config.xml'), + core=core) + + def _load_instance(self, hostname): + """ Load an instance from EC2 whose private DNS name matches + the given hostname """ + self.debug_log("AWSTags: Loading instance with private-dns-name=%s" % + hostname) + filters = {'private-dns-name': hostname} + reservations = self._ec2.get_all_instances(filters=filters) + if reservations: + res = reservations[0] + if res.instances: + return res.instances[0] + raise NoInstanceFound( + "AWSTags: No instance found with private-dns-name=%s" % + hostname) + + def _get_tags_from_ec2(self, hostname): + """ Get tags for the given host from EC2. This does not use + the local caching layer. """ + self.debug_log("AWSTags: Getting tags for %s from AWS" % + hostname) + try: + return self._load_instance(hostname).tags + except NoInstanceFound: + self.debug_log(sys.exc_info()[1]) + return dict() + + def get_tags(self, metadata): + """ Get tags for the given host. This caches the tags locally + if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is + true. """ + if not self._keep_cache: + return self._get_tags_from_ec2(metadata) + + if metadata.hostname not in self._tagcache: + self._tagcache[metadata.hostname] = \ + self._get_tags_from_ec2(metadata.hostname) + return self._tagcache[metadata.hostname] + + def expire_cache(self, key=None): + self._tagcache.expire(key=key) + + def start_client_run(self, metadata): + self.expire_cache(self, key=metadata.hostname) + + def get_additional_data(self, metadata): + return self.get_tags(metadata) + + def get_additional_groups(self, metadata): + return self.config.get_groups(metadata.hostname, + self.get_tags(metadata)) + + +class AWSTagsLint(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + <server-plugins-connectors-awstags>` patterns for validity. """ + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py new file mode 100644 index 000000000..05e0bb9a1 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py @@ -0,0 +1,140 @@ +import os +import sys +import lxml.etree +import Bcfg2.Server.Plugin +from mock import Mock, MagicMock, patch +try: + from Bcfg2.Server.Plugins.AWSTags import * + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestPlugin import TestPlugin, TestConnector, TestClientRunHooks + +config = ''' +<AWSTags> + <Tag name="name-only"> + <Group>group1</Group> + <Group>group2</Group> + </Tag> + <Tag name="name-and-value" value="value"> + <Group>group3</Group> + </Tag> + <Tag name="regex-(.*)"> + <Group>group-$1</Group> + </Tag> + <Tag name="regex-value" value="(.*)"> + <Group>group-$1</Group> + </Tag> +</AWSTags> +''' + +tags = { + "empty.example.com": {}, + "no-matches.example.com": {"nameonly": "foo", + "Name": "no-matches", + "foo": "bar"}, + "foo.example.com": {"name-only": "name-only", + "name-and-value": "wrong", + "regex-name": "foo"}, + "bar.example.com": {"name-and-value": "value", + "regex-value": "bar"}} + +groups = { + "empty.example.com": [], + "no-matches.example.com": [], + "foo.example.com": ["group1", "group2", "group-name"], + "bar.example.com": ["group3", "group-value", "group-bar"]} + + +def make_instance(name): + rv = Mock() + rv.private_dns_name = name + rv.tags = tags[name] + return rv + + +instances = [make_instance(n) for n in tags.keys()] + + +def get_all_instances(filters=None): + insts = [i for i in instances + if i.private_dns_name == filters['private-dns-name']] + res = Mock() + res.instances = insts + return [res] + + +if HAS_BOTO: + class TestAWSTags(TestPlugin, TestClientRunHooks, TestConnector): + test_obj = AWSTags + + def get_obj(self, core=None): + @patchIf(not isinstance(Bcfg2.Server.Plugins.AWSTags.connect_ec2, + Mock), + "Bcfg2.Server.Plugins.AWSTags.connect_ec2", Mock()) + @patch("lxml.etree.Element", Mock()) + def inner(): + obj = TestPlugin.get_obj(self, core=core) + obj.config.data = config + obj.config.Index() + return obj + return inner() + + @patch("Bcfg2.Server.Plugins.AWSTags.connect_ec2") + def test_connect(self, mock_connect_ec2): + """ Test connection to EC2 """ + key_id = "a09sdbipasdf" + access_key = "oiilb234ipwe9" + + def cfp_get(section, option): + if option == "access_key_id": + return key_id + elif option == "secret_access_key": + return access_key + else: + return Mock() + + core = Mock() + core.setup.cfp.get = Mock(side_effect=cfp_get) + awstags = self.get_obj(core=core) + mock_connect_ec2.assert_called_with( + aws_access_key_id=key_id, + aws_secret_access_key=access_key) + + def test_get_additional_data(self): + """ Test AWSTags.get_additional_data() """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in tags.items(): + metadata = Mock() + metadata.hostname = hostname + self.assertItemsEqual(awstags.get_additional_data(metadata), + expected) + + def test_get_additional_groups_caching(self): + """ Test AWSTags.get_additional_groups() with caching enabled """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in groups.items(): + metadata = Mock() + metadata.hostname = hostname + actual = awstags.get_additional_groups(metadata) + msg = """%s has incorrect groups: +actual: %s +expected: %s""" % (hostname, actual, expected) + self.assertItemsEqual(actual, expected, msg) |