diff options
| -rw-r--r-- | doc/server/plugins/connectors/properties.txt | 17 | ||||
| -rw-r--r-- | doc/server/plugins/probes/index.txt | 43 | ||||
| -rw-r--r-- | src/lib/SSLServer.py | 8 | ||||
| -rw-r--r-- | src/lib/Server/Plugin.py | 92 | ||||
| -rw-r--r-- | src/lib/Server/Plugins/Probes.py | 101 | ||||
| -rw-r--r-- | src/lib/Server/Plugins/Properties.py | 39 |
6 files changed, 198 insertions, 102 deletions
diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt index 1cbc4cf65..3329f48bd 100644 --- a/doc/server/plugins/connectors/properties.txt +++ b/doc/server/plugins/connectors/properties.txt @@ -38,7 +38,7 @@ Usage Specific property files can be referred to in templates as ``metadata.Properties[<filename>]``. The -data attribute is an LXML element object. (Documented +``xdata`` attribute is an LXML element object. (Documented `here <http://codespeak.net/lxml/tutorial.html#the-element-class>`_) Currently, only one access method is defined for this data, ``Match``. @@ -56,10 +56,21 @@ more details on how Group and Client tags are parsed.) For instance:: As we formulate more common use cases, we will add them to the !PropertyFile class as methods. This will simplify templates. +You can also access the XML data that comprises a property file +directly in one of several ways: + +* ``metadata.Properties['prop-file'].xdata`` is an lxml.etree._Element + object representing the top-level element in the file. +* ``metadata.Properties['prop-file'].data`` is the raw contents of the + property file as a string. +* ``metadata.Properties['prop-file'].entries`` is a list of + lxml.etree._Element objects representing the direct children of the + top-level element. (I.e., everything directly under the + ``<Properties>`` tag.) + Accessing Properties contents from TGenshi ========================================== Access contents of ``Properties/auth.xml``:: - - ${metadata.Properties['auth.xml'].data.find('file').find('bcfg2.key').text} + ${metadata.Properties['auth.xml'].xdata.find('file').find('bcfg2.key').text} diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt index b9f698a0f..d52e7bb95 100644 --- a/doc/server/plugins/probes/index.txt +++ b/doc/server/plugins/probes/index.txt @@ -85,10 +85,12 @@ file looks like:: #end if Any Probe script you run will store its output in -``$self.metadata.Probes["scriptname"]``, so we get to our `scratchlocal` -script's output as seen above. Note that we had to wrap the output in an -`int()` call; the script output is treated as a string, so it needs to -be converted before it can be tested numerically. +``$self.metadata.Probes["scriptname"]``, so we get to our +`scratchlocal` script's output as seen above. (See `Handling Probe +Output`_, below, for more information on how this is done.) Note that +we had to wrap the output in an `int()` call; the script output is +treated as a string, so it needs to be converted before it can be +tested numerically. With all of these pieces in place, the following series of events will happen when the client is run: @@ -110,8 +112,37 @@ is to add the ``/etc/auto.master`` to a Bundle: <Path name='/etc/auto.master'/> -Host and Group Specific probes -============================== +Handling Probe Output +===================== + +Bcfg2 stores output from probes in the ``Probes`` property of a +client's metadata object. To access this data in TGenshi, for +instance, you could do:: + + ${metadata.Probes['script-name']} + +This is not the full output of the probe; any lines that start with +"group:" have been stripped from the output. The data is a +string-like object that has some interesting and salient features: + +* If the data is a valid XML document, then + ``metadata.Probes['script-name'].xdata`` will be an + ``lxml.etree._Element`` object representing the XML data. +* If the data is a valid JSON document, and the Python ``json`` module + is installed (included in Python 2.6 onward), then + ``metadata.Probes['script-name'].json`` will be a data structure + representing the JSON data. +* If the data is a valid YAML document, and either the Python ``yaml`` + module or ``syck`` module is installed, then + ``metadata.Probes['script-name'].yaml`` will be a data structure + representing the YAML data. + +If these conditions are not met, then the named properties will be +``None``. In all other fashions, the probe data objects should act +like strings. + +Host- and Group-Specific probes +=============================== Bcfg2 has the ability to alter probes based on client hostname and group membership. These files work similarly to files in Cfg. diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index bb0f61198..d5d2f2b3a 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -195,13 +195,17 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): self.logger.error("No authentication data presented") return False auth_type, auth_content = header.split() - auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) + try: + # py3k compatibility + auth_content = base64.standard_b64decode(auth_content) + except TypeError: + auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) try: # py3k compatibility try: username, password = auth_content.split(":") except TypeError: - username, pw = auth_content.split(b":") + username, pw = auth_content.split(bytes(":", encoding='utf-8')) password = pw.decode('utf-8') except ValueError: username = auth_content diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3a36baf8e..740de247a 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked): def Index(self): """Build local data structures.""" try: - xdata = XML(self.data) + self.xdata = XML(self.data) except XMLSyntaxError: logger.error("Failed to parse %s" % (self.name)) return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() + self.entries = self.xdata.getchildren() + if self.__identifier__ is not None: + self.label = self.xdata.attrib[self.__identifier__] def __iter__(self): return iter(self.entries) @@ -455,53 +456,52 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" + __identifier__ = None + def __init__(self, name): XMLFileBacked.__init__(self, name) - self.fragments = {} - - def Index(self): - """Build internal data structures.""" - try: - xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - logger.error("Failed to parse file %s" % self.name) - return - self.fragments = {} - work = {lambda x: True: xdata.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + self.matches = {} + + def _match(self, item, metadata): + """ recursive helper for Match() """ + if isinstance(item, lxml.etree._Comment): + return [] + elif item.tag == 'Group': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') not in metadata.groups) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') in metadata.groups)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + elif item.tag == 'Client': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') != metadata.hostname) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') == metadata.hostname)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + else: + rv = copy.deepcopy(item) + for child in rv.iterchildren(): + rv.remove(child) + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return [rv] + def Match(self, metadata): """Return matching fragments of independent.""" - matching = [frag for (pred, frag) in list(self.fragments.items()) - if pred(metadata)] - if matching: - return reduce(lambda x, y: x + y, matching) - logger.error("File %s got null match" % (self.name)) - return [] + if metadata.hostname not in self.matches: + rv = [] + for child in self.entries: + rv.extend(self._match(child, metadata)) + if not rv: + logger.error("File %s got null match" % (self.name)) + self.matches[metadata.hostname] = rv + return self.matches[metadata.hostname] class INode: diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 3599f2af1..b07c4dfd3 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -2,12 +2,96 @@ import lxml.etree import operator import re +try: + import json + has_json = True +except ImportError: + has_json = False + +try: + import syck + has_syck = True +except ImportError: + has_syck = False + +try: + import yaml + has_yaml = True +except ImportError: + has_yaml = False + import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P<basename>\S+)") +class ProbeData (object): + """ a ProbeData object emulates a str object, but also has .xdata + and .json properties to provide convenient ways to use ProbeData + objects as XML or JSON data """ + def __init__(self, data): + self.data = data + self._xdata = None + self._json = None + self._yaml = None + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __getattr__(self, name): + """ make ProbeData act like a str object """ + return getattr(self.data, name) + + def __complex__(self): + return complex(self.data) + + def __int__(self): + return int(self.data) + + def __long__(self): + return long(self.data) + + def __float__(self): + return float(self.data) + + @property + def xdata(self): + if self._xdata is None: + try: + self._xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + pass + return self._xdata + + @property + def json(self): + if self._json is None and has_json: + try: + self._json = json.loads(self.data) + except ValueError: + pass + return self._json + + @property + def yaml(self): + if self._yaml is None: + if has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml.YAMLError: + pass + elif has_syck: + try: + self._yaml = syck.load(self.data) + except syck.error: + pass + return self._yaml + + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") @@ -81,7 +165,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, cx = lxml.etree.SubElement(top, 'Client', name=client) for probe in sorted(probed): lxml.etree.SubElement(cx, 'Probe', name=probe, - value=self.probedata[client][probe]) + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) data = lxml.etree.tostring(top, encoding='UTF-8', @@ -106,7 +190,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') + self.probedata[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get('value')) elif (pdata.tag == 'Group'): self.cgroups[client.get('name')].append(pdata.get('name')) @@ -129,9 +214,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.logger.error("Got null response to probe %s from %s" % \ (data.get('name'), client.hostname)) try: - self.probedata[client.hostname].update({data.get('name'): ''}) + self.probedata[client.hostname].update({data.get('name'): + ProbeData('')}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): ''} + self.probedata[client.hostname] = \ + {data.get('name'): ProbeData('')} return dlines = data.text.split('\n') self.logger.debug("%s:probe:%s:%s" % (client.hostname, @@ -142,11 +229,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, if newgroup not in self.cgroups[client.hostname]: self.cgroups[client.hostname].append(newgroup) dlines.remove(line) - dtext = "\n".join(dlines) + dobj = ProbeData("\n".join(dlines)) try: - self.probedata[client.hostname].update({data.get('name'): dtext}) + self.probedata[client.hostname].update({data.get('name'): dobj}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): dtext} + self.probedata[client.hostname] = {data.get('name'): dobj} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index dea797a10..54c5def57 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,44 +6,7 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - def Index(self): - """Build internal data structures.""" - if type(self.data) is not lxml.etree._Element: - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % - self.name) - - self.fragments = {} - work = {lambda x: True: self.data.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - - + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile |