diff options
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/GroupNames.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin/helpers.py | 52 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Bundler.py | 96 | ||||
-rw-r--r-- | testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py | 118 |
5 files changed, 138 insertions, 147 deletions
diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index 4ce12eae7..e41ed867e 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -3,7 +3,6 @@ import os import re import Bcfg2.Server.Lint -from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile class GroupNames(Bcfg2.Server.Lint.ServerPlugin): @@ -38,8 +37,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_bundles(self): """ Check groups used in the Bundler plugin for validity """ for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - not isinstance(bundle, BundleTemplateFile)): + if self.HandlesFile(bundle.name) and bundle.template is None: self.check_entries(bundle.xdata.xpath("//Group"), bundle.name) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index bf72d26d0..60525d5a1 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -3,12 +3,10 @@ verified with an XML schema alone""" import os import re -import lxml.etree import Bcfg2.Server.Lint import Bcfg2.Client.Tools.POSIX import Bcfg2.Client.Tools.VCS from Bcfg2.Server.Plugins.Packages import Apt, Yum -from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile # format verifying functions @@ -178,16 +176,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): return for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - not isinstance(bundle, BundleTemplateFile)): - try: - xdata = lxml.etree.XML(bundle.data) - except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = \ - lxml.etree.parse(bundle.template.filepath).getroot() - - for path in \ - xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): + if self.HandlesFile(bundle.name) and bundle.template is None: + for path in bundle.xdata.xpath( + "//*[substring(name(), 1, 5) = 'Bound']"): self.check_entry(path, bundle.name) def check_entry(self, entry, filename): diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index c85253be6..9d76337e0 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -5,6 +5,7 @@ import re import sys import copy import time +import genshi import logging import operator import lxml.etree @@ -17,7 +18,6 @@ from Bcfg2.Server.Plugin.base import Debuggable, Plugin from Bcfg2.Server.Plugin.interfaces import Generator from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ PluginExecutionError -import genshi.core try: import Bcfg2.Encryption @@ -590,9 +590,31 @@ class StructFile(XMLFileBacked): def __init__(self, filename, should_monitor=False): XMLFileBacked.__init__(self, filename, should_monitor=should_monitor) self.setup = Bcfg2.Options.get_option_parser() + self.encoding = self.setup['encoding'] + self.template = None def Index(self): XMLFileBacked.Index(self) + if (self.name.endswith('.genshi') or + ('py' in self.xdata.nsmap and + self.xdata.nsmap['py'] == 'http://genshi.edgewall.org/')): + try: + loader = genshi.template.TemplateLoader() + self.template = loader.load(self.name, + cls=genshi.template.MarkupTemplate, + encoding=self.encoding) + except LookupError: + err = sys.exc_info()[1] + LOGGER.error('Genshi lookup error in %s: %s' % (self.name, + err)) + except genshi.template.TemplateError: + err = sys.exc_info()[1] + LOGGER.error('Genshi template error in %s: %s' % (self.name, + err)) + except genshi.input.ParseError: + err = sys.exc_info()[1] + LOGGER.error('Genshi parse error in %s: %s' % (self.name, err)) + if self.encryption and HAS_CRYPTO: strict = self.xdata.get( "decrypt", @@ -644,6 +666,21 @@ class StructFile(XMLFileBacked): else: return True + def _render(self, metadata): + """ Render the template for the given client metadata + + :param metadata: Client metadata to match against. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: lxml.etree._Element object representing the rendered + XML data + """ + stream = self.template.generate( + metadata=metadata, + repo=self.setup['repo']).filter(removecomment) + return lxml.etree.XML(stream.render('xml', + strip_whitespace=False), + parser=Bcfg2.Server.XMLParser) + def _match(self, item, metadata): """ recursive helper for Match() """ if self._include_element(item, metadata): @@ -677,7 +714,13 @@ class StructFile(XMLFileBacked): :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: list of lxml.etree._Element objects """ rv = [] - for child in self.entries: + if self.template is None: + entries = self.entries + else: + entries = self._render(metadata).getchildren() + print "rendered: %s" % lxml.etree.tostring(self._render(metadata), + pretty_print=True) + for child in entries: rv.extend(self._match(child, metadata)) return rv @@ -713,7 +756,10 @@ class StructFile(XMLFileBacked): :param metadata: Client metadata to match against. :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: lxml.etree._Element """ - rv = copy.deepcopy(self.xdata) + if self.template is None: + rv = copy.deepcopy(self.xdata) + else: + rv = self._render(metadata) for child in rv.iterchildren(): self._xml_match(child, metadata) return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 3907794ae..051443e22 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -4,101 +4,35 @@ import os import re import sys import copy -import logging import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.Lint -from Bcfg2.Options import get_option_parser - -import genshi.input -from genshi.template import TemplateLoader, MarkupTemplate, TemplateError +from genshi.template import TemplateError class BundleFile(Bcfg2.Server.Plugin.StructFile): """ Representation of a bundle XML file """ def get_xml_value(self, metadata): """ get the XML data that applies to the given client """ - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) + bundle = lxml.etree.Element('Bundle', name=self.xdata.get("name")) for item in self.Match(metadata): bundle.append(copy.copy(item)) return bundle -class BundleTemplateFile(Bcfg2.Server.Plugin.StructFile): - """ Representation of a Genshi-templated bundle XML file """ - - def __init__(self, name, encoding): - Bcfg2.Server.Plugin.StructFile.__init__(self, name) - self.encoding = encoding - self.logger = logging.getLogger(name) - self.template = None - - def HandleEvent(self, event=None): - """Handle all fs events for this template.""" - if event and event.code2str() == 'deleted': - return - try: - loader = TemplateLoader() - self.template = loader.load(self.name, cls=MarkupTemplate, - encoding=self.encoding) - except LookupError: - err = sys.exc_info()[1] - self.logger.error('Genshi lookup error in %s: %s' % - (self.name, err)) - except TemplateError: - err = sys.exc_info()[1] - self.logger.error('Genshi template error in %s: %s' % - (self.name, err)) - except genshi.input.ParseError: - err = sys.exc_info()[1] - self.logger.error('Genshi parse error in %s: %s' % - (self.name, err)) - - def get_xml_value(self, metadata): - """ get the rendered XML data that applies to the given - client """ - if not hasattr(self, 'template'): - msg = "No parsed template information for %s" % self.name - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate( - metadata=metadata, - repo=get_option_parser()['repo'] - ).filter(Bcfg2.Server.Plugin.removecomment) - data = lxml.etree.XML(stream.render('xml', - strip_whitespace=False), - parser=Bcfg2.Server.XMLParser) - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata, data): - bundle.append(copy.deepcopy(item)) - return bundle - - def Match(self, metadata, xdata): # pylint: disable=W0221 - """Return matching fragments of parsed template.""" - rv = [] - for child in xdata.getchildren(): - rv.extend(self._match(child, metadata)) - self.logger.debug("File %s got %d match(es)" % (self.name, - len(rv))) - return rv - - class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Structure, Bcfg2.Server.Plugin.XMLDirectoryBacked): """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' + __child__ = BundleFile patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$') def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) - self.encoding = core.setup['encoding'] - self.__child__ = self.template_dispatch try: Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) except OSError: @@ -107,19 +41,6 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) - def template_dispatch(self, name, _): - """ Add the correct child entry type to Bundler depending on - whether the XML file in question is a plain XML file or a - templated bundle """ - bundle = lxml.etree.parse(name, - parser=Bcfg2.Server.XMLParser) - nsmap = bundle.getroot().nsmap - if (name.endswith('.genshi') or - ('py' in nsmap and nsmap['py'] == 'http://genshi.edgewall.org/')): - return BundleTemplateFile(name, self.encoding) - else: - return BundleFile(name) - def BuildStructures(self, metadata): """Build all structures for client (metadata).""" bundleset = [] @@ -156,8 +77,7 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): """ run plugin """ self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): - if (self.HandlesFile(bundle.name) and - not isinstance(bundle, BundleTemplateFile)): + if self.HandlesFile(bundle.name): self.bundle_names(bundle) @classmethod @@ -186,14 +106,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): def bundle_names(self, bundle): """ verify bundle name attribute matches filename """ - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - fname = os.path.splitext(os.path.basename(bundle.name))[0] - bname = xdata.get('name') + bname = bundle.xdata.get('name') if fname != bname: self.LintError("inconsistent-bundle-name", "Inconsistent bundle name: filename is %s, " diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 0b6f3fd87..0d44e7284 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -1,6 +1,7 @@ import os import sys import copy +import genshi import lxml.etree import Bcfg2.Server import genshi.core @@ -679,10 +680,47 @@ class TestStructFile(TestXMLFileBacked): children[4] = standalone return (xdata, groups, subgroups, children, subchildren, standalone) - def test_Index(self): + def _get_template_test_data(self): + (xdata, groups, subgroups, children, subchildren, standalone) = \ + self._get_test_data() + template_xdata = \ + lxml.etree.Element("Test", name="test", + nsmap=dict(py='http://genshi.edgewall.org/')) + template_xdata.extend(xdata.getchildren()) + return (template_xdata, groups, subgroups, children, subchildren, + standalone) + + @patch("genshi.template.TemplateLoader") + def test_Index(self, mock_TemplateLoader): has_crypto = Bcfg2.Server.Plugin.helpers.HAS_CRYPTO Bcfg2.Server.Plugin.helpers.HAS_CRYPTO = False TestXMLFileBacked.test_Index(self) + + sf = self.get_obj() + sf.encoding = Mock() + (xdata, groups, subgroups, children, subchildren, standalone) = \ + self._get_test_data() + sf.data = lxml.etree.tostring(xdata) + + mock_TemplateLoader.reset_mock() + sf.Index() + self.assertFalse(mock_TemplateLoader.called) + + mock_TemplateLoader.reset_mock() + template_xdata = \ + lxml.etree.Element("Test", name="test", + nsmap=dict(py='http://genshi.edgewall.org/')) + template_xdata.extend(xdata.getchildren()) + sf.data = lxml.etree.tostring(template_xdata) + sf.Index() + mock_TemplateLoader.assert_called_with() + loader = mock_TemplateLoader.return_value + loader.load.assert_called_with(sf.name, + cls=genshi.template.MarkupTemplate, + encoding=sf.encoding) + self.assertEqual(sf.template, + loader.load.return_value) + Bcfg2.Server.Plugin.helpers.HAS_CRYPTO = has_crypto @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") @@ -813,16 +851,15 @@ class TestStructFile(TestXMLFileBacked): self.assertTrue(inc("Other")) - @patch("Bcfg2.Server.Plugin.helpers.%s._include_element" % - test_obj.__name__) - def test__match(self, mock_include): + def test__match(self): sf = self.get_obj() + sf._include_element = Mock() metadata = Mock() (xdata, groups, subgroups, children, subchildren, standalone) = \ self._get_test_data() - mock_include.side_effect = \ + sf._include_element.side_effect = \ lambda x, _: (x.tag not in ['Client', 'Group'] or x.get("include") == "true") @@ -842,46 +879,51 @@ class TestStructFile(TestXMLFileBacked): for el in standalone: self.assertXMLEqual(el, sf._match(el, metadata)[0]) - @patch("Bcfg2.Server.Plugin.helpers.%s._match" % test_obj.__name__) - def test_Match(self, mock_match): + def test_Match(self): sf = self.get_obj() + sf._match = Mock() metadata = Mock() - (xdata, groups, subgroups, children, subchildren, standalone) = \ - self._get_test_data() - sf.entries.extend(copy.deepcopy(xdata).getchildren()) - - def match_rv(el, _): - if el.tag not in ['Client', 'Group']: - return [el] - elif x.get("include") == "true": - return el.getchildren() - else: - return [] - mock_match.side_effect = match_rv - actual = sf.Match(metadata) - expected = reduce(lambda x, y: x + y, - list(children.values()) + list(subgroups.values())) - self.assertEqual(len(actual), len(expected)) - # easiest way to compare the values is actually to make - # them into an XML document and let assertXMLEqual compare - # them - xactual = lxml.etree.Element("Container") - xactual.extend(actual) - xexpected = lxml.etree.Element("Container") - xexpected.extend(expected) - self.assertXMLEqual(xactual, xexpected) + for test_data in [self._get_test_data(), + self._get_template_test_data()]: + (xdata, groups, subgroups, children, subchildren, standalone) = \ + test_data + sf.data = lxml.etree.tostring(xdata) + sf.Index() + + def match_rv(el, _): + if el.tag not in ['Client', 'Group']: + return [el] + elif el.get("include") == "true": + return el.getchildren() + else: + return [] + sf._match.side_effect = match_rv + actual = sf.Match(metadata) + expected = reduce(lambda x, y: x + y, + list(children.values()) + list(subgroups.values())) + print "doc: %s" % lxml.etree.tostring(xdata, pretty_print=True) + print "actual: %s" % [lxml.etree.tostring(el) for el in actual] + print "expected: %s" % [lxml.etree.tostring(el) for el in expected] + self.assertEqual(len(actual), len(expected)) + # easiest way to compare the values is actually to make + # them into an XML document and let assertXMLEqual compare + # them + xactual = lxml.etree.Element("Container") + xactual.extend(actual) + xexpected = lxml.etree.Element("Container") + xexpected.extend(expected) + self.assertXMLEqual(xactual, xexpected) - @patch("Bcfg2.Server.Plugin.helpers.%s._include_element" % - test_obj.__name__) - def test__xml_match(self, mock_include): + def test__xml_match(self): sf = self.get_obj() + sf._include_element = Mock() metadata = Mock() (xdata, groups, subgroups, children, subchildren, standalone) = \ self._get_test_data() - mock_include.side_effect = \ + sf._include_element.side_effect = \ lambda x, _: (x.tag not in ['Client', 'Group'] or x.get("include") == "true") @@ -895,9 +937,9 @@ class TestStructFile(TestXMLFileBacked): expected.extend(standalone) self.assertXMLEqual(actual, expected) - @patch("Bcfg2.Server.Plugin.helpers.%s._xml_match" % test_obj.__name__) - def test_Match(self, mock_xml_match): + def test_XMLMatch(self): sf = self.get_obj() + sf._xml_match = Mock() metadata = Mock() (sf.xdata, groups, subgroups, children, subchildren, standalone) = \ @@ -905,7 +947,7 @@ class TestStructFile(TestXMLFileBacked): sf.XMLMatch(metadata) actual = [] - for call in mock_xml_match.call_args_list: + for call in sf._xml_match.call_args_list: actual.append(call[0][0]) self.assertEqual(call[0][1], metadata) expected = list(groups.values()) + standalone |