added genshi support to StructFile

5 files changed, 138 insertions, 147 deletions

diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py
index 4ce12eae7..e41ed867e 100644
--- a/src/lib/Bcfg2/Server/Lint/GroupNames.py
+++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py
@@ -3,7 +3,6 @@
 import os
 import re
 import Bcfg2.Server.Lint
-from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile
 
 
 class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
@@ -38,8 +37,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
     def check_bundles(self):
         """ Check groups used in the Bundler plugin for validity """
         for bundle in self.core.plugins['Bundler'].entries.values():
-            if (self.HandlesFile(bundle.name) and
-                not isinstance(bundle, BundleTemplateFile)):
+            if self.HandlesFile(bundle.name) and bundle.template is None:
                 self.check_entries(bundle.xdata.xpath("//Group"),
                                    bundle.name)
 
diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
index bf72d26d0..60525d5a1 100644
--- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
+++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
@@ -3,12 +3,10 @@ verified with an XML schema alone"""
 
 import os
 import re
-import lxml.etree
 import Bcfg2.Server.Lint
 import Bcfg2.Client.Tools.POSIX
 import Bcfg2.Client.Tools.VCS
 from Bcfg2.Server.Plugins.Packages import Apt, Yum
-from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile
 
 
 # format verifying functions
@@ -178,16 +176,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
             return
 
         for bundle in self.core.plugins['Bundler'].entries.values():
-            if (self.HandlesFile(bundle.name) and
-                 not isinstance(bundle, BundleTemplateFile)):
-                try:
-                    xdata = lxml.etree.XML(bundle.data)
-                except (lxml.etree.XMLSyntaxError, AttributeError):
-                    xdata = \
-                        lxml.etree.parse(bundle.template.filepath).getroot()
-
-                for path in \
-                        xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"):
+            if self.HandlesFile(bundle.name) and bundle.template is None:
+                for path in bundle.xdata.xpath(
+                    "//*[substring(name(), 1, 5) = 'Bound']"):
                     self.check_entry(path, bundle.name)
 
     def check_entry(self, entry, filename):
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index c85253be6..9d76337e0 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -5,6 +5,7 @@ import re
 import sys
 import copy
 import time
+import genshi
 import logging
 import operator
 import lxml.etree
@@ -17,7 +18,6 @@ from Bcfg2.Server.Plugin.base import Debuggable, Plugin
 from Bcfg2.Server.Plugin.interfaces import Generator
 from Bcfg2.Server.Plugin.exceptions import SpecificityError, \
     PluginExecutionError
-import genshi.core
 
 try:
     import Bcfg2.Encryption
@@ -590,9 +590,31 @@ class StructFile(XMLFileBacked):
     def __init__(self, filename, should_monitor=False):
         XMLFileBacked.__init__(self, filename, should_monitor=should_monitor)
         self.setup = Bcfg2.Options.get_option_parser()
+        self.encoding = self.setup['encoding']
+        self.template = None
 
     def Index(self):
         XMLFileBacked.Index(self)
+        if (self.name.endswith('.genshi') or
+            ('py' in self.xdata.nsmap and
+             self.xdata.nsmap['py'] == 'http://genshi.edgewall.org/')):
+            try:
+                loader = genshi.template.TemplateLoader()
+                self.template = loader.load(self.name,
+                                            cls=genshi.template.MarkupTemplate,
+                                            encoding=self.encoding)
+            except LookupError:
+                err = sys.exc_info()[1]
+                LOGGER.error('Genshi lookup error in %s: %s' % (self.name,
+                                                                err))
+            except genshi.template.TemplateError:
+                err = sys.exc_info()[1]
+                LOGGER.error('Genshi template error in %s: %s' % (self.name,
+                                                                  err))
+            except genshi.input.ParseError:
+                err = sys.exc_info()[1]
+                LOGGER.error('Genshi parse error in %s: %s' % (self.name, err))
+
         if self.encryption and HAS_CRYPTO:
             strict = self.xdata.get(
                 "decrypt",
@@ -644,6 +666,21 @@ class StructFile(XMLFileBacked):
         else:
             return True
 
+    def _render(self, metadata):
+        """ Render the template for the given client metadata
+
+        :param metadata: Client metadata to match against.
+        :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+        :returns: lxml.etree._Element object representing the rendered
+                  XML data
+        """
+        stream = self.template.generate(
+            metadata=metadata,
+            repo=self.setup['repo']).filter(removecomment)
+        return lxml.etree.XML(stream.render('xml',
+                                            strip_whitespace=False),
+                              parser=Bcfg2.Server.XMLParser)
+
     def _match(self, item, metadata):
         """ recursive helper for Match() """
         if self._include_element(item, metadata):
@@ -677,7 +714,13 @@ class StructFile(XMLFileBacked):
         :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
         :returns: list of lxml.etree._Element objects """
         rv = []
-        for child in self.entries:
+        if self.template is None:
+            entries = self.entries
+        else:
+            entries = self._render(metadata).getchildren()
+            print "rendered: %s" % lxml.etree.tostring(self._render(metadata),
+                                                       pretty_print=True)
+        for child in entries:
             rv.extend(self._match(child, metadata))
         return rv
 
@@ -713,7 +756,10 @@ class StructFile(XMLFileBacked):
         :param metadata: Client metadata to match against.
         :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
         :returns: lxml.etree._Element """
-        rv = copy.deepcopy(self.xdata)
+        if self.template is None:
+            rv = copy.deepcopy(self.xdata)
+        else:
+            rv = self._render(metadata)
         for child in rv.iterchildren():
             self._xml_match(child, metadata)
         return rv
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index 3907794ae..051443e22 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -4,101 +4,35 @@ import os
 import re
 import sys
 import copy
-import logging
 import lxml.etree
 import Bcfg2.Server
 import Bcfg2.Server.Plugin
 import Bcfg2.Server.Lint
-from Bcfg2.Options import get_option_parser
-
-import genshi.input
-from genshi.template import TemplateLoader, MarkupTemplate, TemplateError
+from genshi.template import TemplateError
 
 
 class BundleFile(Bcfg2.Server.Plugin.StructFile):
     """ Representation of a bundle XML file """
     def get_xml_value(self, metadata):
         """ get the XML data that applies to the given client """
-        bundlename = os.path.splitext(os.path.basename(self.name))[0]
-        bundle = lxml.etree.Element('Bundle', name=bundlename)
+        bundle = lxml.etree.Element('Bundle', name=self.xdata.get("name"))
         for item in self.Match(metadata):
             bundle.append(copy.copy(item))
         return bundle
 
 
-class BundleTemplateFile(Bcfg2.Server.Plugin.StructFile):
-    """ Representation of a Genshi-templated bundle XML file """
-
-    def __init__(self, name, encoding):
-        Bcfg2.Server.Plugin.StructFile.__init__(self, name)
-        self.encoding = encoding
-        self.logger = logging.getLogger(name)
-        self.template = None
-
-    def HandleEvent(self, event=None):
-        """Handle all fs events for this template."""
-        if event and event.code2str() == 'deleted':
-            return
-        try:
-            loader = TemplateLoader()
-            self.template = loader.load(self.name, cls=MarkupTemplate,
-                                        encoding=self.encoding)
-        except LookupError:
-            err = sys.exc_info()[1]
-            self.logger.error('Genshi lookup error in %s: %s' %
-                              (self.name, err))
-        except TemplateError:
-            err = sys.exc_info()[1]
-            self.logger.error('Genshi template error in %s: %s' %
-                              (self.name, err))
-        except genshi.input.ParseError:
-            err = sys.exc_info()[1]
-            self.logger.error('Genshi parse error in %s: %s' %
-                              (self.name, err))
-
-    def get_xml_value(self, metadata):
-        """ get the rendered XML data that applies to the given
-        client """
-        if not hasattr(self, 'template'):
-            msg = "No parsed template information for %s" % self.name
-            self.logger.error(msg)
-            raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
-        stream = self.template.generate(
-            metadata=metadata,
-            repo=get_option_parser()['repo']
-            ).filter(Bcfg2.Server.Plugin.removecomment)
-        data = lxml.etree.XML(stream.render('xml',
-                                            strip_whitespace=False),
-                              parser=Bcfg2.Server.XMLParser)
-        bundlename = os.path.splitext(os.path.basename(self.name))[0]
-        bundle = lxml.etree.Element('Bundle', name=bundlename)
-        for item in self.Match(metadata, data):
-            bundle.append(copy.deepcopy(item))
-        return bundle
-
-    def Match(self, metadata, xdata):  # pylint: disable=W0221
-        """Return matching fragments of parsed template."""
-        rv = []
-        for child in xdata.getchildren():
-            rv.extend(self._match(child, metadata))
-        self.logger.debug("File %s got %d match(es)" % (self.name,
-                                                        len(rv)))
-        return rv
-
-
 class Bundler(Bcfg2.Server.Plugin.Plugin,
               Bcfg2.Server.Plugin.Structure,
               Bcfg2.Server.Plugin.XMLDirectoryBacked):
     """ The bundler creates dependent clauses based on the
     bundle/translation scheme from Bcfg1. """
     __author__ = 'bcfg-dev@mcs.anl.gov'
+    __child__ = BundleFile
     patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$')
 
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Structure.__init__(self)
-        self.encoding = core.setup['encoding']
-        self.__child__ = self.template_dispatch
         try:
             Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data)
         except OSError:
@@ -107,19 +41,6 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
             self.logger.error(msg)
             raise Bcfg2.Server.Plugin.PluginInitError(msg)
 
-    def template_dispatch(self, name, _):
-        """ Add the correct child entry type to Bundler depending on
-        whether the XML file in question is a plain XML file or a
-        templated bundle """
-        bundle = lxml.etree.parse(name,
-                                  parser=Bcfg2.Server.XMLParser)
-        nsmap = bundle.getroot().nsmap
-        if (name.endswith('.genshi') or
-            ('py' in nsmap and nsmap['py'] == 'http://genshi.edgewall.org/')):
-            return BundleTemplateFile(name, self.encoding)
-        else:
-            return BundleFile(name)
-
     def BuildStructures(self, metadata):
         """Build all structures for client (metadata)."""
         bundleset = []
@@ -156,8 +77,7 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
         """ run plugin """
         self.missing_bundles()
         for bundle in self.core.plugins['Bundler'].entries.values():
-            if (self.HandlesFile(bundle.name) and
-                not isinstance(bundle, BundleTemplateFile)):
+            if self.HandlesFile(bundle.name):
                 self.bundle_names(bundle)
 
     @classmethod
@@ -186,14 +106,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
 
     def bundle_names(self, bundle):
         """ verify bundle name attribute matches filename """
-        try:
-            xdata = lxml.etree.XML(bundle.data)
-        except AttributeError:
-            # genshi template
-            xdata = lxml.etree.parse(bundle.template.filepath).getroot()
-
         fname = os.path.splitext(os.path.basename(bundle.name))[0]
-        bname = xdata.get('name')
+        bname = bundle.xdata.get('name')
         if fname != bname:
             self.LintError("inconsistent-bundle-name",
                            "Inconsistent bundle name: filename is %s, "
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
index 0b6f3fd87..0d44e7284 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
@@ -1,6 +1,7 @@
 import os
 import sys
 import copy
+import genshi
 import lxml.etree
 import Bcfg2.Server
 import genshi.core
@@ -679,10 +680,47 @@ class TestStructFile(TestXMLFileBacked):
         children[4] = standalone
         return (xdata, groups, subgroups, children, subchildren, standalone)
 
-    def test_Index(self):
+    def _get_template_test_data(self):
+        (xdata, groups, subgroups, children, subchildren, standalone) = \
+            self._get_test_data()
+        template_xdata = \
+            lxml.etree.Element("Test", name="test",
+                               nsmap=dict(py='http://genshi.edgewall.org/'))
+        template_xdata.extend(xdata.getchildren())
+        return (template_xdata, groups, subgroups, children, subchildren,
+                standalone)
+
+    @patch("genshi.template.TemplateLoader")
+    def test_Index(self, mock_TemplateLoader):
         has_crypto = Bcfg2.Server.Plugin.helpers.HAS_CRYPTO
         Bcfg2.Server.Plugin.helpers.HAS_CRYPTO = False
         TestXMLFileBacked.test_Index(self)
+
+        sf = self.get_obj()
+        sf.encoding = Mock()
+        (xdata, groups, subgroups, children, subchildren, standalone) = \
+            self._get_test_data()
+        sf.data = lxml.etree.tostring(xdata)
+
+        mock_TemplateLoader.reset_mock()
+        sf.Index()
+        self.assertFalse(mock_TemplateLoader.called)
+
+        mock_TemplateLoader.reset_mock()
+        template_xdata = \
+            lxml.etree.Element("Test", name="test",
+                               nsmap=dict(py='http://genshi.edgewall.org/'))
+        template_xdata.extend(xdata.getchildren())
+        sf.data = lxml.etree.tostring(template_xdata)
+        sf.Index()
+        mock_TemplateLoader.assert_called_with()
+        loader = mock_TemplateLoader.return_value
+        loader.load.assert_called_with(sf.name,
+                                       cls=genshi.template.MarkupTemplate,
+                                       encoding=sf.encoding)
+        self.assertEqual(sf.template,
+                         loader.load.return_value)
+
         Bcfg2.Server.Plugin.helpers.HAS_CRYPTO = has_crypto
 
     @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping")
@@ -813,16 +851,15 @@ class TestStructFile(TestXMLFileBacked):
 
         self.assertTrue(inc("Other"))
 
-    @patch("Bcfg2.Server.Plugin.helpers.%s._include_element" %
-           test_obj.__name__)
-    def test__match(self, mock_include):
+    def test__match(self):
         sf = self.get_obj()
+        sf._include_element = Mock()
         metadata = Mock()
 
         (xdata, groups, subgroups, children, subchildren, standalone) = \
             self._get_test_data()
 
-        mock_include.side_effect = \
+        sf._include_element.side_effect = \
             lambda x, _: (x.tag not in ['Client', 'Group'] or
                           x.get("include") == "true")
 
@@ -842,46 +879,51 @@ class TestStructFile(TestXMLFileBacked):
         for el in standalone:
             self.assertXMLEqual(el, sf._match(el, metadata)[0])
 
-    @patch("Bcfg2.Server.Plugin.helpers.%s._match" % test_obj.__name__)
-    def test_Match(self, mock_match):
+    def test_Match(self):
         sf = self.get_obj()
+        sf._match = Mock()
         metadata = Mock()
 
-        (xdata, groups, subgroups, children, subchildren, standalone) = \
-            self._get_test_data()
-        sf.entries.extend(copy.deepcopy(xdata).getchildren())
-
-        def match_rv(el, _):
-            if el.tag not in ['Client', 'Group']:
-                return [el]
-            elif x.get("include") == "true":
-                return el.getchildren()
-            else:
-                return []
-        mock_match.side_effect = match_rv
-        actual = sf.Match(metadata)
-        expected = reduce(lambda x, y: x + y,
-                          list(children.values()) + list(subgroups.values()))
-        self.assertEqual(len(actual), len(expected))
-        # easiest way to compare the values is actually to make
-        # them into an XML document and let assertXMLEqual compare
-        # them
-        xactual = lxml.etree.Element("Container")
-        xactual.extend(actual)
-        xexpected = lxml.etree.Element("Container")
-        xexpected.extend(expected)
-        self.assertXMLEqual(xactual, xexpected)
+        for test_data in [self._get_test_data(),
+                          self._get_template_test_data()]:
+            (xdata, groups, subgroups, children, subchildren, standalone) = \
+                test_data
+            sf.data = lxml.etree.tostring(xdata)
+            sf.Index()
+
+            def match_rv(el, _):
+                if el.tag not in ['Client', 'Group']:
+                    return [el]
+                elif el.get("include") == "true":
+                    return el.getchildren()
+                else:
+                    return []
+            sf._match.side_effect = match_rv
+            actual = sf.Match(metadata)
+            expected = reduce(lambda x, y: x + y,
+                              list(children.values()) + list(subgroups.values()))
+            print "doc: %s" % lxml.etree.tostring(xdata, pretty_print=True)
+            print "actual: %s" % [lxml.etree.tostring(el) for el in actual]
+            print "expected: %s" % [lxml.etree.tostring(el) for el in expected]
+            self.assertEqual(len(actual), len(expected))
+            # easiest way to compare the values is actually to make
+            # them into an XML document and let assertXMLEqual compare
+            # them
+            xactual = lxml.etree.Element("Container")
+            xactual.extend(actual)
+            xexpected = lxml.etree.Element("Container")
+            xexpected.extend(expected)
+            self.assertXMLEqual(xactual, xexpected)
 
-    @patch("Bcfg2.Server.Plugin.helpers.%s._include_element" %
-           test_obj.__name__)
-    def test__xml_match(self, mock_include):
+    def test__xml_match(self):
         sf = self.get_obj()
+        sf._include_element = Mock()
         metadata = Mock()
 
         (xdata, groups, subgroups, children, subchildren, standalone) = \
             self._get_test_data()
 
-        mock_include.side_effect = \
+        sf._include_element.side_effect = \
             lambda x, _: (x.tag not in ['Client', 'Group'] or
                           x.get("include") == "true")
 
@@ -895,9 +937,9 @@ class TestStructFile(TestXMLFileBacked):
         expected.extend(standalone)
         self.assertXMLEqual(actual, expected)
 
-    @patch("Bcfg2.Server.Plugin.helpers.%s._xml_match" % test_obj.__name__)
-    def test_Match(self, mock_xml_match):
+    def test_XMLMatch(self):
         sf = self.get_obj()
+        sf._xml_match = Mock()
         metadata = Mock()
 
         (sf.xdata, groups, subgroups, children, subchildren, standalone) = \
@@ -905,7 +947,7 @@ class TestStructFile(TestXMLFileBacked):
 
         sf.XMLMatch(metadata)
         actual = []
-        for call in mock_xml_match.call_args_list:
+        for call in sf._xml_match.call_args_list:
             actual.append(call[0][0])
             self.assertEqual(call[0][1], metadata)
         expected = list(groups.values()) + standalone