First pass at Jinja2 support for Cfg.

12 files changed, 306 insertions, 14 deletions

diff --git a/doc/development/cfg.txt b/doc/development/cfg.txt
index f93bb42c7..4e967368b 100644
--- a/doc/development/cfg.txt
+++ b/doc/development/cfg.txt
@@ -64,9 +64,11 @@ Generators
 .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator.CfgPlaintextGenerator
 .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.CfgGenshiGenerator
 .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.CfgCheetahGenerator
+.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.CfgJinja2Generator
 .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.CfgEncryptedGenerator
 .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator.CfgEncryptedGenshiGenerator
 .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator.CfgEncryptedCheetahGenerator
+.. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator.CfgEncryptedJinja2Generator
 .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CfgAuthorizedKeysGenerator
 
 Creators
diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt
index 9ebbd2666..8e0bb7137 100644
--- a/doc/server/plugins/generators/cfg.txt
+++ b/doc/server/plugins/generators/cfg.txt
@@ -30,8 +30,8 @@ in ``Cfg/etc/passwd/passwd``, while the ssh pam module config file,
 ``/etc/pam.d/sshd``, goes in ``Cfg/etc/pam.d/sshd/sshd``. The reason for
 the like-name directory is to allow multiple versions of each file to
 exist, as described below. Note that these files are exact copies of what
-will appear on the client machine (except when using Genshi or Cheetah
-templating -- see below).
+will appear on the client machine (except when using templates -- see 
+below).
 
 Group-Specific Files
 ====================
@@ -242,6 +242,27 @@ comment to appear in the final config file.::
     # This is a comment in my template which will be stripped when it's processed through Cheetah
     \# This comment will appear in the generated config file.
 
+.. _server-plugins-generators-cfg-jinja2:
+
+Jinja2 Templates
+-----------------
+
+Jinja2 templates allow you to use the `jinja2 templating system
+<http://jinja.pocoo.org/>`_.  Jinja2 templates should be
+named with a ``.jinja2`` extension, e.g.::
+
+    % ls Cfg/etc/motd
+    info.xml  motd.jinja2
+
+Examples
+~~~~~~~~
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   examples/jinja2/*
+
 Inside Templates
 ----------------
 
@@ -263,10 +284,10 @@ Several variables are pre-defined inside templates:
 | repo        | The path to the Bcfg2 repository on the filesystem     |
 +-------------+--------------------------------------------------------+
 | path        | In Genshi templates, ``path`` is a synonym for         |
-|             | ``source_path``.  In Cheetah templates, it's a synonym |
-|             | for ``name``.  For this reason, use of ``path`` is     |
-|             | discouraged, and it may be deprecated in a future      |
-|             | release.                                               |
+|             | ``source_path``.  In Cheetah templates and Jinja2      |
+|             | templates, it's a synonym for ``name``.  For this      |
+|             | reason, use of ``path`` is discouraged, and it may be  |
+|             | deprecated in a future release.                                               |
 +-------------+--------------------------------------------------------+
 
 To access these variables in a Genshi template, you can simply use the
@@ -274,6 +295,10 @@ name, e.g.::
 
     Path to this file: ${name}
 
+Similarly, in a Jinja2 template::
+
+	Path to this file: {{ name }}
+
 In a Cheetah template, the variables are properties of ``self``,
 e.g.::
 
@@ -283,15 +308,15 @@ Notes on Using Templates
 ------------------------
 
 Templates can be host and group specific as well.  Deltas will not be
-processed for any Genshi or Cheetah base file.
+processed for any Genshi, Cheetah, or Jinja2 base file.
 
 .. note::
 
     If you are using templating in combination with host-specific
     or group-specific files, you will need to ensure that the ``.genshi``
-    or ``.cheetah`` extension is at the **end** of the filename. Using the
-    examples from above for *host.example.com* and group *server* you would
-    have the following::
+    ``.cheetah`` or ``.jinja2`` extension is at the **end** of the filename. 
+    Using the examples from above for *host.example.com* and group *server*
+    you would have the following::
 
         Cfg/etc/fstab/fstab.H_host.example.com.genshi
         Cfg/etc/fstab/fstab.G50_server.cheetah
@@ -345,7 +370,7 @@ An encrypted file should end with ``.crypt``, e.g.::
     Cfg/etc/foo.conf/foo.conf.crypt
     Cfg/etc/foo.conf/foo.conf.G10_foo.crypt
 
-Encrypted Genshi or Cheetah templates can have the extensions in
+Encrypted Genshi, Cheetah, and Jinja2 templates can have the extensions in
 either order, e.g.::
 
     Cfg/etc/foo.conf/foo.conf.crypt.genshi
diff --git a/doc/server/plugins/generators/examples/jinja2/simple.txt b/doc/server/plugins/generators/examples/jinja2/simple.txt
new file mode 100644
index 000000000..59fc3b89c
--- /dev/null
+++ b/doc/server/plugins/generators/examples/jinja2/simple.txt
@@ -0,0 +1,53 @@
+.. -*- mode: rst -*-
+
+=========================
+ Basic Jinja2 Templates
+=========================
+
+This simple example demonstrates basic usage of Jinja2 templates.
+
+``/var/lib/bcfg2/Cfg/foo/foo.jinja2``
+
+.. code-block:: none
+
+    Hostname is {{ metadata.hostname }}
+    Filename is {{ path }}
+    Template is {{ source_path }}
+    Groups:
+    {% for group in metadata.groups -%}
+     * {{ group }}
+    {% endfor %}
+    Categories:
+    {% for category in metadata.categories -%}
+     * {{ category }} -- {{ metadata.categories[category] }}
+    {% endfor %}
+
+    Probes:
+    {% for probe in metadata.Probes -%}
+     * {{ probe }} -- {{ metadata.Probes[probe] }}
+    {% endfor %}
+
+Output
+======
+
+.. code-block:: xml
+
+    <Path type="file" name="/foo" owner="root" mode="0644" group="root">
+    Hostname is topaz.mcs.anl.gov
+    Filename is /foo
+    Template is /var/lib/bcfg2/Cfg/foo/foo.jinja2
+    Groups:
+     * desktop
+     * mcs-base
+     * ypbound
+     * workstation
+     * xserver
+     * debian-sarge
+     * debian
+     * a
+    Categories:
+     * test -- a
+
+    Probes:
+     * os -- debian
+    </Path>
diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py
index e2d1ec597..fc4506c12 100644
--- a/src/lib/Bcfg2/Server/Lint/Comments.py
+++ b/src/lib/Bcfg2/Server/Lint/Comments.py
@@ -9,6 +9,7 @@ from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \
     import CfgPlaintextGenerator
 from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
 from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator
+from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator
 from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML
 
 
@@ -76,6 +77,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
             type=Bcfg2.Options.Types.comma_list, default=[],
             help="Required comments for Cheetah-templated Cfg files"),
         Bcfg2.Options.Option(
+            cf=("Comments", "jinja2_keywords"),
+            type=Bcfg2.Options.Types.comma_list, default=[],
+            help="Required keywords for Jinja2-templated Cfg files"),
+        Bcfg2.Options.Option(
+            cf=("Comments", "jinja2_comments"),
+            type=Bcfg2.Options.Types.comma_list, default=[],
+            help="Required comments for Jinja2-templated Cfg files"),
+        Bcfg2.Options.Option(
             cf=("Comments", "infoxml_keywords"),
             type=Bcfg2.Options.Types.comma_list, default=[],
             help="Required keywords for info.xml files"),
@@ -235,6 +244,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
                         rtype = "cfg"
                     elif isinstance(entry, CfgCheetahGenerator):
                         rtype = "cheetah"
+                    elif isinstance(entry, CfgJinja2Generator):
+                        rtype = "jinja2"
                     elif isinstance(entry, CfgInfoXML):
                         self.check_xml(entry.infoxml.name,
                                        entry.infoxml.pnode.data,
diff --git a/src/lib/Bcfg2/Server/Lint/Jinja2.py b/src/lib/Bcfg2/Server/Lint/Jinja2.py
new file mode 100755
index 000000000..3112d2a6e
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Lint/Jinja2.py
@@ -0,0 +1,40 @@
+""" Check Jinja2 templates for syntax errors. """
+
+import sys
+import Bcfg2.Server.Lint
+from jinja2 import Template, TemplateSyntaxError
+from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator
+
+
+class Jinja2(Bcfg2.Server.Lint.ServerPlugin):
+    """ Check Jinja2 templates for syntax errors. """
+
+    def Run(self):
+        if 'Cfg' in self.core.plugins:
+            self.check_cfg()
+
+    @classmethod
+    def Errors(cls):
+        return {"jinja2-syntax-error": "error",
+                "unknown-jinja2-error": "error"}
+
+    def check_template(self, entry):
+        """ Generic check for all jinja2 templates (XML and text) """
+        try:
+            Template(entry.data.decode(entry.encoding))
+        except TemplateSyntaxError:
+            err = sys.exc_info()[1]
+            self.LintError("jinja2-syntax-error",
+                           "Jinja2 syntax error in %s: %s" % (entry.name, err))
+        except:
+            err = sys.exc_info()[1]
+            self.LintError("unknown-jinja2-error",
+                           "Unknown Jinja2 error in %s: %s" % (entry.name, err))
+
+    def check_cfg(self):
+        """ Check jinja2 templates in Cfg for syntax errors. """
+        for entryset in self.core.plugins['Cfg'].entries.values():
+            for entry in entryset.entries.values():
+                if (self.HandlesFile(entry.name) and
+                        isinstance(entry, CfgJinja2Generator)):
+                    self.check_template(entry)
diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py
index 202a1487d..5a80a5884 100644
--- a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py
+++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py
@@ -8,16 +8,20 @@ from Bcfg2.Server.Plugin import default_path_metadata
 from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML
 from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
 from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator
+from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator
 from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \
     CfgEncryptedGenshiGenerator
 from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \
     CfgEncryptedCheetahGenerator
+from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import \
+    CfgEncryptedJinja2Generator
 
 
 class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin):
     """ Check for templated scripts or executables. """
-    templates = [CfgGenshiGenerator, CfgCheetahGenerator,
-                 CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator]
+    templates = [CfgGenshiGenerator, CfgCheetahGenerator, CfgJinja2Generator,
+                 CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator,
+                 CfgEncryptedJinja2Generator]
     extensions = [".pl", ".py", ".sh", ".rb"]
 
     def Run(self):
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py
new file mode 100644
index 000000000..c8da84ae0
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py
@@ -0,0 +1,25 @@
+""" Handle encrypted Jinja2 templates (.crypt.jinja2 or
+.jinja2.crypt files)"""
+
+from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator
+from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator \
+    import CfgEncryptedGenerator
+
+
+class CfgEncryptedJinja2Generator(CfgJinja2Generator, CfgEncryptedGenerator):
+    """ CfgEncryptedJinja2Generator lets you encrypt your Jinja2
+    :ref:`server-plugins-generators-cfg` files on the server """
+
+    #: handle .crypt.jinja2 or .jinja2.crypt files
+    __extensions__ = ['jinja2.crypt', 'crypt.jinja2']
+
+    #: Override low priority from parent class
+    __priority__ = 0
+
+    def handle_event(self, event):
+        CfgEncryptedGenerator.handle_event(self, event)
+    handle_event.__doc__ = CfgEncryptedGenerator.handle_event.__doc__
+
+    def get_data(self, entry, metadata):
+        return CfgJinja2Generator.get_data(self, entry, metadata)
+    get_data.__doc__ = CfgJinja2Generator.get_data.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py
new file mode 100644
index 000000000..aaf9f4fc0
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py
@@ -0,0 +1,38 @@
+""" The CfgJinja2Generator allows you to use the `Jinja2
+<http://jinja.pocoo.org/>`_ templating system to generate
+:ref:`server-plugins-generators-cfg` files. """
+
+from Bcfg2.Server.Plugin import PluginExecutionError
+from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP
+
+try:
+    from jinja2 import Template
+    HAS_JINJA2 = True
+except ImportError:
+    HAS_JINJA2 = False
+
+
+class CfgJinja2Generator(CfgGenerator):
+    """ The CfgJinja2Generator allows you to use the `Jinja2
+    <http://jinja.pocoo.org/>`_ templating system to generate
+    :ref:`server-plugins-generators-cfg` files. """
+
+    #: Handle .jinja2 files
+    __extensions__ = ['jinja2']
+
+    #: Low priority to avoid matching host- or group-specific
+    #: .crypt.jinja2 files
+    __priority__ = 50
+
+    def __init__(self, fname, spec, encoding):
+        CfgGenerator.__init__(self, fname, spec, encoding)
+        if not HAS_JINJA2:
+            raise PluginExecutionError("Jinja2 is not available")
+    __init__.__doc__ = CfgGenerator.__init__.__doc__
+
+    def get_data(self, entry, metadata):
+        template = Template(self.data.decode(self.encoding))
+        name = entry.get('realname', entry.get('name'))
+        return template.render(metadata=metadata, name=name, path=name,
+                               source_path=name, repo=SETUP['repo'])
+    get_data.__doc__ = CfgGenerator.get_data.__doc__
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py
new file mode 100644
index 000000000..281ecb2e7
--- /dev/null
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedJinja2Generator.py
@@ -0,0 +1,47 @@
+import os
+import sys
+from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import *
+
+# add all parent testsuite directories to sys.path to allow (most)
+# relative imports in python 2.4
+path = os.path.dirname(__file__)
+while path != "/":
+    if os.path.basename(path).lower().startswith("test"):
+        sys.path.append(path)
+    if os.path.basename(path) == "testsuite":
+        break
+    path = os.path.dirname(path)
+from common import *
+
+try:
+    from TestServer.TestPlugins.TestCfg.TestCfgJinja2Generator import \
+        TestCfgJinja2Generator
+    from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import HAS_JINJA2
+except ImportError:
+    TestCfgJinja2Generator = object
+    HAS_JINJA2 = False
+
+try:
+    from TestServer.TestPlugins.TestCfg.TestCfgEncryptedGenerator import \
+        TestCfgEncryptedGenerator
+    from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import HAS_CRYPTO
+except ImportError:
+    TestCfgEncryptedGenerator = object
+    HAS_CRYPTO = False
+
+
+if can_skip or (HAS_CRYPTO and HAS_JINJA2):
+    class TestCfgEncryptedJinja2Generator(TestCfgJinja2Generator,
+                                           TestCfgEncryptedGenerator):
+        test_obj = CfgEncryptedJinja2Generator
+
+        @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping")
+        @skipUnless(HAS_JINJA2, "Jinja2 libraries not found, skipping")
+        def setUp(self):
+            pass
+
+        def test_handle_event(self):
+            TestCfgEncryptedGenerator.test_handle_event(self)
+
+        def test_get_data(self):
+            TestCfgJinja2Generator.test_get_data(self)
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py
new file mode 100644
index 000000000..333d6b978
--- /dev/null
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgJinja2Generator.py
@@ -0,0 +1,44 @@
+import os
+import sys
+import lxml.etree
+from mock import Mock, MagicMock, patch
+from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import *
+
+# add all parent testsuite directories to sys.path to allow (most)
+# relative imports in python 2.4
+path = os.path.dirname(__file__)
+while path != "/":
+    if os.path.basename(path).lower().startswith("test"):
+        sys.path.append(path)
+    if os.path.basename(path) == "testsuite":
+        break
+    path = os.path.dirname(path)
+from common import *
+from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator
+
+
+if HAS_JINJA2 or can_skip:
+    class TestCfgJinja2Generator(TestCfgGenerator):
+        test_obj = CfgJinja2Generator
+
+        @skipUnless(HAS_JINJA2, "Jinja2 libraries not found, skipping")
+        def setUp(self):
+            pass
+
+        @patch("Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.Template")
+        def test_get_data(self, mock_Template):
+            ccg = self.get_obj(encoding='UTF-8')
+            ccg.data = "data"
+            entry = lxml.etree.Element("Path", name="/test.txt")
+            metadata = Mock()
+            Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.SETUP = MagicMock()
+
+            self.assertEqual(ccg.get_data(entry, metadata),
+                             mock_Template.return_value.render.return_value)
+            Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.SETUP.__getitem__.assert_called_with("repo")
+            mock_Template.assert_called_with("data".decode(ccg.encoding))
+            tmpl = mock_Template.return_value
+            name = entry.get("name")
+            tmpl.render.assert_called_with(metadata=metadata, name=name, path=name,
+                                           source_path=name,
+                                           repo=Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator.SETUP.__getitem__.return_value)
diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py
index 2b8b05926..77b170809 100644
--- a/testsuite/Testsrc/test_code_checks.py
+++ b/testsuite/Testsrc/test_code_checks.py
@@ -38,6 +38,7 @@ contingent_checks = {
     ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]},
     ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]},
     ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]},
+    ("jinja2",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgJinja2Generator.py"]},
     ("M2Crypto",): {"lib/Bcfg2": ["Encryption.py"],
                     "lib/Bcfg2/Server/Plugins/Cfg":
                         ["CfgEncryptedGenerator.py"]},
@@ -45,6 +46,8 @@ contingent_checks = {
                                  ["CfgEncryptedGenshiGenerator.py"]},
     ("M2Crypto", "Cheetah"): {"lib/Bcfg2/Server/Plugins/Cfg":
                                   ["CfgEncryptedCheetahGenerator.py"]},
+    ("M2Crypto", "jinja2"): {"lib/Bcfg2/Server/Plugins/Cfg":
+                                  ["CfgEncryptedJinja2Generator.py"]},
     }
 
 # perform only error checking on the listed files
diff --git a/testsuite/install.sh b/testsuite/install.sh
index 50b91a4d2..6f36d4bef 100755
--- a/testsuite/install.sh
+++ b/testsuite/install.sh
@@ -11,7 +11,7 @@ if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" ]]; then
 fi
 
 if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then
-    pip install --use-mirrors PyYAML pyinotify boto pylibacl 'django<1.5'
+    pip install --use-mirrors PyYAML pyinotify boto pylibacl 'django<1.5' Jinja2
     easy_install https://fedorahosted.org/released/python-augeas/python-augeas-0.4.1.tar.gz
     if [[ ${PYVER:0:1} == "2" ]]; then
         # django supports py3k, but South doesn't, and the django bits