Merge branch 'maint'

Conflicts:
	doc/appendix/guides/fedora.txt
	misc/bcfg2.spec
	schemas/types.xsd
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Options.py
	src/lib/Bcfg2/Server/Admin/Client.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/Lint/Validate.py
	src/lib/Bcfg2/Server/Plugin/helpers.py
	src/lib/Bcfg2/Server/Plugins/Bundler.py
	src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py
	src/lib/Bcfg2/Server/Plugins/Probes.py
	src/sbin/bcfg2-crypt
	testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
	testsuite/common.py
	testsuite/install.sh

3 files changed, 158 insertions, 3 deletions

diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py
new file mode 100644
index 000000000..202a1487d
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py
@@ -0,0 +1,76 @@
+""" Check for templated scripts or executables. """
+
+import os
+import stat
+import Bcfg2.Server.Lint
+from Bcfg2.Compat import any  # pylint: disable=W0622
+from Bcfg2.Server.Plugin import default_path_metadata
+from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML
+from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
+from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator
+from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \
+    CfgEncryptedGenshiGenerator
+from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \
+    CfgEncryptedCheetahGenerator
+
+
+class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin):
+    """ Check for templated scripts or executables. """
+    templates = [CfgGenshiGenerator, CfgCheetahGenerator,
+                 CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator]
+    extensions = [".pl", ".py", ".sh", ".rb"]
+
+    def Run(self):
+        if 'Cfg' in self.core.plugins:
+            for entryset in self.core.plugins['Cfg'].entries.values():
+                for entry in entryset.entries.values():
+                    if (self.HandlesFile(entry.name) and
+                        any(isinstance(entry, t) for t in self.templates)):
+                        self.check_template(entryset, entry)
+
+    @classmethod
+    def Errors(cls):
+        return {"templated-script": "warning",
+                "templated-executable": "warning"}
+
+    def check_template(self, entryset, entry):
+        """ Check a template to see if it's a script or an executable. """
+        # first, check for a known script extension
+        ext = os.path.splitext(entryset.path)[1]
+        if ext in self.extensions:
+            self.LintError("templated-script",
+                           "Templated script found: %s\n"
+                           "File has a known script extension: %s\n"
+                           "Template a config file for the script instead" %
+                           (entry.name, ext))
+            return
+
+        # next, check for a shebang line
+        firstline = open(entry.name).readline()
+        if firstline.startswith("#!"):
+            self.LintError("templated-script",
+                           "Templated script found: %s\n"
+                           "File starts with a shebang: %s\n"
+                           "Template a config file for the script instead" %
+                           (entry.name, firstline))
+            return
+
+        # finally, check for executable permissions in info.xml
+        for entry in entryset.entries.values():
+            if isinstance(entry, CfgInfoXML):
+                for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"):
+                    try:
+                        mode = int(
+                            pinfo.get("mode",
+                                      default_path_metadata()['mode']), 8)
+                    except ValueError:
+                        # LintError will be produced by RequiredAttrs plugin
+                        self.logger.warning("Non-octal mode: %s" % mode)
+                        continue
+                    if mode & stat.S_IXUSR != 0:
+                        self.LintError(
+                            "templated-executable",
+                            "Templated executable found: %s\n"
+                            "Template a config file for the executable instead"
+                            % entry.name)
+                        return
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index e38619355..3ad78ade4 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -113,9 +113,16 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
         :type filename: string
         :returns: lxml.etree._ElementTree - the parsed data"""
         try:
-            return lxml.etree.parse(filename)
-        except SyntaxError:
-            result = self.cmd.run(["xmllint", filename])
+            xdata = lxml.etree.parse(filename)
+            if self.files is None:
+                xdata.xinclude()
+            return xdata
+        except (lxml.etree.XIncludeError, SyntaxError):
+            cmd = ["xmllint", "--noout"]
+            if self.files is None:
+                cmd.append("--xinclude")
+            cmd.append(filename)
+            result = self.cmd.run(cmd)
             self.LintError("xml-failed-to-parse",
                            "%s fails to parse:\n%s" %
                            (filename, result.stdout + result.stderr))
@@ -146,6 +153,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
             if not schema:
                 return False
         datafile = self.parse(filename)
+        if not datafile:
+            return False
         if not schema.validate(datafile):
             cmd = ["xmllint"]
             if self.files is None:
diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
new file mode 100644
index 000000000..04151d764
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
@@ -0,0 +1,70 @@
+"""Ensure that all JSON files in the Bcfg2 repository are
+valid. Currently, the only plugins that uses JSON are Ohai and
+Properties."""
+
+import os
+import sys
+import glob
+import fnmatch
+import Bcfg2.Server.Lint
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+
+class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin):
+    """Ensure that all JSON files in the Bcfg2 repository are
+    valid. Currently, the only plugins that uses JSON are Ohai and
+    Properties. """
+
+    def __init__(self, *args, **kwargs):
+        Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs)
+
+        #: A list of file globs that give the path to JSON files.  The
+        #: globs are extended :mod:`fnmatch` globs that also support
+        #: ``**``, which matches any number of any characters,
+        #: including forward slashes.
+        self.globs = ["Properties/*.json", "Ohai/*.json"]
+        self.files = self.get_files()
+
+    def Run(self):
+        for path in self.files:
+            self.logger.debug("Validating JSON in %s" % path)
+            try:
+                json.load(open(path))
+            except ValueError:
+                self.LintError("json-failed-to-parse",
+                               "%s does not contain valid JSON: %s" %
+                               (path, sys.exc_info()[1]))
+
+    @classmethod
+    def Errors(cls):
+        return {"json-failed-to-parse": "error"}
+
+    def get_files(self):
+        """Return a list of all JSON files to validate, based on
+        :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """
+        if self.files is not None:
+            listfiles = lambda p: fnmatch.filter(self.files,
+                                                 os.path.join('*', p))
+        else:
+            listfiles = lambda p: glob.glob(
+                os.path.join(Bcfg2.Options.setup.repository, p))
+
+        rv = []
+        for path in self.globs:
+            if '/**/' in path:
+                if self.files is not None:
+                    rv.extend(listfiles(path))
+                else:  # self.files is None
+                    fpath, fname = path.split('/**/')
+                    for root, _, files in os.walk(
+                            os.path.join(Bcfg2.Options.setup.repository,
+                                         fpath)):
+                        rv.extend([os.path.join(root, f)
+                                   for f in files if f == fname])
+            else:
+                rv.extend(listfiles(path))
+        return rv