From 504741f24c050684d1707cac057fdd4677afdd64 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 18 Nov 2013 09:19:34 -0500 Subject: bcfg2-lint: resolve XIncludes when parsing XML for validation --- src/lib/Bcfg2/Server/Lint/Validate.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Lint') diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index c537877a0..ced16770b 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -107,9 +107,16 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): :type filename: string :returns: lxml.etree._ElementTree - the parsed data""" try: - return lxml.etree.parse(filename) - except SyntaxError: - lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) + xdata = lxml.etree.parse(filename) + if self.files is None: + xdata.xinclude() + return xdata + except (lxml.etree.XIncludeError, SyntaxError): + cmd = ["xmllint", "--noout"] + if self.files is None: + cmd.append("--xinclude") + cmd.append(filename) + lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) self.LintError("xml-failed-to-parse", "%s fails to parse:\n%s" % (filename, lint.communicate()[0])) @@ -141,6 +148,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if not schema: return False datafile = self.parse(filename) + if not datafile: + return False if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: -- cgit v1.2.3-1-g7c22 From 9a4bc278b4cebf7dfe292951204e760ef70b828e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Sun, 8 Dec 2013 21:21:50 -0500 Subject: bcfg2-lint: New TemplateAbuse plugin detects templated scripts TemplateAbuse detects templated scripts (either files that end with a known extension, or that start with a shebang line) and executables (based off of their permissions in info.xml) and warns about them, since templating scripts is dicey at best, and almost always better done by templating a config file for the script to read instead. --- src/lib/Bcfg2/Server/Lint/TemplateAbuse.py | 75 ++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Lint/TemplateAbuse.py (limited to 'src/lib/Bcfg2/Server/Lint') diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py new file mode 100644 index 000000000..fca9d14a9 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -0,0 +1,75 @@ +""" Check for templated scripts or executables. """ + +import os +import stat +import Bcfg2.Server.Lint +from Bcfg2.Compat import any # pylint: disable=W0622 +from Bcfg2.Server.Plugin import DEFAULT_FILE_METADATA +from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \ + CfgEncryptedGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \ + CfgEncryptedCheetahGenerator + + +class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): + """ Check for templated scripts or executables. """ + templates = [CfgGenshiGenerator, CfgCheetahGenerator, + CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator] + extensions = [".pl", ".py", ".sh", ".rb"] + + def Run(self): + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + any(isinstance(entry, t) for t in self.templates)): + self.check_template(entryset, entry) + + @classmethod + def Errors(cls): + return {"templated-script": "warning", + "templated-executable": "warning"} + + def check_template(self, entryset, entry): + """ Check a template to see if it's a script or an executable. """ + # first, check for a known script extension + ext = os.path.splitext(entryset.path)[1] + if ext in self.extensions: + self.LintError("templated-script", + "Templated script found: %s\n" + "File has a known script extension: %s\n" + "Template a config file for the script instead" % + (entry.name, ext)) + return + + # next, check for a shebang line + firstline = open(entry.name).readline() + if firstline.startswith("#!"): + self.LintError("templated-script", + "Templated script found: %s\n" + "File starts with a shebang: %s\n" + "Template a config file for the script instead" % + (entry.name, firstline)) + return + + # finally, check for executable permissions in info.xml + for entry in entryset.entries.values(): + if isinstance(entry, CfgInfoXML): + for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + try: + mode = int(pinfo.get("mode", + DEFAULT_FILE_METADATA['mode']), 8) + except ValueError: + # LintError will be produced by RequiredAttrs plugin + self.logger.warning("Non-octal mode: %s" % mode) + continue + if mode & stat.S_IXUSR != 0: + self.LintError( + "templated-executable", + "Templated executable found: %s\n" + "Template a config file for the executable instead" + % entry.name) + return -- cgit v1.2.3-1-g7c22 From 14578f009d568850a918930180600807baa6c194 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Sun, 8 Dec 2013 21:57:04 -0500 Subject: bcfg2-lint: New ValidateJSON plugin This plugin validates the JSON files that can be used by Properties and Ohai. --- src/lib/Bcfg2/Server/Lint/ValidateJSON.py | 70 +++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Lint/ValidateJSON.py (limited to 'src/lib/Bcfg2/Server/Lint') diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py new file mode 100644 index 000000000..c4a82a5d2 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -0,0 +1,70 @@ +"""Ensure that all JSON files in the Bcfg2 repository are +valid. Currently, the only plugins that uses JSON are Ohai and +Properties.""" + +import os +import sys +import glob +import fnmatch +import Bcfg2.Server.Lint + +try: + import json +except ImportError: + import simplejson as json + + +class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): + """Ensure that all JSON files in the Bcfg2 repository are + valid. Currently, the only plugins that uses JSON are Ohai and + Properties. """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A list of file globs that give the path to JSON files. The + #: globs are extended :mod:`fnmatch` globs that also support + #: ``**``, which matches any number of any characters, + #: including forward slashes. + self.globs = ["Properties/*.json", "Ohai/*.json"] + self.files = self.get_files() + + def Run(self): + for path in self.files: + self.logger.debug("Validating JSON in %s" % path) + try: + json.load(open(path)) + except ValueError: + self.LintError("json-failed-to-parse", + "%s does not contain valid JSON: %s" % + (path, sys.exc_info()[1])) + + @classmethod + def Errors(cls): + return {"json-failed-to-parse": "error"} + + def get_files(self): + """Return a list of all JSON files to validate, based on + :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """ + if self.files is not None: + listfiles = lambda p: fnmatch.filter(self.files, + os.path.join('*', p)) + else: + listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], + p)) + + rv = [] + for path in self.globs: + if '/**/' in path: + if self.files is not None: + rv.extend(listfiles(path)) + else: # self.files is None + fpath, fname = path.split('/**/') + for root, _, files in \ + os.walk(os.path.join(self.config['repo'], + fpath)): + rv.extend([os.path.join(root, f) + for f in files if f == fname]) + else: + rv.extend(listfiles(path)) + return rv -- cgit v1.2.3-1-g7c22