From b5810882e8c6b1e6b76a8239f70a129d415ecee6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 20 Apr 2011 09:41:07 -0400 Subject: Rewrote bcfg2-repo-validate as bcfg2-lint, which uses a plugin interface to be lots more flexible and extensible. Added several more tests. If bcfg2-lint is run as bcfg2-repo-validate, it roughly emulates the functionality of that program. TODO: Need to figure out correct way to symlink bcfg2-repo-validate to bcfg2-lint on install. --- src/lib/Server/Lint/Comments.py | 183 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 src/lib/Server/Lint/Comments.py (limited to 'src/lib/Server/Lint/Comments.py') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py new file mode 100644 index 000000000..0b50df373 --- /dev/null +++ b/src/lib/Server/Lint/Comments.py @@ -0,0 +1,183 @@ +import os.path +import lxml.etree +import Bcfg2.Server.Lint + +class Comments(Bcfg2.Server.Lint.ServerPlugin): + """ check files for various required headers """ + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) + self.config_cache = {} + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + self.check_bundles() + self.check_properties() + self.check_metadata() + self.check_cfg() + self.check_infoxml() + self.check_probes() + + def required_keywords(self, rtype): + """ given a file type, fetch the list of required VCS keywords + from the bcfg2-lint config """ + return self.required_items(rtype, "keyword", default=["Id"]) + + def required_comments(self, rtype): + """ given a file type, fetch the list of required comments + from the bcfg2-lint config """ + return self.required_items(rtype, "comment") + + def required_items(self, rtype, itype, default=None): + """ given a file type and item type (comment or keyword), + fetch the list of required items from the bcfg2-lint config """ + if itype not in self.config_cache: + self.config_cache[itype] = {} + + if rtype not in self.config_cache[itype]: + rv = [] + global_item = "global_%ss" % itype + if global_item in self.config: + rv.extend(self.config[global_item].split(",")) + elif default is not None: + rv.extend(default) + + item = "%s_%ss" % (rtype.lower(), itype) + if item in self.config: + if self.config[item]: + rv.extend(self.config[item].split(",")) + else: + # config explicitly specifies nothing + rv = [] + self.config_cache[itype][rtype] = rv + return self.config_cache[itype][rtype] + + def check_bundles(self): + """ check bundle files for required headers """ + for bundle in self.core.plugins['Bundler'].entries.values(): + xdata = None + rtype = "" + try: + xdata = lxml.etree.XML(bundle.data) + rtype = "bundler" + except AttributeError: + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + rtype = "sgenshi" + + self.check_xml(bundle.name, xdata, rtype) + + def check_properties(self): + """ check properties files for required headers """ + if 'Properties' in self.core.plugins: + props = self.core.plugins['Properties'] + for propfile, pdata in props.store.entries.items(): + if os.path.splitext(propfile)[1] == ".xml": + self.check_xml(pdata.name, pdata.data, 'properties') + + def check_metadata(self): + """ check metadata files for required headers """ + metadata = self.core.plugins['Metadata'] + if self.has_all_xincludes("groups.xml"): + self.check_xml(os.path.join(metadata.data, "groups.xml"), + metadata.groups_xml.data, + "metadata") + if self.has_all_xincludes("clients.xml"): + self.check_xml(os.path.join(metadata.data, "clients.xml"), + metadata.clients_xml.data, + "metadata") + + def check_cfg(self): + """ check Cfg files for required headers """ + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if entry.name.endswith(".genshi"): + rtype = "tgenshi" + else: + rtype = "cfg" + self.check_plaintext(entry.name, entry.data, rtype) + + def check_infoxml(self): + """ check info.xml files for required headers """ + for entryset in self.core.plugins['Cfg'].entries.items(): + if hasattr(entryset, "infoxml") and entryset.infoxml is not None: + self.check_xml(entryset.infoxml.name, + entryset.infoxml.pnode.data, + "infoxml") + + def check_probes(self): + """ check probes for required headers """ + if 'Probes' in self.core.plugins: + for probe in self.core.plugins['Probes'].probes.entries.values(): + self.check_plaintext(probe.name, probe.data, "probes") + + def check_xml(self, filename, xdata, rtype): + """ check generic XML files for required headers """ + self.check_lines(filename, + [str(el) + for el in xdata.getiterator(lxml.etree.Comment)], + rtype) + + def check_plaintext(self, filename, data, rtype): + """ check generic plaintex files for required headers """ + self.check_lines(filename, data.splitlines(), rtype) + + def check_lines(self, filename, lines, rtype): + """ generic header check for a set of lines """ + if self.HandlesFile(filename): + # found is trivalent: + # False == not found + # None == found but not expanded + # True == found and expanded + found = dict((k, False) for k in self.required_keywords(rtype)) + + for line in lines: + # we check for both '$:' and '$$' to see + # if the keyword just hasn't been expanded + for (keyword, status) in found.items(): + if not status: + if '$%s:' % keyword in line: + found[keyword] = True + elif '$%s$' % keyword in line: + found[keyword] = None + + unexpanded = [keyword for (keyword, status) in found.items() + if status is None] + if unexpanded: + self.LintError("%s: Required keywords(s) found but not expanded: %s" % + (filename, ", ".join(unexpanded))) + missing = [keyword for (keyword, status) in found.items() + if status is False] + if missing: + self.LintError("%s: Required keywords(s) not found: %s" % + (filename, ", ".join(missing))) + + # next, check for required comments. found is just + # boolean + found = dict((k, False) for k in self.required_comments(rtype)) + + for line in lines: + for (comment, status) in found.items(): + if not status: + found[comment] = comment in line + + missing = [comment for (comment, status) in found.items() + if status is False] + if missing: + self.LintError("%s: Required comments(s) not found: %s" % + (filename, ", ".join(missing))) + + def has_all_xincludes(self, mfile): + """ return true if self.files includes all XIncludes listed in + the specified metadata type, false otherwise""" + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): + if not self.has_all_xincludes(el.get('href')): + self.LintWarning("Broken XInclude chain: could not include %s" % path) + return False + + return True + -- cgit v1.2.3-1-g7c22 From 24f6403f1ba483b813e30ea15446a59090d90d90 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 21 Apr 2011 14:43:39 -0400 Subject: Misc. bcfg2-lint fixes and tweaks: * fixed bcfg2-lint bug with older pythons * made bcfg2-lint silent by default on success * adjusted bcfg2-lint defaults and alerting levels to work better out-of-the-box --- src/lib/Server/Lint/Comments.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/lib/Server/Lint/Comments.py') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 0b50df373..1b75bb25e 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -20,14 +20,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def required_keywords(self, rtype): """ given a file type, fetch the list of required VCS keywords from the bcfg2-lint config """ - return self.required_items(rtype, "keyword", default=["Id"]) + return self.required_items(rtype, "keyword") def required_comments(self, rtype): """ given a file type, fetch the list of required comments from the bcfg2-lint config """ return self.required_items(rtype, "comment") - def required_items(self, rtype, itype, default=None): + def required_items(self, rtype, itype): """ given a file type and item type (comment or keyword), fetch the list of required items from the bcfg2-lint config """ if itype not in self.config_cache: @@ -38,8 +38,6 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): global_item = "global_%ss" % itype if global_item in self.config: rv.extend(self.config[global_item].split(",")) - elif default is not None: - rv.extend(default) item = "%s_%ss" % (rtype.lower(), itype) if item in self.config: -- cgit v1.2.3-1-g7c22 From 17b8ceb17e0ee775a667d2f92b2b192e567b2df6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Apr 2011 10:45:41 -0400 Subject: Various bcfg2-lint fixes: * check for all plugins before referencing them, since in --stdin mode even plugins like Bundler may not be instantiated * formatting fixes * made Bundles plugin work with or without genshi installed * fixed name of plugin in example bcfg2-lint.conf --- src/lib/Server/Lint/Comments.py | 63 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 30 deletions(-) (limited to 'src/lib/Server/Lint/Comments.py') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 1b75bb25e..587f20603 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -51,17 +51,18 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def check_bundles(self): """ check bundle files for required headers """ - for bundle in self.core.plugins['Bundler'].entries.values(): - xdata = None - rtype = "" - try: - xdata = lxml.etree.XML(bundle.data) - rtype = "bundler" - except AttributeError: - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - rtype = "sgenshi" - - self.check_xml(bundle.name, xdata, rtype) + if 'Bundler' in self.core.plugins: + for bundle in self.core.plugins['Bundler'].entries.values(): + xdata = None + rtype = "" + try: + xdata = lxml.etree.XML(bundle.data) + rtype = "bundler" + except AttributeError: + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + rtype = "sgenshi" + + self.check_xml(bundle.name, xdata, rtype) def check_properties(self): """ check properties files for required headers """ @@ -73,33 +74,35 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def check_metadata(self): """ check metadata files for required headers """ - metadata = self.core.plugins['Metadata'] if self.has_all_xincludes("groups.xml"): - self.check_xml(os.path.join(metadata.data, "groups.xml"), - metadata.groups_xml.data, + self.check_xml(os.path.join(self.metadata.data, "groups.xml"), + self.metadata.groups_xml.data, "metadata") if self.has_all_xincludes("clients.xml"): - self.check_xml(os.path.join(metadata.data, "clients.xml"), - metadata.clients_xml.data, + self.check_xml(os.path.join(self.metadata.data, "clients.xml"), + self.metadata.clients_xml.data, "metadata") def check_cfg(self): """ check Cfg files for required headers """ - for entryset in self.core.plugins['Cfg'].entries.values(): - for entry in entryset.entries.values(): - if entry.name.endswith(".genshi"): - rtype = "tgenshi" - else: - rtype = "cfg" - self.check_plaintext(entry.name, entry.data, rtype) + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if entry.name.endswith(".genshi"): + rtype = "tgenshi" + else: + rtype = "cfg" + self.check_plaintext(entry.name, entry.data, rtype) def check_infoxml(self): """ check info.xml files for required headers """ - for entryset in self.core.plugins['Cfg'].entries.items(): - if hasattr(entryset, "infoxml") and entryset.infoxml is not None: - self.check_xml(entryset.infoxml.name, - entryset.infoxml.pnode.data, - "infoxml") + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.items(): + if (hasattr(entryset, "infoxml") and + entryset.infoxml is not None): + self.check_xml(entryset.infoxml.name, + entryset.infoxml.pnode.data, + "infoxml") def check_probes(self): """ check probes for required headers """ @@ -145,8 +148,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): missing = [keyword for (keyword, status) in found.items() if status is False] if missing: - self.LintError("%s: Required keywords(s) not found: %s" % - (filename, ", ".join(missing))) + self.LintError("%s: Required keywords(s) not found: $%s$" % + (filename, "$, $".join(missing))) # next, check for required comments. found is just # boolean -- cgit v1.2.3-1-g7c22 From 5f7092b061cb200afef2eff2aa39fc150a6ea838 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Apr 2011 10:58:48 -0400 Subject: unexpanded vcs keywords raise warning, not error --- src/lib/Server/Lint/Comments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Server/Lint/Comments.py') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 587f20603..8c83545b3 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -143,8 +143,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): unexpanded = [keyword for (keyword, status) in found.items() if status is None] if unexpanded: - self.LintError("%s: Required keywords(s) found but not expanded: %s" % - (filename, ", ".join(unexpanded))) + self.LintWarning("%s: Required keywords(s) found but not expanded: %s" % + (filename, ", ".join(unexpanded))) missing = [keyword for (keyword, status) in found.items() if status is False] if missing: -- cgit v1.2.3-1-g7c22 From 23ae3d201af82292ad4e939569a50f2e32c689a3 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 5 May 2011 08:16:51 -0400 Subject: made bcfg2-lint error handling configurable on a much more granular level --- src/lib/Server/Lint/Comments.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'src/lib/Server/Lint/Comments.py') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8c83545b3..8e86cc564 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -143,12 +143,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): unexpanded = [keyword for (keyword, status) in found.items() if status is None] if unexpanded: - self.LintWarning("%s: Required keywords(s) found but not expanded: %s" % - (filename, ", ".join(unexpanded))) + self.LintError("unexpanded-keywords", + "%s: Required keywords(s) found but not expanded: %s" % + (filename, ", ".join(unexpanded))) missing = [keyword for (keyword, status) in found.items() if status is False] if missing: - self.LintError("%s: Required keywords(s) not found: $%s$" % + self.LintError("keywords-not-found", + "%s: Required keywords(s) not found: $%s$" % (filename, "$, $".join(missing))) # next, check for required comments. found is just @@ -163,7 +165,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): missing = [comment for (comment, status) in found.items() if status is False] if missing: - self.LintError("%s: Required comments(s) not found: %s" % + self.LintError("comments-not-found", + "%s: Required comments(s) not found: %s" % (filename, ", ".join(missing))) def has_all_xincludes(self, mfile): @@ -177,7 +180,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): xdata = lxml.etree.parse(path) for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): if not self.has_all_xincludes(el.get('href')): - self.LintWarning("Broken XInclude chain: could not include %s" % path) + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not include %s" % path) return False return True -- cgit v1.2.3-1-g7c22