From 6d4d8df68717780239fad273dd722359db10e64b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 24 Sep 2012 13:07:15 -0400 Subject: expanded pylint tests --- src/lib/Bcfg2/Server/Lint/Comments.py | 38 +++++-------- src/lib/Bcfg2/Server/Lint/Duplicates.py | 33 +++--------- src/lib/Bcfg2/Server/Lint/Genshi.py | 8 ++- src/lib/Bcfg2/Server/Lint/GroupNames.py | 21 ++++++-- src/lib/Bcfg2/Server/Lint/InfoXML.py | 18 ++++--- src/lib/Bcfg2/Server/Lint/MergeFiles.py | 29 ++++++---- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 70 ++++++++++++++---------- src/lib/Bcfg2/Server/Lint/Validate.py | 85 +++++++++++++++-------------- src/lib/Bcfg2/Server/Lint/__init__.py | 87 +++++++++++++++++++----------- 9 files changed, 218 insertions(+), 171 deletions(-) (limited to 'src/lib/Bcfg2/Server/Lint') diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 59d18fc57..bb1217f92 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -1,12 +1,15 @@ +""" check files for various required comments """ + import os import lxml.etree import Bcfg2.Server.Lint -from Bcfg2.Server import XI, XI_NAMESPACE -from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator import CfgPlaintextGenerator +from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ + import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML + class Comments(Bcfg2.Server.Lint.ServerPlugin): """ check files for various required headers """ def __init__(self, *args, **kwargs): @@ -22,10 +25,9 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"unexpanded-keywords":"warning", - "keywords-not-found":"warning", - "comments-not-found":"warning", - "broken-xinclude-chain":"warning"} + return {"unexpanded-keywords": "warning", + "keywords-not-found": "warning", + "comments-not-found": "warning"} def required_keywords(self, rtype): """ given a file type, fetch the list of required VCS keywords @@ -69,7 +71,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): xdata = lxml.etree.XML(bundle.data) rtype = "bundler" except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = lxml.etree.parse(bundle.template.filepath).getroot() + xdata = \ + lxml.etree.parse(bundle.template.filepath).getroot() rtype = "sgenshi" self.check_xml(bundle.name, xdata, rtype) @@ -153,7 +156,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): if status is None] if unexpanded: self.LintError("unexpanded-keywords", - "%s: Required keywords(s) found but not expanded: %s" % + "%s: Required keywords(s) found but not " + "expanded: %s" % (filename, ", ".join(unexpanded))) missing = [keyword for (keyword, status) in found.items() if status is False] @@ -177,21 +181,3 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.LintError("comments-not-found", "%s: Required comments(s) not found: %s" % (filename, ", ".join(missing))) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./%sinclude' % XI_NAMESPACE): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True - diff --git a/src/lib/Bcfg2/Server/Lint/Duplicates.py b/src/lib/Bcfg2/Server/Lint/Duplicates.py index 60a02ffb9..9b36f054b 100644 --- a/src/lib/Bcfg2/Server/Lint/Duplicates.py +++ b/src/lib/Bcfg2/Server/Lint/Duplicates.py @@ -1,10 +1,11 @@ -import os -import lxml.etree +""" Find duplicate clients, groups, etc. """ + import Bcfg2.Server.Lint -from Bcfg2.Server import XI, XI_NAMESPACE + class Duplicates(Bcfg2.Server.Lint.ServerPlugin): """ Find duplicate clients, groups, etc. """ + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.groups_xdata = None @@ -25,11 +26,10 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"broken-xinclude-chain":"warning", - "duplicate-client":"error", - "duplicate-group":"error", - "duplicate-package":"error", - "multiple-default-groups":"error"} + return {"duplicate-client": "error", + "duplicate-group": "error", + "duplicate-package": "error", + "multiple-default-groups": "error"} def load_xdata(self): """ attempt to load XML data for groups and clients. only @@ -71,20 +71,3 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): self.LintError("multiple-default-groups", "Multiple default groups defined: %s" % ",".join(default_groups)) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./%sinclude' % XI_NAMESPACE): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index 74142b446..18b4ae28a 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,9 +1,13 @@ +""" Check Genshi templates for syntax errors """ + import sys import genshi.template import Bcfg2.Server.Lint + class Genshi(Bcfg2.Server.Lint.ServerPlugin): """ Check Genshi templates for syntax errors """ + def Run(self): """ run plugin """ loader = genshi.template.TemplateLoader() @@ -14,9 +18,11 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"genshi-syntax-error":"error"} + return {"genshi-syntax-error": "error"} def check_files(self, entries, loader=None): + """ Check genshi templates in a list of entries for syntax + errors """ if loader is None: loader = genshi.template.TemplateLoader() diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index 5df98a30e..52e42aa7b 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -1,11 +1,14 @@ +""" ensure that all named groups are valid group names """ + import os import re import Bcfg2.Server.Lint try: from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile - has_genshi = True + HAS_GENSHI = True except ImportError: - has_genshi = False + HAS_GENSHI = False + class GroupNames(Bcfg2.Server.Lint.ServerPlugin): """ ensure that all named groups are valid group names """ @@ -28,6 +31,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): return {"invalid-group-name": "error"} def check_rules(self): + """ Check groups used in the Rules plugin for validity """ for rules in self.core.plugins['Rules'].entries.values(): if not self.HandlesFile(rules.name): continue @@ -36,20 +40,23 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): os.path.join(self.config['repo'], rules.name)) def check_bundles(self): - """ check bundles for BoundPath entries with missing attrs """ + """ Check groups used in the Bundler plugin for validity """ for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and - (not has_genshi or + (not HAS_GENSHI or not isinstance(bundle, BundleTemplateFile))): self.check_entries(bundle.xdata.xpath("//Group"), bundle.name) def check_metadata(self): + """ Check groups used or declared in the Metadata plugin for + validity """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), os.path.join(self.config['repo'], self.metadata.groups_xml.name)) def check_grouppatterns(self): + """ Check groups used in the GroupPatterns plugin for validity """ cfg = self.core.plugins['GroupPatterns'].config if not self.HandlesFile(cfg.name): return @@ -60,7 +67,9 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): (cfg.name, self.RenderXML(grp, keep_text=True))) def check_cfg(self): - for root, dirs, files in os.walk(self.core.plugins['Cfg'].data): + """ Check groups used in group-specific files in the Cfg + plugin for validity """ + for root, _, files in os.walk(self.core.plugins['Cfg'].data): for fname in files: basename = os.path.basename(root) if (re.search(r'^%s\.G\d\d_' % basename, fname) and @@ -71,6 +80,8 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): os.path.join(root, fname)) def check_entries(self, entries, fname): + """ Check a generic list of XML entries for tags with + invalid name attributes """ for grp in entries: if not self.valid.search(grp.get("name")): self.LintError("invalid-group-name", diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 5e4e21e18..e34f387ff 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -1,9 +1,12 @@ +""" ensure that all config files have an info.xml file""" + import os import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML from Bcfg2.Server.Plugins.Cfg.CfgLegacyInfo import CfgLegacyInfo + class InfoXML(Bcfg2.Server.Lint.ServerPlugin): """ ensure that all config files have an info.xml file""" def Run(self): @@ -34,13 +37,14 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"no-infoxml":"warning", - "deprecated-info-file":"warning", - "paranoid-false":"warning", - "broken-xinclude-chain":"warning", - "required-infoxml-attrs-missing":"error"} + return {"no-infoxml": "warning", + "deprecated-info-file": "warning", + "paranoid-false": "warning", + "broken-xinclude-chain": "warning", + "required-infoxml-attrs-missing": "error"} def check_infoxml(self, fname, xdata): + """ verify that info.xml contains everything it should """ for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: @@ -50,7 +54,8 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if missing: self.LintError("required-infoxml-attrs-missing", "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), fname, self.RenderXML(info))) + (",".join(missing), fname, + self.RenderXML(info))) if ((Bcfg2.Options.MDATA_PARANOID.value and info.get("paranoid") is not None and @@ -61,4 +66,3 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): self.LintError("paranoid-false", "Paranoid must be true in %s:%s" % (fname, self.RenderXML(info))) - diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 68d010316..44d02c2ff 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -1,9 +1,13 @@ +""" find Probes or Cfg files with multiple similar files that might be +merged into one """ + import os import copy from difflib import SequenceMatcher import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg import CfgGenerator + class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ find Probes or Cfg files with multiple similar files that might be merged into one """ @@ -15,11 +19,11 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"merge-cfg":"warning", - "merge-probes":"warning"} - + return {"merge-cfg": "warning", + "merge-probes": "warning"} def check_cfg(self): + """ check Cfg for similar files """ for filename, entryset in self.core.plugins['Cfg'].entries.items(): candidates = dict([(f, e) for f, e in entryset.entries.items() if isinstance(e, CfgGenerator)]) @@ -32,6 +36,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): for p in mset])) def check_probes(self): + """ check Probes for similar files """ probes = self.core.plugins['Probes'].probes.entries for mset in self.get_similar(probes): self.LintError("merge-probes", @@ -40,6 +45,9 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): ", ".join([p for p in mset])) def get_similar(self, entries): + """ Get a list of similar files from the entry dict. Return + value is a list of lists, each of which gives the filenames of + similar files """ if "threshold" in self.config: # accept threshold either as a percent (e.g., "threshold=75") or # as a ratio (e.g., "threshold=.75") @@ -61,17 +69,20 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): return rv def _find_similar(self, ftuple, others, threshold): + """ Find files similar to the one described by ftupe in the + list of other files. ftuple is a tuple of (filename, data); + others is a list of such tuples. threshold is a float between + 0 and 1 that describes how similar two files much be to rate + as 'similar' """ fname, fdata = ftuple rv = [fname] while others: cname, cdata = others.pop(0) - sm = SequenceMatcher(None, fdata.data, cdata.data) + seqmatch = SequenceMatcher(None, fdata.data, cdata.data) # perform progressively more expensive comparisons - if (sm.real_quick_ratio() > threshold and - sm.quick_ratio() > threshold and - sm.ratio() > threshold): + if (seqmatch.real_quick_ratio() > threshold and + seqmatch.quick_ratio() > threshold and + seqmatch.ratio() > threshold): rv.extend(self._find_similar((cname, cdata), copy.copy(others), threshold)) return rv - - diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index b9d5d79c4..299d6a246 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -1,3 +1,6 @@ +""" verify attributes for configuration entries that cannot be +verified with an XML schema alone""" + import os import re import lxml.etree @@ -7,41 +10,51 @@ import Bcfg2.Client.Tools.VCS from Bcfg2.Server.Plugins.Packages import Apt, Yum try: from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile - has_genshi = True + HAS_GENSHI = True except ImportError: - has_genshi = False + HAS_GENSHI = False + # format verifying functions def is_filename(val): + """ Return True if val is a string describing a valid full path + """ return val.startswith("/") and len(val) > 1 -def is_relative_filename(val): - return len(val) > 1 def is_selinux_type(val): + """ Return True if val is a string describing a valid (although + not necessarily existent) SELinux type """ return re.match(r'^[a-z_]+_t', val) + def is_selinux_user(val): + """ Return True if val is a string describing a valid (although + not necessarily existent) SELinux user """ return re.match(r'^[a-z_]+_u', val) + def is_octal_mode(val): + """ Return True if val is a string describing a valid octal + permissions mode """ return re.match(r'[0-7]{3,4}', val) + def is_username(val): + """ Return True if val is a string giving either a positive + integer uid, or a valid Unix username """ return re.match(r'^([a-z]\w{0,30}|\d+)$', val) + def is_device_mode(val): - try: - # checking upper bound seems like a good way to discover some - # obscure OS with >8-bit device numbers - return int(val) > 0 - except: - return False + """ Return True if val is a string describing a positive integer + """ + return re.match(r'^\d+$', val) class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): - """ verify attributes for configuration entries (as defined in - doc/server/configurationentries) """ + """ verify attributes for configuration entries that cannot be + verified with an XML schema alone """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = dict( @@ -56,7 +69,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): group=is_username, perms=is_octal_mode, __text__=None), hardlink=dict(name=is_filename, to=is_filename), - symlink=dict(name=is_filename, to=is_relative_filename), + symlink=dict(name=is_filename), ignore=dict(name=is_filename), nonexistent=dict(name=is_filename), permissions=dict(name=is_filename, owner=is_username, @@ -83,7 +96,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): access=dict(scope=lambda v: v in ['user', 'group'], perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', v)), - mask=dict(perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', v))), + mask=dict(perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', + v))), Package={None: dict(name=None)}, SELinux=dict( boolean=dict(name=None, @@ -163,15 +177,17 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): if 'Bundler' in self.core.plugins: for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and - (not has_genshi or + (not HAS_GENSHI or not isinstance(bundle, BundleTemplateFile))): try: xdata = lxml.etree.XML(bundle.data) except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = \ - lxml.etree.parse(bundle.template.filepath).getroot() + xdata = lxml.etree.parse( + bundle.template.filepath).getroot() - for path in xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): + for path in xdata.xpath( + "//*[substring(name(), 1, 5) = 'Bound']" + ): self.check_entry(path, bundle.name) def check_entry(self, entry, filename): @@ -219,14 +235,15 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): self.RenderXML(entry))) if not attrs.issuperset(required_attrs.keys()): - self.LintError("required-attrs-missing", - "The following required attribute(s) are " - "missing for %s %s in %s: %s\n%s" % - (tag, name, filename, - ", ".join([attr - for attr in - set(required_attrs.keys()).difference(attrs)]), - self.RenderXML(entry))) + self.LintError( + "required-attrs-missing", + "The following required attribute(s) are missing for %s " + "%s in %s: %s\n%s" % + (tag, name, filename, + ", ".join([attr + for attr in + set(required_attrs.keys()).difference(attrs)]), + self.RenderXML(entry))) for attr, fmt in required_attrs.items(): if fmt and attr in attrs and not fmt(entry.attrib[attr]): @@ -235,4 +252,3 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "malformed\n%s" % (attr, tag, name, filename, self.RenderXML(entry))) - diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index e4c4bddd0..d6fd1df0c 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -1,34 +1,38 @@ +""" Ensure that the repo validates """ + import os import sys import glob import fnmatch import lxml.etree from subprocess import Popen, PIPE, STDOUT -from Bcfg2.Server import XI, XI_NAMESPACE +from Bcfg2.Server import XI_NAMESPACE import Bcfg2.Server.Lint + class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that the repo validates """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) - self.filesets = {"metadata:groups":"%s/metadata.xsd", - "metadata:clients":"%s/clients.xsd", - "info":"%s/info.xsd", - "%s/Bundler/*.xml":"%s/bundle.xsd", - "%s/Bundler/*.genshi":"%s/bundle.xsd", - "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", - "%s/Base/*.xml":"%s/base.xsd", - "%s/Rules/*.xml":"%s/rules.xsd", - "%s/Defaults/*.xml":"%s/defaults.xsd", - "%s/etc/report-configuration.xml":"%s/report-configuration.xsd", - "%s/Deps/*.xml":"%s/deps.xsd", - "%s/Decisions/*.xml":"%s/decisions.xsd", - "%s/Packages/sources.xml":"%s/packages.xsd", - "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", - "%s/FileProbes/config.xml":"%s/fileprobes.xsd", - } + self.filesets = \ + {"metadata:groups": "%s/metadata.xsd", + "metadata:clients": "%s/clients.xsd", + "info": "%s/info.xsd", + "%s/Bundler/*.xml": "%s/bundle.xsd", + "%s/Bundler/*.genshi": "%s/bundle.xsd", + "%s/Pkgmgr/*.xml": "%s/pkglist.xsd", + "%s/Base/*.xml": "%s/base.xsd", + "%s/Rules/*.xml": "%s/rules.xsd", + "%s/Defaults/*.xml": "%s/defaults.xsd", + "%s/etc/report-configuration.xml": "%s/report-configuration.xsd", + "%s/Deps/*.xml": "%s/deps.xsd", + "%s/Decisions/*.xml": "%s/decisions.xsd", + "%s/Packages/sources.xml": "%s/packages.xsd", + "%s/GroupPatterns/config.xml": "%s/grouppatterns.xsd", + "%s/NagiosGen/config.xml": "%s/nagiosgen.xsd", + "%s/FileProbes/config.xml": "%s/fileprobes.xsd", + } self.filelists = {} self.get_filelists() @@ -54,13 +58,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): @classmethod def Errors(cls): - return {"broken-xinclude-chain":"warning", - "schema-failed-to-parse":"warning", - "properties-schema-not-found":"warning", - "xml-failed-to-parse":"error", - "xml-failed-to-read":"error", - "xml-failed-to-verify":"error", - "input-output-error":"error"} + return {"broken-xinclude-chain": "warning", + "schema-failed-to-parse": "warning", + "properties-schema-not-found": "warning", + "xml-failed-to-parse": "error", + "xml-failed-to-read": "error", + "xml-failed-to-verify": "error", + "input-output-error": "error"} def check_properties(self): """ check Properties files against their schemas """ @@ -124,12 +128,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filelists[path] = \ [f for f in self.files if os.path.basename(f) == 'info.xml'] - else: # self.files is None + else: # self.files is None self.filelists[path] = [] - for infodir in ['Cfg', 'TGenshi', 'TCheetah']: - for root, dirs, files in os.walk('%s/%s' % - (self.config['repo'], - infodir)): + for infodir in ['Cfg', 'SSHbase', 'SSLCA', 'TGenshi', + 'TCheetah']: + for root, _, files in \ + os.walk(os.path.join(self.config['repo'], + infodir)): self.filelists[path].extend([os.path.join(root, f) for f in files if f == 'info.xml']) @@ -146,7 +151,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if (fname not in self.filelists['metadata:groups'] and fname not in self.filelists['metadata:clients']): self.LintError("broken-xinclude-chain", - "Broken XInclude chain: Could not determine file type of %s" % fname) + "Broken XInclude chain: Could not determine " + "file type of %s" % fname) def get_metadata_list(self, mtype): """ get all metadata files for the specified type (clients or @@ -163,11 +169,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: rv.extend(self.follow_xinclude(rv[0])) except lxml.etree.XMLSyntaxError: - e = sys.exc_info()[1] + err = sys.exc_info()[1] self.LintError("xml-failed-to-parse", - "%s fails to parse:\n%s" % (rv[0], e)) - - + "%s fails to parse:\n%s" % (rv[0], err)) return rv def follow_xinclude(self, xfile): @@ -193,21 +197,22 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): elif self.HandlesFile(path): rv.append(path) groupdata = lxml.etree.parse(path) - included.update(el for el in groupdata.findall('./%sinclude' % + included.update(el for el in groupdata.findall('./%sinclude' % XI_NAMESPACE)) included.discard(filename) return rv def _load_schema(self, filename): + """ load an XML schema document, returning the Schema object """ try: return lxml.etree.XMLSchema(lxml.etree.parse(filename)) except IOError: - e = sys.exc_info()[1] - self.LintError("input-output-error", str(e)) + err = sys.exc_info()[1] + self.LintError("input-output-error", str(err)) except lxml.etree.XMLSchemaParseError: - e = sys.exc_info()[1] + err = sys.exc_info()[1] self.LintError("schema-failed-to-parse", "Failed to process schema %s: %s" % - (filename, e)) + (filename, err)) return None diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index f4a9b74c2..eea205b75 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -1,51 +1,47 @@ -__all__ = ['Bundler', - 'Comments', - 'Duplicates', - 'InfoXML', - 'MergeFiles', - 'Pkgmgr', - 'RequiredAttrs', - 'Validate', - 'Genshi'] +""" Base classes for Lint plugins and error handling """ -import logging import os import sys +import logging from copy import copy import textwrap import lxml.etree -import Bcfg2.Logger import fcntl import termios import struct +from Bcfg2.Server import XI_NAMESPACE -def _ioctl_GWINSZ(fd): + +def _ioctl_GWINSZ(fd): # pylint: disable=C0103 + """ get a tuple of (height, width) giving the size of the window + from the given file descriptor """ try: - cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - except: + return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + except: # pylint: disable=W0702 return None - return cr + def get_termsize(): """ get a tuple of (width, height) giving the size of the terminal """ if not sys.stdout.isatty(): return None - cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) - if not cr: + dims = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) + if not dims: try: fd = os.open(os.ctermid(), os.O_RDONLY) - cr = _ioctl_GWINSZ(fd) + dims = _ioctl_GWINSZ(fd) os.close(fd) - except: + except: # pylint: disable=W0702 pass - if not cr: + if not dims: try: - cr = (os.environ['LINES'], os.environ['COLUMNS']) + dims = (os.environ['LINES'], os.environ['COLUMNS']) except KeyError: return None - return int(cr[1]), int(cr[0]) + return int(dims[1]), int(dims[0]) -class Plugin (object): + +class Plugin(object): """ base class for ServerlessPlugin and ServerPlugin """ def __init__(self, config, errorhandler=None, files=None): @@ -78,6 +74,7 @@ class Plugin (object): fname)) in self.files) def LintError(self, err, msg): + """ record an error in the lint process """ self.errorhandler.dispatch(err, msg) def RenderXML(self, element, keep_text=False): @@ -88,16 +85,20 @@ class Plugin (object): el = copy(element) if el.text and not keep_text: el.text = '...' - [el.remove(c) for c in el.iterchildren()] - xml = lxml.etree.tostring(el, - xml_declaration=False).decode("UTF-8").strip() + for child in el.iterchildren(): + el.remove(child) + xml = lxml.etree.tostring( + el, + xml_declaration=False).decode("UTF-8").strip() else: - xml = lxml.etree.tostring(element, - xml_declaration=False).decode("UTF-8").strip() + xml = lxml.etree.tostring( + element, + xml_declaration=False).decode("UTF-8").strip() return " line %s: %s" % (element.sourceline, xml) class ErrorHandler (object): + """ a class to handle errors for bcfg2-lint plugins """ def __init__(self, config=None): self.errors = 0 self.warnings = 0 @@ -124,6 +125,8 @@ class ErrorHandler (object): self._handlers[err] = self.debug def RegisterErrors(self, errors): + """ Register a dict of errors (name: default level) that a + plugin may raise """ for err, action in errors.items(): if err not in self._handlers: if "warn" in action: @@ -132,8 +135,9 @@ class ErrorHandler (object): self._handlers[err] = self.error else: self._handlers[err] = self.debug - + def dispatch(self, err, msg): + """ Dispatch an error to the correct handler """ if err in self._handlers: self._handlers[err](msg) self.logger.debug(" (%s)" % err) @@ -157,6 +161,8 @@ class ErrorHandler (object): self._log(msg, self.logger.debug) def _log(self, msg, logfunc, prefix=""): + """ Generic log function that logs a message with the given + function after wrapping it for the terminal width """ # a message may itself consist of multiple lines. wrap() will # elide them all into a single paragraph, which we don't want. # so we split the message into its paragraphs and wrap each @@ -186,8 +192,27 @@ class ServerlessPlugin (Plugin): class ServerPlugin (Plugin): """ base class for plugins that check things that require the running Bcfg2 server """ - def __init__(self, lintCore, config, **kwargs): + def __init__(self, core, config, **kwargs): Plugin.__init__(self, config, **kwargs) - self.core = lintCore + self.core = core self.logger = self.core.logger self.metadata = self.core.metadata + self.errorhandler.RegisterErrors({"broken-xinclude-chain": "warning"}) + + def has_all_xincludes(self, mfile): + """ return true if self.files includes all XIncludes listed in + the specified metadata type, false otherwise""" + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./%sinclude' % XI_NAMESPACE): + if not self.has_all_xincludes(el.get('href')): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not " + "include %s" % path) + return False + + return True -- cgit v1.2.3-1-g7c22