diff options
Diffstat (limited to 'src/lib/Server/Lint')
| -rw-r--r-- | src/lib/Server/Lint/Bundles.py | 61 | ||||
| -rw-r--r-- | src/lib/Server/Lint/Comments.py | 187 | ||||
| -rw-r--r-- | src/lib/Server/Lint/Deltas.py | 19 | ||||
| -rw-r--r-- | src/lib/Server/Lint/Duplicates.py | 81 | ||||
| -rwxr-xr-x | src/lib/Server/Lint/Genshi.py | 28 | ||||
| -rw-r--r-- | src/lib/Server/Lint/GroupPatterns.py | 31 | ||||
| -rw-r--r-- | src/lib/Server/Lint/InfoXML.py | 42 | ||||
| -rw-r--r-- | src/lib/Server/Lint/MergeFiles.py | 69 | ||||
| -rw-r--r-- | src/lib/Server/Lint/Pkgmgr.py | 35 | ||||
| -rw-r--r-- | src/lib/Server/Lint/RequiredAttrs.py | 138 | ||||
| -rw-r--r-- | src/lib/Server/Lint/Validate.py | 200 | ||||
| -rw-r--r-- | src/lib/Server/Lint/__init__.py | 218 |
12 files changed, 12 insertions, 1097 deletions
diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py deleted file mode 100644 index 472915cfd..000000000 --- a/src/lib/Server/Lint/Bundles.py +++ /dev/null @@ -1,61 +0,0 @@ -import lxml.etree -import Bcfg2.Server.Lint - -class Bundles(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ - - def Run(self): - """ run plugin """ - if 'Bundler' in self.core.plugins: - self.missing_bundles() - for bundle in self.core.plugins['Bundler'].entries.values(): - if self.HandlesFile(bundle.name): - if (not Bcfg2.Server.Plugins.Bundler.have_genshi or - type(bundle) is not - Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): - self.bundle_names(bundle) - - def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ - if self.files is None: - # when given a list of files on stdin, this check is - # useless, so skip it - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) - - allbundles = self.core.plugins['Bundler'].entries.keys() - for bundle in ref_bundles: - xmlbundle = "%s.xml" % bundle - genshibundle = "%s.genshi" % bundle - if (xmlbundle not in allbundles and - genshibundle not in allbundles): - self.LintError("bundle-not-found", - "Bundle %s referenced, but does not exist" % - bundle) - - def bundle_names(self, bundle): - """ verify bundle name attribute matches filename """ - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - - fname = bundle.name.split('Bundler/')[1].split('.')[0] - bname = xdata.get('name') - if fname != bname: - self.LintError("inconsistent-bundle-name", - "Inconsistent bundle name: filename is %s, bundle name is %s" % - (fname, bname)) - - def sgenshi_groups(self, bundle): - """ ensure that Genshi Bundles do not include <Group> tags, - which are not supported """ - xdata = lxml.etree.parse(bundle.name) - groups = [self.RenderXML(g) - for g in xdata.getroottree().findall("//Group")] - if groups: - self.LintError("group-tag-not-allowed", - "<Group> tag is not allowed in SGenshi Bundle:\n%s" % - "\n".join(groups)) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py deleted file mode 100644 index 19fae1b08..000000000 --- a/src/lib/Server/Lint/Comments.py +++ /dev/null @@ -1,187 +0,0 @@ -import os.path -import lxml.etree -import Bcfg2.Server.Lint - -class Comments(Bcfg2.Server.Lint.ServerPlugin): - """ check files for various required headers """ - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.config_cache = {} - - def Run(self): - self.check_bundles() - self.check_properties() - self.check_metadata() - self.check_cfg() - self.check_infoxml() - self.check_probes() - - def required_keywords(self, rtype): - """ given a file type, fetch the list of required VCS keywords - from the bcfg2-lint config """ - return self.required_items(rtype, "keyword") - - def required_comments(self, rtype): - """ given a file type, fetch the list of required comments - from the bcfg2-lint config """ - return self.required_items(rtype, "comment") - - def required_items(self, rtype, itype): - """ given a file type and item type (comment or keyword), - fetch the list of required items from the bcfg2-lint config """ - if itype not in self.config_cache: - self.config_cache[itype] = {} - - if rtype not in self.config_cache[itype]: - rv = [] - global_item = "global_%ss" % itype - if global_item in self.config: - rv.extend(self.config[global_item].split(",")) - - item = "%s_%ss" % (rtype.lower(), itype) - if item in self.config: - if self.config[item]: - rv.extend(self.config[item].split(",")) - else: - # config explicitly specifies nothing - rv = [] - self.config_cache[itype][rtype] = rv - return self.config_cache[itype][rtype] - - def check_bundles(self): - """ check bundle files for required headers """ - if 'Bundler' in self.core.plugins: - for bundle in self.core.plugins['Bundler'].entries.values(): - xdata = None - rtype = "" - try: - xdata = lxml.etree.XML(bundle.data) - rtype = "bundler" - except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - rtype = "sgenshi" - - self.check_xml(bundle.name, xdata, rtype) - - def check_properties(self): - """ check properties files for required headers """ - if 'Properties' in self.core.plugins: - props = self.core.plugins['Properties'] - for propfile, pdata in props.store.entries.items(): - if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.xdata, 'properties') - - def check_metadata(self): - """ check metadata files for required headers """ - if self.has_all_xincludes("groups.xml"): - self.check_xml(os.path.join(self.metadata.data, "groups.xml"), - self.metadata.groups_xml.data, - "metadata") - if self.has_all_xincludes("clients.xml"): - self.check_xml(os.path.join(self.metadata.data, "clients.xml"), - self.metadata.clients_xml.data, - "metadata") - - def check_cfg(self): - """ check Cfg files for required headers """ - if 'Cfg' in self.core.plugins: - for entryset in self.core.plugins['Cfg'].entries.values(): - for entry in entryset.entries.values(): - if entry.name.endswith(".genshi"): - rtype = "tgenshi" - else: - rtype = "cfg" - self.check_plaintext(entry.name, entry.data, rtype) - - def check_infoxml(self): - """ check info.xml files for required headers """ - if 'Cfg' in self.core.plugins: - for entryset in self.core.plugins['Cfg'].entries.items(): - if (hasattr(entryset, "infoxml") and - entryset.infoxml is not None): - self.check_xml(entryset.infoxml.name, - entryset.infoxml.pnode.data, - "infoxml") - - def check_probes(self): - """ check probes for required headers """ - if 'Probes' in self.core.plugins: - for probe in self.core.plugins['Probes'].probes.entries.values(): - self.check_plaintext(probe.name, probe.data, "probes") - - def check_xml(self, filename, xdata, rtype): - """ check generic XML files for required headers """ - self.check_lines(filename, - [str(el) - for el in xdata.getiterator(lxml.etree.Comment)], - rtype) - - def check_plaintext(self, filename, data, rtype): - """ check generic plaintex files for required headers """ - self.check_lines(filename, data.splitlines(), rtype) - - def check_lines(self, filename, lines, rtype): - """ generic header check for a set of lines """ - if self.HandlesFile(filename): - # found is trivalent: - # False == not found - # None == found but not expanded - # True == found and expanded - found = dict((k, False) for k in self.required_keywords(rtype)) - - for line in lines: - # we check for both '$<keyword>:' and '$<keyword>$' to see - # if the keyword just hasn't been expanded - for (keyword, status) in found.items(): - if not status: - if '$%s:' % keyword in line: - found[keyword] = True - elif '$%s$' % keyword in line: - found[keyword] = None - - unexpanded = [keyword for (keyword, status) in found.items() - if status is None] - if unexpanded: - self.LintError("unexpanded-keywords", - "%s: Required keywords(s) found but not expanded: %s" % - (filename, ", ".join(unexpanded))) - missing = [keyword for (keyword, status) in found.items() - if status is False] - if missing: - self.LintError("keywords-not-found", - "%s: Required keywords(s) not found: $%s$" % - (filename, "$, $".join(missing))) - - # next, check for required comments. found is just - # boolean - found = dict((k, False) for k in self.required_comments(rtype)) - - for line in lines: - for (comment, status) in found.items(): - if not status: - found[comment] = comment in line - - missing = [comment for (comment, status) in found.items() - if status is False] - if missing: - self.LintError("comments-not-found", - "%s: Required comments(s) not found: %s" % - (filename, ", ".join(missing))) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True - diff --git a/src/lib/Server/Lint/Deltas.py b/src/lib/Server/Lint/Deltas.py index cf91d1d13..114f2e348 100644 --- a/src/lib/Server/Lint/Deltas.py +++ b/src/lib/Server/Lint/Deltas.py @@ -1,4 +1,5 @@ import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.Cfg import CfgFilter class Deltas(Bcfg2.Server.Lint.ServerPlugin): """ Warn about usage of .cat and .diff files """ @@ -10,11 +11,15 @@ class Deltas(Bcfg2.Server.Lint.ServerPlugin): for basename, entry in list(cfg.entries.items()): self.check_entry(basename, entry) + @classmethod + def Errors(cls): + return {"cat-file-used":"warning", + "diff-file-used":"warning"} + def check_entry(self, basename, entry): - for fname in list(entry.entries.keys()): - if self.HandlesFile(fname): - match = entry.specific.delta_reg.match(fname) - if match: - self.LintError("%s-file-used" % match.group('delta'), - "%s file used on %s: %s" % - (match.group('delta'), basename, fname)) + for fname, processor in entry.entries.items(): + if self.HandlesFile(fname) and isinstance(processor, CfgFilter): + extension = fname.split(".")[-1] + self.LintError("%s-file-used" % extension, + "%s file used on %s: %s" % + (extension, basename, fname)) diff --git a/src/lib/Server/Lint/Duplicates.py b/src/lib/Server/Lint/Duplicates.py deleted file mode 100644 index 75f620603..000000000 --- a/src/lib/Server/Lint/Duplicates.py +++ /dev/null @@ -1,81 +0,0 @@ -import os.path -import lxml.etree -import Bcfg2.Server.Lint - -class Duplicates(Bcfg2.Server.Lint.ServerPlugin): - """ Find duplicate clients, groups, etc. """ - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.groups_xdata = None - self.clients_xdata = None - self.load_xdata() - - def Run(self): - """ run plugin """ - # only run this plugin if we were not given a list of files. - # not only is it marginally silly to run this plugin with a - # partial list of files, it turns out to be really freaking - # hard to get only a fragment of group or client metadata - if self.groups_xdata is not None: - self.duplicate_groups() - self.duplicate_defaults() - if self.clients_xdata is not None: - self.duplicate_clients() - - def load_xdata(self): - """ attempt to load XML data for groups and clients. only - actually load data if all documents reference in XIncludes can - be found in self.files""" - if self.has_all_xincludes("groups.xml"): - self.groups_xdata = self.metadata.clients_xml.xdata - if self.has_all_xincludes("clients.xml"): - self.clients_xdata = self.metadata.clients_xml.xdata - - def duplicate_groups(self): - """ find duplicate groups """ - self.duplicate_entries(self.clients_xdata.xpath('//Groups/Group'), - 'group') - - def duplicate_clients(self): - """ find duplicate clients """ - self.duplicate_entries(self.clients_xdata.xpath('//Clients/Client'), - 'client') - - def duplicate_entries(self, data, etype): - """ generic duplicate entry finder """ - seen = {} - for el in data: - if el.get('name') not in seen: - seen[el.get('name')] = el - else: - self.LintError("duplicate-%s" % etype, - "Duplicate %s '%s':\n%s\n%s" % - (etype, el.get('name'), - self.RenderXML(seen[el.get('name')]), - self.RenderXML(el))) - - def duplicate_defaults(self): - """ check for multiple default group definitions """ - default_groups = [g for g in self.groups_xdata.findall('.//Group') - if g.get('default') == 'true'] - if len(default_groups) > 1: - self.LintError("multiple-default-groups", - "Multiple default groups defined: %s" % - ",".join(default_groups)) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True diff --git a/src/lib/Server/Lint/Genshi.py b/src/lib/Server/Lint/Genshi.py deleted file mode 100755 index 56803246c..000000000 --- a/src/lib/Server/Lint/Genshi.py +++ /dev/null @@ -1,28 +0,0 @@ -import genshi.template -import Bcfg2.Server.Lint - -class Genshi(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ - - def Run(self): - """ run plugin """ - loader = genshi.template.TemplateLoader() - for plugin in ['Cfg', 'TGenshi']: - if plugin in self.core.plugins: - self.check_files(self.core.plugins[plugin].entries, - loader=loader) - - def check_files(self, entries, loader=None): - if loader is None: - loader = genshi.template.TemplateLoader() - - for eset in entries.values(): - for fname, sdata in list(eset.entries.items()): - if (self.HandlesFile(fname) and - (fname.endswith(".genshi") or fname.endswith(".newtxt"))): - try: - loader.load(sdata.name, - cls=genshi.template.NewTextTemplate) - except genshi.template.TemplateSyntaxError, err: - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) diff --git a/src/lib/Server/Lint/GroupPatterns.py b/src/lib/Server/Lint/GroupPatterns.py deleted file mode 100644 index b69d7a5d8..000000000 --- a/src/lib/Server/Lint/GroupPatterns.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import Bcfg2.Server.Lint -from Bcfg2.Server.Plugins.GroupPatterns import PatternMap - -class GroupPatterns(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ - - def Run(self): - """ run plugin """ - if 'GroupPatterns' in self.core.plugins: - cfg = self.core.plugins['GroupPatterns'].config - for entry in cfg.xdata.xpath('//GroupPattern'): - groups = [g.text for g in entry.findall('Group')] - self.check(entry, groups, ptype='NamePattern') - self.check(entry, groups, ptype='NameRange') - - def check(self, entry, groups, ptype="NamePattern"): - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - - for el in entry.findall(ptype): - pat = el.text - try: - pmap(pat) - except: - err = sys.exc_info()[1] - self.LintError("pattern-fails-to-initialize", - "Failed to initialize %s %s for %s: %s" % - (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py deleted file mode 100644 index 2054e23bf..000000000 --- a/src/lib/Server/Lint/InfoXML.py +++ /dev/null @@ -1,42 +0,0 @@ -import os.path -import Bcfg2.Options -import Bcfg2.Server.Lint - -class InfoXML(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all config files have an info.xml file""" - - def Run(self): - if 'Cfg' in self.core.plugins: - for filename, entryset in self.core.plugins['Cfg'].entries.items(): - infoxml_fname = os.path.join(entryset.path, "info.xml") - if self.HandlesFile(infoxml_fname): - if (hasattr(entryset, "infoxml") and - entryset.infoxml is not None): - self.check_infoxml(infoxml_fname, - entryset.infoxml.pnode.data) - else: - self.LintError("no-infoxml", - "No info.xml found for %s" % filename) - - def check_infoxml(self, fname, xdata): - for info in xdata.getroottree().findall("//Info"): - required = [] - if "required_attrs" in self.config: - required = self.config["required_attrs"].split(",") - - missing = [attr for attr in required if info.get(attr) is None] - if missing: - self.LintError("required-infoxml-attrs-missing", - "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), fname, self.RenderXML(info))) - - if ((Bcfg2.Options.MDATA_PARANOID.value and - info.get("paranoid") is not None and - info.get("paranoid").lower() == "false") or - (not Bcfg2.Options.MDATA_PARANOID.value and - (info.get("paranoid") is None or - info.get("paranoid").lower() != "true"))): - self.LintError("paranoid-false", - "Paranoid must be true in %s:%s" % - (fname, self.RenderXML(info))) - diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py deleted file mode 100644 index ff6e3449a..000000000 --- a/src/lib/Server/Lint/MergeFiles.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import copy -from difflib import SequenceMatcher -import Bcfg2.Server.Lint - -class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): - """ find Probes or Cfg files with multiple similar files that - might be merged into one """ - - def Run(self): - if 'Cfg' in self.core.plugins: - self.check_cfg() - if 'Probes' in self.core.plugins: - self.check_probes() - - def check_cfg(self): - for filename, entryset in self.core.plugins['Cfg'].entries.items(): - for mset in self.get_similar(entryset.entries): - self.LintError("merge-cfg", - "The following files are similar: %s. " - "Consider merging them into a single Genshi " - "template." % - ", ".join([os.path.join(filename, p) - for p in mset])) - - def check_probes(self): - probes = self.core.plugins['Probes'].probes.entries - for mset in self.get_similar(probes): - self.LintError("merge-cfg", - "The following probes are similar: %s. " - "Consider merging them into a single probe." % - ", ".join([p for p in mset])) - - def get_similar(self, entries): - if "threshold" in self.config: - # accept threshold either as a percent (e.g., "threshold=75") or - # as a ratio (e.g., "threshold=.75") - threshold = float(self.config['threshold']) - if threshold > 1: - threshold /= 100 - else: - threshold = 0.75 - rv = [] - elist = entries.items() - while elist: - result = self._find_similar(elist.pop(0), copy.copy(elist), - threshold) - if len(result) > 1: - elist = [(fname, fdata) - for fname, fdata in elist - if fname not in result] - rv.append(result) - return rv - - def _find_similar(self, ftuple, others, threshold): - fname, fdata = ftuple - rv = [fname] - while others: - cname, cdata = others.pop(0) - sm = SequenceMatcher(None, fdata.data, cdata.data) - # perform progressively more expensive comparisons - if (sm.real_quick_ratio() > threshold and - sm.quick_ratio() > threshold and - sm.ratio() > threshold): - rv.extend(self._find_similar((cname, cdata), copy.copy(others), - threshold)) - return rv - - diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Server/Lint/Pkgmgr.py deleted file mode 100644 index 8f099163a..000000000 --- a/src/lib/Server/Lint/Pkgmgr.py +++ /dev/null @@ -1,35 +0,0 @@ -import glob -import lxml.etree -import Bcfg2.Server.Lint - -class Pkgmgr(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ - - def Run(self): - pset = set() - for pfile in glob.glob("%s/Pkgmgr/*.xml" % self.config['repo']): - if self.HandlesFile(pfile): - xdata = lxml.etree.parse(pfile).getroot() - # get priority, type, group - priority = xdata.get('priority') - ptype = xdata.get('type') - for pkg in xdata.xpath("//Package"): - if pkg.getparent().tag == 'Group': - grp = pkg.getparent().get('name') - if (type(grp) is not str and - grp.getparent().tag == 'Group'): - pgrp = grp.getparent().get('name') - else: - pgrp = 'none' - else: - grp = 'none' - pgrp = 'none' - ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) - # check if package is already listed with same - # priority, type, grp - if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) - else: - pset.add(ptuple) diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Server/Lint/RequiredAttrs.py deleted file mode 100644 index 55206d2ba..000000000 --- a/src/lib/Server/Lint/RequiredAttrs.py +++ /dev/null @@ -1,138 +0,0 @@ -import os.path -import lxml.etree -import Bcfg2.Server.Lint -from Bcfg2.Server.Plugins.Packages import Apt, Yum - -class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): - """ verify attributes for configuration entries (as defined in - doc/server/configurationentries) """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.required_attrs = { - 'Path': { - 'device': ['name', 'owner', 'group', 'dev_type'], - 'directory': ['name', 'owner', 'group', 'perms'], - 'file': ['name', 'owner', 'group', 'perms', '__text__'], - 'hardlink': ['name', 'to'], - 'symlink': ['name', 'to'], - 'ignore': ['name'], - 'nonexistent': ['name'], - 'permissions': ['name', 'owner', 'group', 'perms'], - 'vcs': ['vcstype', 'revision', 'sourceurl']}, - 'Service': { - 'chkconfig': ['name'], - 'deb': ['name'], - 'rc-update': ['name'], - 'smf': ['name', 'FMRI'], - 'upstart': ['name']}, - 'Action': ['name', 'timing', 'when', 'status', 'command'], - 'Package': ['name']} - - def Run(self): - self.check_packages() - if "Defaults" in self.core.plugins: - self.logger.info("Defaults plugin enabled; skipping required " - "attribute checks") - else: - self.check_rules() - self.check_bundles() - - def check_packages(self): - """ check package sources for Source entries with missing attrs """ - if 'Packages' in self.core.plugins: - for source in self.core.plugins['Packages'].sources: - if isinstance(source, Yum.YumSource): - if (not source.pulp_id and not source.url and - not source.rawurl): - self.LintError("required-attrs-missing", - "A %s source must have either a url, " - "rawurl, or pulp_id attribute: %s" % - (source.ptype, - self.RenderXML(source.xsource))) - elif not source.url and not source.rawurl: - self.LintError("required-attrs-missing", - "A %s source must have either a url or " - "rawurl attribute: %s" % - (source.ptype, - self.RenderXML(source.xsource))) - - if (not isinstance(source, Apt.AptSource) and - source.recommended): - self.LintError("extra-attrs", - "The recommended attribute is not " - "supported on %s sources: %s" % - (source.ptype, - self.RenderXML(source.xsource))) - - def check_rules(self): - """ check Rules for Path entries with missing attrs """ - if 'Rules' in self.core.plugins: - for rules in self.core.plugins['Rules'].entries.values(): - xdata = rules.pnode.data - for path in xdata.xpath("//Path"): - self.check_entry(path, os.path.join(self.config['repo'], - rules.name)) - - def check_bundles(self): - """ check bundles for BoundPath entries with missing attrs """ - if 'Bundler' in self.core.plugins: - for bundle in self.core.plugins['Bundler'].entries.values(): - try: - xdata = lxml.etree.XML(bundle.data) - except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - - for path in xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): - self.check_entry(path, bundle.name) - - def check_entry(self, entry, filename): - """ generic entry check """ - if self.HandlesFile(filename): - name = entry.get('name') - tag = entry.tag - if tag.startswith("Bound"): - tag = tag[5:] - if tag not in self.required_attrs: - self.LintError("unknown-entry-tag", - "Unknown entry tag '%s': %s" % - (entry.tag, self.RenderXML(entry))) - - if isinstance(self.required_attrs[tag], dict): - etype = entry.get('type') - if etype in self.required_attrs[tag]: - required_attrs = set(self.required_attrs[tag][etype] + - ['type']) - else: - self.LintError("unknown-entry-type", - "Unknown %s type %s: %s" % - (tag, etype, self.RenderXML(entry))) - return - else: - required_attrs = set(self.required_attrs[tag]) - attrs = set(entry.attrib.keys()) - - if 'dev_type' in required_attrs: - dev_type = entry.get('dev_type') - if dev_type in ['block', 'char']: - # check if major/minor are specified - required_attrs |= set(['major', 'minor']) - - if '__text__' in required_attrs: - required_attrs.remove('__text__') - if (not entry.text and - not entry.get('empty', 'false').lower() == 'true'): - self.LintError("required-attrs-missing", - "Text missing for %s %s in %s: %s" % - (entry.tag, name, filename, - self.RenderXML(entry))) - - if not attrs.issuperset(required_attrs): - self.LintError("required-attrs-missing", - "The following required attribute(s) are " - "missing for %s %s in %s: %s\n%s" % - (entry.tag, name, filename, - ", ".join([attr - for attr in - required_attrs.difference(attrs)]), - self.RenderXML(entry))) diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py deleted file mode 100644 index 952a65365..000000000 --- a/src/lib/Server/Lint/Validate.py +++ /dev/null @@ -1,200 +0,0 @@ -import fnmatch -import glob -import lxml.etree -import os -from subprocess import Popen, PIPE, STDOUT -import sys - -import Bcfg2.Server.Lint - -class Validate(Bcfg2.Server.Lint.ServerlessPlugin): - """ Ensure that the repo validates """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) - self.filesets = {"metadata:groups":"%s/metadata.xsd", - "metadata:clients":"%s/clients.xsd", - "info":"%s/info.xsd", - "%s/Bundler/*.xml":"%s/bundle.xsd", - "%s/Bundler/*.genshi":"%s/bundle.xsd", - "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", - "%s/Base/*.xml":"%s/base.xsd", - "%s/Rules/*.xml":"%s/rules.xsd", - "%s/Defaults/*.xml":"%s/defaults.xsd", - "%s/etc/report-configuration.xml":"%s/report-configuration.xsd", - "%s/Svcmgr/*.xml":"%s/services.xsd", - "%s/Deps/*.xml":"%s/deps.xsd", - "%s/Decisions/*.xml":"%s/decisions.xsd", - "%s/Packages/sources.xml":"%s/packages.xsd", - "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", - "%s/FileProbes/config.xml":"%s/fileprobes.xsd", - } - - self.filelists = {} - self.get_filelists() - - def Run(self): - schemadir = self.config['schema'] - - for path, schemaname in self.filesets.items(): - try: - filelist = self.filelists[path] - except KeyError: - filelist = [] - - if filelist: - # avoid loading schemas for empty file lists - schemafile = schemaname % schemadir - try: - schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) - except IOError: - e = sys.exc_info()[1] - self.LintError("input-output-error", str(e)) - continue - except lxml.etree.XMLSchemaParseError: - e = sys.exc_info()[1] - self.LintError("schema-failed-to-parse", - "Failed to process schema %s: %s" % - (schemafile, e)) - continue - for filename in filelist: - self.validate(filename, schemafile, schema=schema) - - self.check_properties() - - def check_properties(self): - """ check Properties files against their schemas """ - for filename in self.filelists['props']: - schemafile = "%s.xsd" % os.path.splitext(filename)[0] - if os.path.exists(schemafile): - self.validate(filename, schemafile) - else: - self.LintError("properties-schema-not-found", - "No schema found for %s" % filename) - - def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ - if schema is None: - # if no schema object was provided, instantiate one - try: - schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) - except: - self.LintError("schema-failed-to-parse", - "Failed to process schema %s" % schemafile) - return False - - try: - datafile = lxml.etree.parse(filename) - except SyntaxError: - lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) - self.LintError("xml-failed-to-parse", - "%s fails to parse:\n%s" % (filename, - lint.communicate()[0])) - lint.wait() - return False - except IOError: - self.LintError("xml-failed-to-read", - "Failed to open file %s" % filename) - return False - - if not schema.validate(datafile): - cmd = ["xmllint"] - if self.files is None: - cmd.append("--xinclude") - cmd.extend(["--noout", "--schema", schemafile, filename]) - lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) - output = lint.communicate()[0] - if lint.wait(): - self.LintError("xml-failed-to-verify", - "%s fails to verify:\n%s" % (filename, output)) - return False - return True - - def get_filelists(self): - """ get lists of different kinds of files to validate """ - if self.files is not None: - listfiles = lambda p: fnmatch.filter(self.files, p % "*") - else: - listfiles = lambda p: glob.glob(p % self.config['repo']) - - for path in self.filesets.keys(): - if path.startswith("metadata:"): - mtype = path.split(":")[1] - self.filelists[path] = self.get_metadata_list(mtype) - elif path == "info": - if self.files is not None: - self.filelists[path] = \ - [f for f in self.files - if os.path.basename(f) == 'info.xml'] - else: # self.files is None - self.filelists[path] = [] - for infodir in ['Cfg', 'TGenshi', 'TCheetah']: - for root, dirs, files in os.walk('%s/%s' % - (self.config['repo'], - infodir)): - self.filelists[path].extend([os.path.join(root, f) - for f in files - if f == 'info.xml']) - else: - self.filelists[path] = listfiles(path) - - self.filelists['props'] = listfiles("%s/Properties/*.xml") - all_metadata = listfiles("%s/Metadata/*.xml") - - # if there are other files in Metadata that aren't xincluded - # from clients.xml or groups.xml, we can't verify them. warn - # about those. - for fname in all_metadata: - if (fname not in self.filelists['metadata:groups'] and - fname not in self.filelists['metadata:clients']): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: Could not determine file type of %s" % fname) - - def get_metadata_list(self, mtype): - """ get all metadata files for the specified type (clients or - group) """ - if self.files is not None: - rv = fnmatch.filter(self.files, "*/Metadata/%s.xml" % mtype) - else: - rv = glob.glob("%s/Metadata/%s.xml" % (self.config['repo'], mtype)) - - # attempt to follow XIncludes. if the top-level files aren't - # listed in self.files, though, there's really nothing we can - # do to guess what a file in Metadata is - if rv: - try: - rv.extend(self.follow_xinclude(rv[0])) - except lxml.etree.XMLSyntaxError: - e = sys.exc_info()[1] - self.LintError("xml-failed-to-parse", - "%s fails to parse:\n%s" % (rv[0], e)) - - - return rv - - def follow_xinclude(self, xfile): - """ follow xincludes in the given file """ - xdata = lxml.etree.parse(xfile) - included = set([ent.get('href') for ent in - xdata.findall('./{http://www.w3.org/2001/XInclude}include')]) - rv = [] - - while included: - try: - filename = included.pop() - except KeyError: - continue - - path = os.path.join(os.path.dirname(xfile), filename) - if self.HandlesFile(path): - rv.append(path) - groupdata = lxml.etree.parse(path) - [included.add(el.get('href')) - for el in - groupdata.findall('./{http://www.w3.org/2001/XInclude}include')] - included.discard(filename) - - return rv - diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py deleted file mode 100644 index 4d6df8c8f..000000000 --- a/src/lib/Server/Lint/__init__.py +++ /dev/null @@ -1,218 +0,0 @@ -__revision__ = '$Revision$' - -__all__ = ['Bundles', - 'Comments', - 'Duplicates', - 'InfoXML', - 'MergeFiles', - 'Pkgmgr', - 'RequiredAttrs', - 'Validate', - 'Genshi', - 'Deltas'] - -import logging -import os -import sys -from copy import copy -import textwrap -import lxml.etree -import Bcfg2.Logger -import fcntl -import termios -import struct - -def _ioctl_GWINSZ(fd): - try: - cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - except: - return None - return cr - -def get_termsize(): - """ get a tuple of (width, height) giving the size of the terminal """ - if not sys.stdout.isatty(): - return None - cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - cr = _ioctl_GWINSZ(fd) - os.close(fd) - except: - pass - if not cr: - try: - cr = (os.environ['LINES'], os.environ['COLUMNS']) - except KeyError: - return None - return int(cr[1]), int(cr[0]) - -class Plugin (object): - """ base class for ServerlessPlugin and ServerPlugin """ - - def __init__(self, config, errorhandler=None, files=None): - self.files = files - self.config = config - self.logger = logging.getLogger('bcfg2-lint') - if errorhandler is None: - self.errorHandler = ErrorHandler() - else: - self.errorHandler = errorhandler - - def Run(self): - """ run the plugin. must be overloaded by child classes """ - pass - - def HandlesFile(self, fname): - """ returns true if the given file should be handled by the - plugin according to the files list, false otherwise """ - return (self.files is None or - fname in self.files or - os.path.join(self.config['repo'], fname) in self.files or - os.path.abspath(fname) in self.files or - os.path.abspath(os.path.join(self.config['repo'], - fname)) in self.files) - - def LintError(self, err, msg): - self.errorHandler.dispatch(err, msg) - - def RenderXML(self, element): - """render an XML element for error output -- line number - prefixed, no children""" - xml = None - if len(element) or element.text: - el = copy(element) - if el.text: - el.text = '...' - [el.remove(c) for c in el.iterchildren()] - xml = lxml.etree.tostring(el).strip() - else: - xml = lxml.etree.tostring(element).strip() - return " line %s: %s" % (element.sourceline, xml) - - -class ErrorHandler (object): - # how to handle different errors by default - _errors = {"no-infoxml":"warning", - "paranoid-false":"warning", - "bundle-not-found":"error", - "inconsistent-bundle-name":"warning", - "group-tag-not-allowed":"error", - "unexpanded-keywords":"warning", - "keywords-not-found":"warning", - "comments-not-found":"warning", - "broken-xinclude-chain":"warning", - "duplicate-client":"error", - "duplicate-group":"error", - "duplicate-package":"error", - "multiple-default-groups":"error", - "required-infoxml-attrs-missing":"error", - "unknown-entry-type":"error", - "required-attrs-missing":"error", - "extra-attrs":"warning", - "schema-failed-to-parse":"warning", - "properties-schema-not-found":"warning", - "xml-failed-to-parse":"error", - "xml-failed-to-read":"error", - "xml-failed-to-verify":"error", - "merge-cfg":"warning", - "merge-probes":"warning", - "input-output-error":"error", - "genshi-syntax-error":"error", - "pattern-fails-to-initialize":"error", - "cat-file-used":"warning", - "diff-file-used":"warning"} - - def __init__(self, config=None): - self.errors = 0 - self.warnings = 0 - - self.logger = logging.getLogger('bcfg2-lint') - - termsize = get_termsize() - if termsize is not None: - twrap = textwrap.TextWrapper(initial_indent=" ", - subsequent_indent=" ", - width=termsize[0]) - self._wrapper = twrap.wrap - else: - self._wrapper = lambda s: [s] - - self._handlers = {} - if config is not None: - for err, action in config.items(): - if "warn" in action: - self._handlers[err] = self.warn - elif "err" in action: - self._handlers[err] = self.error - else: - self._handlers[err] = self.debug - - for err, action in self._errors.items(): - if err not in self._handlers: - if "warn" in action: - self._handlers[err] = self.warn - elif "err" in action: - self._handlers[err] = self.error - else: - self._handlers[err] = self.debug - - def dispatch(self, err, msg): - if err in self._handlers: - self._handlers[err](msg) - self.logger.debug(" (%s)" % err) - else: - # assume that it's an error, but complain - self.error(msg) - self.logger.warning("Unknown error %s" % err) - - def error(self, msg): - """ log an error condition """ - self.errors += 1 - self._log(msg, self.logger.error, prefix="ERROR: ") - - def warn(self, msg): - """ log a warning condition """ - self.warnings += 1 - self._log(msg, self.logger.warning, prefix="WARNING: ") - - def debug(self, msg): - """ log a silent/debug condition """ - self._log(msg, self.logger.debug) - - def _log(self, msg, logfunc, prefix=""): - # a message may itself consist of multiple lines. wrap() will - # elide them all into a single paragraph, which we don't want. - # so we split the message into its paragraphs and wrap each - # paragraph individually. this means, unfortunately, that we - # lose textwrap's built-in initial indent functionality, - # because we want to only treat the very first line of the - # first paragraph specially. so we do some silliness. - rawlines = msg.splitlines() - firstline = True - for rawline in rawlines: - lines = self._wrapper(rawline) - for line in lines: - if firstline: - logfunc(prefix + line.lstrip()) - firstline = False - else: - logfunc(line) - - -class ServerlessPlugin (Plugin): - """ base class for plugins that are run before the server starts - up (i.e., plugins that check things that may prevent the server - from starting up) """ - pass - - -class ServerPlugin (Plugin): - """ base class for plugins that check things that require the - running Bcfg2 server """ - def __init__(self, lintCore, config, **kwargs): - Plugin.__init__(self, config, **kwargs) - self.core = lintCore - self.logger = self.core.logger - self.metadata = self.core.metadata |