diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Lint')
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/Bundles.py | 61 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/Comments.py | 187 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/Duplicates.py | 81 | ||||
| -rwxr-xr-x | src/lib/Bcfg2/Server/Lint/Genshi.py | 27 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/GroupPatterns.py | 31 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/InfoXML.py | 42 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/MergeFiles.py | 69 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/Pkgmgr.py | 35 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 138 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py | 200 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Lint/__init__.py | 215 |
11 files changed, 1086 insertions, 0 deletions
diff --git a/src/lib/Bcfg2/Server/Lint/Bundles.py b/src/lib/Bcfg2/Server/Lint/Bundles.py new file mode 100644 index 000000000..0c95f7e72 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Bundles.py @@ -0,0 +1,61 @@ +import lxml.etree +import Bcfg2.Server.Lint + +class Bundles(Bcfg2.Server.Lint.ServerPlugin): + """ Perform various bundle checks """ + + def Run(self): + """ run plugin """ + if 'Bundler' in self.core.plugins: + self.missing_bundles() + for bundle in self.core.plugins['Bundler'].entries.values(): + if self.HandlesFile(bundle.name): + if (not Bcfg2.Server.Plugins.Bundler.have_genshi or + type(bundle) is not + Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): + self.bundle_names(bundle) + + def missing_bundles(self): + """ find bundles listed in Metadata but not implemented in Bundler """ + if self.files is None: + # when given a list of files on stdin, this check is + # useless, so skip it + groupdata = self.metadata.groups_xml.xdata + ref_bundles = set([b.get("name") + for b in groupdata.findall("//Bundle")]) + + allbundles = self.core.plugins['Bundler'].entries.keys() + for bundle in ref_bundles: + xmlbundle = "%s.xml" % bundle + genshibundle = "%s.genshi" % bundle + if (xmlbundle not in allbundles and + genshibundle not in allbundles): + self.LintError("bundle-not-found", + "Bundle %s referenced, but does not exist" % + bundle) + + def bundle_names(self, bundle): + """ verify bundle name attribute matches filename """ + try: + xdata = lxml.etree.XML(bundle.data) + except AttributeError: + # genshi template + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + + fname = bundle.name.split('Bundler/')[1].split('.')[0] + bname = xdata.get('name') + if fname != bname: + self.LintError("inconsistent-bundle-name", + "Inconsistent bundle name: filename is %s, bundle name is %s" % + (fname, bname)) + + def sgenshi_groups(self, bundle): + """ ensure that Genshi Bundles do not include <Group> tags, + which are not supported """ + xdata = lxml.etree.parse(bundle.name) + groups = [self.RenderXML(g) + for g in xdata.getroottree().findall("//Group")] + if groups: + self.LintError("group-tag-not-allowed", + "<Group> tag is not allowed in SGenshi Bundle:\n%s" % + "\n".join(groups)) diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py new file mode 100644 index 000000000..19e6c1167 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -0,0 +1,187 @@ +import os.path +import lxml.etree +import Bcfg2.Server.Lint + +class Comments(Bcfg2.Server.Lint.ServerPlugin): + """ check files for various required headers """ + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) + self.config_cache = {} + + def Run(self): + self.check_bundles() + self.check_properties() + self.check_metadata() + self.check_cfg() + self.check_infoxml() + self.check_probes() + + def required_keywords(self, rtype): + """ given a file type, fetch the list of required VCS keywords + from the bcfg2-lint config """ + return self.required_items(rtype, "keyword") + + def required_comments(self, rtype): + """ given a file type, fetch the list of required comments + from the bcfg2-lint config """ + return self.required_items(rtype, "comment") + + def required_items(self, rtype, itype): + """ given a file type and item type (comment or keyword), + fetch the list of required items from the bcfg2-lint config """ + if itype not in self.config_cache: + self.config_cache[itype] = {} + + if rtype not in self.config_cache[itype]: + rv = [] + global_item = "global_%ss" % itype + if global_item in self.config: + rv.extend(self.config[global_item].split(",")) + + item = "%s_%ss" % (rtype.lower(), itype) + if item in self.config: + if self.config[item]: + rv.extend(self.config[item].split(",")) + else: + # config explicitly specifies nothing + rv = [] + self.config_cache[itype][rtype] = rv + return self.config_cache[itype][rtype] + + def check_bundles(self): + """ check bundle files for required headers """ + if 'Bundler' in self.core.plugins: + for bundle in self.core.plugins['Bundler'].entries.values(): + xdata = None + rtype = "" + try: + xdata = lxml.etree.XML(bundle.data) + rtype = "bundler" + except (lxml.etree.XMLSyntaxError, AttributeError): + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + rtype = "sgenshi" + + self.check_xml(bundle.name, xdata, rtype) + + def check_properties(self): + """ check properties files for required headers """ + if 'Properties' in self.core.plugins: + props = self.core.plugins['Properties'] + for propfile, pdata in props.store.entries.items(): + if os.path.splitext(propfile)[1] == ".xml": + self.check_xml(pdata.name, pdata.xdata, 'properties') + + def check_metadata(self): + """ check metadata files for required headers """ + if self.has_all_xincludes("groups.xml"): + self.check_xml(os.path.join(self.metadata.data, "groups.xml"), + self.metadata.groups_xml.data, + "metadata") + if self.has_all_xincludes("clients.xml"): + self.check_xml(os.path.join(self.metadata.data, "clients.xml"), + self.metadata.clients_xml.data, + "metadata") + + def check_cfg(self): + """ check Cfg files for required headers """ + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if entry.name.endswith(".genshi"): + rtype = "tgenshi" + else: + rtype = "cfg" + self.check_plaintext(entry.name, entry.data, rtype) + + def check_infoxml(self): + """ check info.xml files for required headers """ + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.items(): + if (hasattr(entryset, "infoxml") and + entryset.infoxml is not None): + self.check_xml(entryset.infoxml.name, + entryset.infoxml.pnode.data, + "infoxml") + + def check_probes(self): + """ check probes for required headers """ + if 'Probes' in self.core.plugins: + for probe in self.core.plugins['Probes'].probes.entries.values(): + self.check_plaintext(probe.name, probe.data, "probes") + + def check_xml(self, filename, xdata, rtype): + """ check generic XML files for required headers """ + self.check_lines(filename, + [str(el) + for el in xdata.getiterator(lxml.etree.Comment)], + rtype) + + def check_plaintext(self, filename, data, rtype): + """ check generic plaintex files for required headers """ + self.check_lines(filename, data.splitlines(), rtype) + + def check_lines(self, filename, lines, rtype): + """ generic header check for a set of lines """ + if self.HandlesFile(filename): + # found is trivalent: + # False == not found + # None == found but not expanded + # True == found and expanded + found = dict((k, False) for k in self.required_keywords(rtype)) + + for line in lines: + # we check for both '$<keyword>:' and '$<keyword>$' to see + # if the keyword just hasn't been expanded + for (keyword, status) in found.items(): + if not status: + if '$%s:' % keyword in line: + found[keyword] = True + elif '$%s$' % keyword in line: + found[keyword] = None + + unexpanded = [keyword for (keyword, status) in found.items() + if status is None] + if unexpanded: + self.LintError("unexpanded-keywords", + "%s: Required keywords(s) found but not expanded: %s" % + (filename, ", ".join(unexpanded))) + missing = [keyword for (keyword, status) in found.items() + if status is False] + if missing: + self.LintError("keywords-not-found", + "%s: Required keywords(s) not found: $%s$" % + (filename, "$, $".join(missing))) + + # next, check for required comments. found is just + # boolean + found = dict((k, False) for k in self.required_comments(rtype)) + + for line in lines: + for (comment, status) in found.items(): + if not status: + found[comment] = comment in line + + missing = [comment for (comment, status) in found.items() + if status is False] + if missing: + self.LintError("comments-not-found", + "%s: Required comments(s) not found: %s" % + (filename, ", ".join(missing))) + + def has_all_xincludes(self, mfile): + """ return true if self.files includes all XIncludes listed in + the specified metadata type, false otherwise""" + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): + if not self.has_all_xincludes(el.get('href')): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not include %s" % path) + return False + + return True + diff --git a/src/lib/Bcfg2/Server/Lint/Duplicates.py b/src/lib/Bcfg2/Server/Lint/Duplicates.py new file mode 100644 index 000000000..afddebb71 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Duplicates.py @@ -0,0 +1,81 @@ +import os.path +import lxml.etree +import Bcfg2.Server.Lint + +class Duplicates(Bcfg2.Server.Lint.ServerPlugin): + """ Find duplicate clients, groups, etc. """ + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) + self.groups_xdata = None + self.clients_xdata = None + self.load_xdata() + + def Run(self): + """ run plugin """ + # only run this plugin if we were not given a list of files. + # not only is it marginally silly to run this plugin with a + # partial list of files, it turns out to be really freaking + # hard to get only a fragment of group or client metadata + if self.groups_xdata is not None: + self.duplicate_groups() + self.duplicate_defaults() + if self.clients_xdata is not None: + self.duplicate_clients() + + def load_xdata(self): + """ attempt to load XML data for groups and clients. only + actually load data if all documents reference in XIncludes can + be found in self.files""" + if self.has_all_xincludes("groups.xml"): + self.groups_xdata = self.metadata.clients_xml.xdata + if self.has_all_xincludes("clients.xml"): + self.clients_xdata = self.metadata.clients_xml.xdata + + def duplicate_groups(self): + """ find duplicate groups """ + self.duplicate_entries(self.clients_xdata.xpath('//Groups/Group'), + 'group') + + def duplicate_clients(self): + """ find duplicate clients """ + self.duplicate_entries(self.clients_xdata.xpath('//Clients/Client'), + 'client') + + def duplicate_entries(self, data, etype): + """ generic duplicate entry finder """ + seen = {} + for el in data: + if el.get('name') not in seen: + seen[el.get('name')] = el + else: + self.LintError("duplicate-%s" % etype, + "Duplicate %s '%s':\n%s\n%s" % + (etype, el.get('name'), + self.RenderXML(seen[el.get('name')]), + self.RenderXML(el))) + + def duplicate_defaults(self): + """ check for multiple default group definitions """ + default_groups = [g for g in self.groups_xdata.findall('.//Group') + if g.get('default') == 'true'] + if len(default_groups) > 1: + self.LintError("multiple-default-groups", + "Multiple default groups defined: %s" % + ",".join(default_groups)) + + def has_all_xincludes(self, mfile): + """ return true if self.files includes all XIncludes listed in + the specified metadata type, false otherwise""" + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): + if not self.has_all_xincludes(el.get('href')): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not include %s" % path) + return False + + return True diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py new file mode 100755 index 000000000..1965f0330 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -0,0 +1,27 @@ +import genshi.template +import Bcfg2.Server.Lint + +class Genshi(Bcfg2.Server.Lint.ServerPlugin): + """ Check Genshi templates for syntax errors """ + + def Run(self): + """ run plugin """ + loader = genshi.template.TemplateLoader() + for plugin in ['Cfg', 'TGenshi']: + if plugin in self.core.plugins: + self.check_files(self.core.plugins[plugin].entries, + loader=loader) + + def check_files(self, entries, loader=None): + if loader is None: + loader = genshi.template.TemplateLoader() + + for eset in entries.values(): + for fname, sdata in list(eset.entries.items()): + if fname.endswith(".genshi") or fname.endswith(".newtxt"): + try: + loader.load(sdata.name, + cls=genshi.template.NewTextTemplate) + except genshi.template.TemplateSyntaxError, err: + self.LintError("genshi-syntax-error", + "Genshi syntax error: %s" % err) diff --git a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py new file mode 100644 index 000000000..2e0dffde7 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py @@ -0,0 +1,31 @@ +import sys +import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.GroupPatterns import PatternMap + +class GroupPatterns(Bcfg2.Server.Lint.ServerPlugin): + """ Check Genshi templates for syntax errors """ + + def Run(self): + """ run plugin """ + if 'GroupPatterns' in self.core.plugins: + cfg = self.core.plugins['GroupPatterns'].config + for entry in cfg.xdata.xpath('//GroupPattern'): + groups = [g.text for g in entry.findall('Group')] + self.check(entry, groups, ptype='NamePattern') + self.check(entry, groups, ptype='NameRange') + + def check(self, entry, groups, ptype="NamePattern"): + if ptype == "NamePattern": + pmap = lambda p: PatternMap(p, None, groups) + else: + pmap = lambda p: PatternMap(None, p, groups) + + for el in entry.findall(ptype): + pat = el.text + try: + pmap(pat) + except: + err = sys.exc_info()[1] + self.LintError("pattern-fails-to-initialize", + "Failed to initialize %s %s for %s: %s" % + (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py new file mode 100644 index 000000000..2054e23bf --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -0,0 +1,42 @@ +import os.path +import Bcfg2.Options +import Bcfg2.Server.Lint + +class InfoXML(Bcfg2.Server.Lint.ServerPlugin): + """ ensure that all config files have an info.xml file""" + + def Run(self): + if 'Cfg' in self.core.plugins: + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + infoxml_fname = os.path.join(entryset.path, "info.xml") + if self.HandlesFile(infoxml_fname): + if (hasattr(entryset, "infoxml") and + entryset.infoxml is not None): + self.check_infoxml(infoxml_fname, + entryset.infoxml.pnode.data) + else: + self.LintError("no-infoxml", + "No info.xml found for %s" % filename) + + def check_infoxml(self, fname, xdata): + for info in xdata.getroottree().findall("//Info"): + required = [] + if "required_attrs" in self.config: + required = self.config["required_attrs"].split(",") + + missing = [attr for attr in required if info.get(attr) is None] + if missing: + self.LintError("required-infoxml-attrs-missing", + "Required attribute(s) %s not found in %s:%s" % + (",".join(missing), fname, self.RenderXML(info))) + + if ((Bcfg2.Options.MDATA_PARANOID.value and + info.get("paranoid") is not None and + info.get("paranoid").lower() == "false") or + (not Bcfg2.Options.MDATA_PARANOID.value and + (info.get("paranoid") is None or + info.get("paranoid").lower() != "true"))): + self.LintError("paranoid-false", + "Paranoid must be true in %s:%s" % + (fname, self.RenderXML(info))) + diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py new file mode 100644 index 000000000..56bca7931 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -0,0 +1,69 @@ +import os +import copy +from difflib import SequenceMatcher +import Bcfg2.Server.Lint + +class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): + """ find Probes or Cfg files with multiple similar files that + might be merged into one """ + + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + if 'Probes' in self.core.plugins: + self.check_probes() + + def check_cfg(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for mset in self.get_similar(entryset.entries): + self.LintError("merge-cfg", + "The following files are similar: %s. " + "Consider merging them into a single Genshi " + "template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + + def check_probes(self): + probes = self.core.plugins['Probes'].probes.entries + for mset in self.get_similar(probes): + self.LintError("merge-cfg", + "The following probes are similar: %s. " + "Consider merging them into a single probe." % + ", ".join([p for p in mset])) + + def get_similar(self, entries): + if "threshold" in self.config: + # accept threshold either as a percent (e.g., "threshold=75") or + # as a ratio (e.g., "threshold=.75") + threshold = float(self.config['threshold']) + if threshold > 1: + threshold /= 100 + else: + threshold = 0.75 + rv = [] + elist = entries.items() + while elist: + result = self._find_similar(elist.pop(0), copy.copy(elist), + threshold) + if len(result) > 1: + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in result] + rv.append(result) + return rv + + def _find_similar(self, ftuple, others, threshold): + fname, fdata = ftuple + rv = [fname] + while others: + cname, cdata = others.pop(0) + sm = SequenceMatcher(None, fdata.data, cdata.data) + # perform progressively more expensive comparisons + if (sm.real_quick_ratio() > threshold and + sm.quick_ratio() > threshold and + sm.ratio() > threshold): + rv.extend(self._find_similar((cname, cdata), copy.copy(others), + threshold)) + return rv + + diff --git a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py new file mode 100644 index 000000000..8f099163a --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py @@ -0,0 +1,35 @@ +import glob +import lxml.etree +import Bcfg2.Server.Lint + +class Pkgmgr(Bcfg2.Server.Lint.ServerlessPlugin): + """ find duplicate Pkgmgr entries with the same priority """ + + def Run(self): + pset = set() + for pfile in glob.glob("%s/Pkgmgr/*.xml" % self.config['repo']): + if self.HandlesFile(pfile): + xdata = lxml.etree.parse(pfile).getroot() + # get priority, type, group + priority = xdata.get('priority') + ptype = xdata.get('type') + for pkg in xdata.xpath("//Package"): + if pkg.getparent().tag == 'Group': + grp = pkg.getparent().get('name') + if (type(grp) is not str and + grp.getparent().tag == 'Group'): + pgrp = grp.getparent().get('name') + else: + pgrp = 'none' + else: + grp = 'none' + pgrp = 'none' + ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) + # check if package is already listed with same + # priority, type, grp + if ptuple in pset: + self.LintError("duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) + else: + pset.add(ptuple) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py new file mode 100644 index 000000000..55206d2ba --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -0,0 +1,138 @@ +import os.path +import lxml.etree +import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.Packages import Apt, Yum + +class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): + """ verify attributes for configuration entries (as defined in + doc/server/configurationentries) """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) + self.required_attrs = { + 'Path': { + 'device': ['name', 'owner', 'group', 'dev_type'], + 'directory': ['name', 'owner', 'group', 'perms'], + 'file': ['name', 'owner', 'group', 'perms', '__text__'], + 'hardlink': ['name', 'to'], + 'symlink': ['name', 'to'], + 'ignore': ['name'], + 'nonexistent': ['name'], + 'permissions': ['name', 'owner', 'group', 'perms'], + 'vcs': ['vcstype', 'revision', 'sourceurl']}, + 'Service': { + 'chkconfig': ['name'], + 'deb': ['name'], + 'rc-update': ['name'], + 'smf': ['name', 'FMRI'], + 'upstart': ['name']}, + 'Action': ['name', 'timing', 'when', 'status', 'command'], + 'Package': ['name']} + + def Run(self): + self.check_packages() + if "Defaults" in self.core.plugins: + self.logger.info("Defaults plugin enabled; skipping required " + "attribute checks") + else: + self.check_rules() + self.check_bundles() + + def check_packages(self): + """ check package sources for Source entries with missing attrs """ + if 'Packages' in self.core.plugins: + for source in self.core.plugins['Packages'].sources: + if isinstance(source, Yum.YumSource): + if (not source.pulp_id and not source.url and + not source.rawurl): + self.LintError("required-attrs-missing", + "A %s source must have either a url, " + "rawurl, or pulp_id attribute: %s" % + (source.ptype, + self.RenderXML(source.xsource))) + elif not source.url and not source.rawurl: + self.LintError("required-attrs-missing", + "A %s source must have either a url or " + "rawurl attribute: %s" % + (source.ptype, + self.RenderXML(source.xsource))) + + if (not isinstance(source, Apt.AptSource) and + source.recommended): + self.LintError("extra-attrs", + "The recommended attribute is not " + "supported on %s sources: %s" % + (source.ptype, + self.RenderXML(source.xsource))) + + def check_rules(self): + """ check Rules for Path entries with missing attrs """ + if 'Rules' in self.core.plugins: + for rules in self.core.plugins['Rules'].entries.values(): + xdata = rules.pnode.data + for path in xdata.xpath("//Path"): + self.check_entry(path, os.path.join(self.config['repo'], + rules.name)) + + def check_bundles(self): + """ check bundles for BoundPath entries with missing attrs """ + if 'Bundler' in self.core.plugins: + for bundle in self.core.plugins['Bundler'].entries.values(): + try: + xdata = lxml.etree.XML(bundle.data) + except (lxml.etree.XMLSyntaxError, AttributeError): + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + + for path in xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): + self.check_entry(path, bundle.name) + + def check_entry(self, entry, filename): + """ generic entry check """ + if self.HandlesFile(filename): + name = entry.get('name') + tag = entry.tag + if tag.startswith("Bound"): + tag = tag[5:] + if tag not in self.required_attrs: + self.LintError("unknown-entry-tag", + "Unknown entry tag '%s': %s" % + (entry.tag, self.RenderXML(entry))) + + if isinstance(self.required_attrs[tag], dict): + etype = entry.get('type') + if etype in self.required_attrs[tag]: + required_attrs = set(self.required_attrs[tag][etype] + + ['type']) + else: + self.LintError("unknown-entry-type", + "Unknown %s type %s: %s" % + (tag, etype, self.RenderXML(entry))) + return + else: + required_attrs = set(self.required_attrs[tag]) + attrs = set(entry.attrib.keys()) + + if 'dev_type' in required_attrs: + dev_type = entry.get('dev_type') + if dev_type in ['block', 'char']: + # check if major/minor are specified + required_attrs |= set(['major', 'minor']) + + if '__text__' in required_attrs: + required_attrs.remove('__text__') + if (not entry.text and + not entry.get('empty', 'false').lower() == 'true'): + self.LintError("required-attrs-missing", + "Text missing for %s %s in %s: %s" % + (entry.tag, name, filename, + self.RenderXML(entry))) + + if not attrs.issuperset(required_attrs): + self.LintError("required-attrs-missing", + "The following required attribute(s) are " + "missing for %s %s in %s: %s\n%s" % + (entry.tag, name, filename, + ", ".join([attr + for attr in + required_attrs.difference(attrs)]), + self.RenderXML(entry))) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py new file mode 100644 index 000000000..012d65894 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -0,0 +1,200 @@ +import fnmatch +import glob +import lxml.etree +import os +from subprocess import Popen, PIPE, STDOUT +import sys + +import Bcfg2.Server.Lint + +class Validate(Bcfg2.Server.Lint.ServerlessPlugin): + """ Ensure that the repo validates """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + self.filesets = {"metadata:groups":"%s/metadata.xsd", + "metadata:clients":"%s/clients.xsd", + "info":"%s/info.xsd", + "%s/Bundler/*.xml":"%s/bundle.xsd", + "%s/Bundler/*.genshi":"%s/bundle.xsd", + "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", + "%s/Base/*.xml":"%s/base.xsd", + "%s/Rules/*.xml":"%s/rules.xsd", + "%s/Defaults/*.xml":"%s/defaults.xsd", + "%s/etc/report-configuration.xml":"%s/report-configuration.xsd", + "%s/Svcmgr/*.xml":"%s/services.xsd", + "%s/Deps/*.xml":"%s/deps.xsd", + "%s/Decisions/*.xml":"%s/decisions.xsd", + "%s/Packages/sources.xml":"%s/packages.xsd", + "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", + "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", + "%s/FileProbes/config.xml":"%s/fileprobes.xsd", + } + + self.filelists = {} + self.get_filelists() + + def Run(self): + schemadir = self.config['schema'] + + for path, schemaname in self.filesets.items(): + try: + filelist = self.filelists[path] + except KeyError: + filelist = [] + + if filelist: + # avoid loading schemas for empty file lists + schemafile = schemaname % schemadir + try: + schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) + except IOError: + e = sys.exc_info()[1] + self.LintError("input-output-error", str(e)) + continue + except lxml.etree.XMLSchemaParseError: + e = sys.exc_info()[1] + self.LintError("schema-failed-to-parse", + "Failed to process schema %s: %s" % + (schemafile, e)) + continue + for filename in filelist: + self.validate(filename, schemafile, schema=schema) + + self.check_properties() + + def check_properties(self): + """ check Properties files against their schemas """ + for filename in self.filelists['props']: + schemafile = "%s.xsd" % os.path.splitext(filename)[0] + if os.path.exists(schemafile): + self.validate(filename, schemafile) + else: + self.LintError("properties-schema-not-found", + "No schema found for %s" % filename) + + def validate(self, filename, schemafile, schema=None): + """validate a file against the given lxml.etree.Schema. + return True on success, False on failure """ + if schema is None: + # if no schema object was provided, instantiate one + try: + schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) + except: + self.LintError("schema-failed-to-parse", + "Failed to process schema %s" % schemafile) + return False + + try: + datafile = lxml.etree.parse(filename) + except SyntaxError: + lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) + self.LintError("xml-failed-to-parse", + "%s fails to parse:\n%s" % (filename, + lint.communicate()[0])) + lint.wait() + return False + except IOError: + self.LintError("xml-failed-to-read", + "Failed to open file %s" % filename) + return False + + if not schema.validate(datafile): + cmd = ["xmllint"] + if self.files is None: + cmd.append("--xinclude") + cmd.extend(["--noout", "--schema", schemafile, filename]) + lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) + output = lint.communicate()[0] + if lint.wait(): + self.LintError("xml-failed-to-verify", + "%s fails to verify:\n%s" % (filename, output)) + return False + return True + + def get_filelists(self): + """ get lists of different kinds of files to validate """ + if self.files is not None: + listfiles = lambda p: fnmatch.filter(self.files, p % "*") + else: + listfiles = lambda p: glob.glob(p % self.config['repo']) + + for path in self.filesets.keys(): + if path.startswith("metadata:"): + mtype = path.split(":")[1] + self.filelists[path] = self.get_metadata_list(mtype) + elif path == "info": + if self.files is not None: + self.filelists[path] = \ + [f for f in self.files + if os.path.basename(f) == 'info.xml'] + else: # self.files is None + self.filelists[path] = [] + for infodir in ['Cfg', 'TGenshi', 'TCheetah']: + for root, dirs, files in os.walk('%s/%s' % + (self.config['repo'], + infodir)): + self.filelists[path].extend([os.path.join(root, f) + for f in files + if f == 'info.xml']) + else: + self.filelists[path] = listfiles(path) + + self.filelists['props'] = listfiles("%s/Properties/*.xml") + all_metadata = listfiles("%s/Metadata/*.xml") + + # if there are other files in Metadata that aren't xincluded + # from clients.xml or groups.xml, we can't verify them. warn + # about those. + for fname in all_metadata: + if (fname not in self.filelists['metadata:groups'] and + fname not in self.filelists['metadata:clients']): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: Could not determine file type of %s" % fname) + + def get_metadata_list(self, mtype): + """ get all metadata files for the specified type (clients or + group) """ + if self.files is not None: + rv = fnmatch.filter(self.files, "*/Metadata/%s.xml" % mtype) + else: + rv = glob.glob("%s/Metadata/%s.xml" % (self.config['repo'], mtype)) + + # attempt to follow XIncludes. if the top-level files aren't + # listed in self.files, though, there's really nothing we can + # do to guess what a file in Metadata is + if rv: + try: + rv.extend(self.follow_xinclude(rv[0])) + except lxml.etree.XMLSyntaxError: + e = sys.exc_info()[1] + self.LintError("xml-failed-to-parse", + "%s fails to parse:\n%s" % (rv[0], e)) + + + return rv + + def follow_xinclude(self, xfile): + """ follow xincludes in the given file """ + xdata = lxml.etree.parse(xfile) + included = set([ent.get('href') for ent in + xdata.findall('./{http://www.w3.org/2001/XInclude}include')]) + rv = [] + + while included: + try: + filename = included.pop() + except KeyError: + continue + + path = os.path.join(os.path.dirname(xfile), filename) + if self.HandlesFile(path): + rv.append(path) + groupdata = lxml.etree.parse(path) + [included.add(el.get('href')) + for el in + groupdata.findall('./{http://www.w3.org/2001/XInclude}include')] + included.discard(filename) + + return rv + diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py new file mode 100644 index 000000000..82770333a --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -0,0 +1,215 @@ +__all__ = ['Bundles', + 'Comments', + 'Duplicates', + 'InfoXML', + 'MergeFiles', + 'Pkgmgr', + 'RequiredAttrs', + 'Validate', + 'Genshi'] + +import logging +import os +import sys +from copy import copy +import textwrap +import lxml.etree +import Bcfg2.Logger +import fcntl +import termios +import struct + +def _ioctl_GWINSZ(fd): + try: + cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + except: + return None + return cr + +def get_termsize(): + """ get a tuple of (width, height) giving the size of the terminal """ + if not sys.stdout.isatty(): + return None + cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) + if not cr: + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + cr = _ioctl_GWINSZ(fd) + os.close(fd) + except: + pass + if not cr: + try: + cr = (os.environ['LINES'], os.environ['COLUMNS']) + except KeyError: + return None + return int(cr[1]), int(cr[0]) + +class Plugin (object): + """ base class for ServerlessPlugin and ServerPlugin """ + + def __init__(self, config, errorhandler=None, files=None): + self.files = files + self.config = config + self.logger = logging.getLogger('bcfg2-lint') + if errorhandler is None: + self.errorHandler = ErrorHandler() + else: + self.errorHandler = errorhandler + + def Run(self): + """ run the plugin. must be overloaded by child classes """ + pass + + def HandlesFile(self, fname): + """ returns true if the given file should be handled by the + plugin according to the files list, false otherwise """ + return (self.files is None or + fname in self.files or + os.path.join(self.config['repo'], fname) in self.files or + os.path.abspath(fname) in self.files or + os.path.abspath(os.path.join(self.config['repo'], + fname)) in self.files) + + def LintError(self, err, msg): + self.errorHandler.dispatch(err, msg) + + def RenderXML(self, element): + """render an XML element for error output -- line number + prefixed, no children""" + xml = None + if len(element) or element.text: + el = copy(element) + if el.text: + el.text = '...' + [el.remove(c) for c in el.iterchildren()] + xml = lxml.etree.tostring(el).strip() + else: + xml = lxml.etree.tostring(element).strip() + return " line %s: %s" % (element.sourceline, xml) + + +class ErrorHandler (object): + # how to handle different errors by default + _errors = {"no-infoxml":"warning", + "paranoid-false":"warning", + "bundle-not-found":"error", + "inconsistent-bundle-name":"warning", + "group-tag-not-allowed":"error", + "unexpanded-keywords":"warning", + "keywords-not-found":"warning", + "comments-not-found":"warning", + "broken-xinclude-chain":"warning", + "duplicate-client":"error", + "duplicate-group":"error", + "duplicate-package":"error", + "multiple-default-groups":"error", + "required-infoxml-attrs-missing":"error", + "unknown-entry-type":"error", + "required-attrs-missing":"error", + "extra-attrs":"warning", + "schema-failed-to-parse":"warning", + "properties-schema-not-found":"warning", + "xml-failed-to-parse":"error", + "xml-failed-to-read":"error", + "xml-failed-to-verify":"error", + "merge-cfg":"warning", + "merge-probes":"warning", + "input-output-error":"error", + "genshi-syntax-error":"error", + "pattern-fails-to-initialize":"error"} + + def __init__(self, config=None): + self.errors = 0 + self.warnings = 0 + + self.logger = logging.getLogger('bcfg2-lint') + + termsize = get_termsize() + if termsize is not None: + self._wrapper = textwrap.TextWrapper(initial_indent=" ", + subsequent_indent=" ", + width=termsize[0]) + else: + self._wrapper = None + + self._handlers = {} + if config is not None: + for err, action in config.items(): + if "warn" in action: + self._handlers[err] = self.warn + elif "err" in action: + self._handlers[err] = self.error + else: + self._handlers[err] = self.debug + + for err, action in self._errors.items(): + if err not in self._handlers: + if "warn" in action: + self._handlers[err] = self.warn + elif "err" in action: + self._handlers[err] = self.error + else: + self._handlers[err] = self.debug + + def dispatch(self, err, msg): + if err in self._handlers: + self._handlers[err](msg) + self.logger.debug(" (%s)" % err) + else: + # assume that it's an error, but complain + self.error(msg) + self.logger.warning("Unknown error %s" % err) + + def error(self, msg): + """ log an error condition """ + self.errors += 1 + self._log(msg, self.logger.error, prefix="ERROR: ") + + def warn(self, msg): + """ log a warning condition """ + self.warnings += 1 + self._log(msg, self.logger.warning, prefix="WARNING: ") + + def debug(self, msg): + """ log a silent/debug condition """ + self._log(msg, self.logger.debug) + + def _log(self, msg, logfunc, prefix=""): + # a message may itself consist of multiple lines. wrap() will + # elide them all into a single paragraph, which we don't want. + # so we split the message into its paragraphs and wrap each + # paragraph individually. this means, unfortunately, that we + # lose textwrap's built-in initial indent functionality, + # because we want to only treat the very first line of the + # first paragraph specially. so we do some silliness. + rawlines = msg.splitlines() + firstline = True + for rawline in rawlines: + if self._wrapper: + lines = self._wrapper.wrap(rawline) + else: + lines = [rawline] + for line in lines: + if firstline: + logfunc("%s%s" % (prefix, line.lstrip())) + firstline = False + else: + logfunc(line) + + +class ServerlessPlugin (Plugin): + """ base class for plugins that are run before the server starts + up (i.e., plugins that check things that may prevent the server + from starting up) """ + pass + + +class ServerPlugin (Plugin): + """ base class for plugins that check things that require the + running Bcfg2 server """ + def __init__(self, lintCore, config, **kwargs): + Plugin.__init__(self, config, **kwargs) + self.core = lintCore + self.logger = self.core.logger + self.metadata = self.core.metadata |