summaryrefslogtreecommitdiffstats
path: root/src/lib/Server/Lint
diff options
context:
space:
mode:
authorSol Jerome <sol.jerome@gmail.com>2012-03-24 11:20:07 -0500
committerSol Jerome <sol.jerome@gmail.com>2012-03-24 11:20:07 -0500
commitdab1d03d81c538966d03fb9318a4588a9e803b44 (patch)
treef51e27fa55887e9fb961766805fe43f0da56c5b9 /src/lib/Server/Lint
parent5cd6238df496a3cea178e4596ecd87967cce1ce6 (diff)
downloadbcfg2-dab1d03d81c538966d03fb9318a4588a9e803b44.tar.gz
bcfg2-dab1d03d81c538966d03fb9318a4588a9e803b44.tar.bz2
bcfg2-dab1d03d81c538966d03fb9318a4588a9e803b44.zip
Allow to run directly from a git checkout (#1037)
Signed-off-by: Sol Jerome <sol.jerome@gmail.com>
Diffstat (limited to 'src/lib/Server/Lint')
-rw-r--r--src/lib/Server/Lint/Bundles.py61
-rw-r--r--src/lib/Server/Lint/Comments.py187
-rw-r--r--src/lib/Server/Lint/Duplicates.py81
-rwxr-xr-xsrc/lib/Server/Lint/Genshi.py27
-rw-r--r--src/lib/Server/Lint/GroupPatterns.py31
-rw-r--r--src/lib/Server/Lint/InfoXML.py42
-rw-r--r--src/lib/Server/Lint/MergeFiles.py69
-rw-r--r--src/lib/Server/Lint/Pkgmgr.py35
-rw-r--r--src/lib/Server/Lint/RequiredAttrs.py138
-rw-r--r--src/lib/Server/Lint/Validate.py200
-rw-r--r--src/lib/Server/Lint/__init__.py215
11 files changed, 0 insertions, 1086 deletions
diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py
deleted file mode 100644
index 472915cfd..000000000
--- a/src/lib/Server/Lint/Bundles.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import lxml.etree
-import Bcfg2.Server.Lint
-
-class Bundles(Bcfg2.Server.Lint.ServerPlugin):
- """ Perform various bundle checks """
-
- def Run(self):
- """ run plugin """
- if 'Bundler' in self.core.plugins:
- self.missing_bundles()
- for bundle in self.core.plugins['Bundler'].entries.values():
- if self.HandlesFile(bundle.name):
- if (not Bcfg2.Server.Plugins.Bundler.have_genshi or
- type(bundle) is not
- Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile):
- self.bundle_names(bundle)
-
- def missing_bundles(self):
- """ find bundles listed in Metadata but not implemented in Bundler """
- if self.files is None:
- # when given a list of files on stdin, this check is
- # useless, so skip it
- groupdata = self.metadata.groups_xml.xdata
- ref_bundles = set([b.get("name")
- for b in groupdata.findall("//Bundle")])
-
- allbundles = self.core.plugins['Bundler'].entries.keys()
- for bundle in ref_bundles:
- xmlbundle = "%s.xml" % bundle
- genshibundle = "%s.genshi" % bundle
- if (xmlbundle not in allbundles and
- genshibundle not in allbundles):
- self.LintError("bundle-not-found",
- "Bundle %s referenced, but does not exist" %
- bundle)
-
- def bundle_names(self, bundle):
- """ verify bundle name attribute matches filename """
- try:
- xdata = lxml.etree.XML(bundle.data)
- except AttributeError:
- # genshi template
- xdata = lxml.etree.parse(bundle.template.filepath).getroot()
-
- fname = bundle.name.split('Bundler/')[1].split('.')[0]
- bname = xdata.get('name')
- if fname != bname:
- self.LintError("inconsistent-bundle-name",
- "Inconsistent bundle name: filename is %s, bundle name is %s" %
- (fname, bname))
-
- def sgenshi_groups(self, bundle):
- """ ensure that Genshi Bundles do not include <Group> tags,
- which are not supported """
- xdata = lxml.etree.parse(bundle.name)
- groups = [self.RenderXML(g)
- for g in xdata.getroottree().findall("//Group")]
- if groups:
- self.LintError("group-tag-not-allowed",
- "<Group> tag is not allowed in SGenshi Bundle:\n%s" %
- "\n".join(groups))
diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py
deleted file mode 100644
index 19fae1b08..000000000
--- a/src/lib/Server/Lint/Comments.py
+++ /dev/null
@@ -1,187 +0,0 @@
-import os.path
-import lxml.etree
-import Bcfg2.Server.Lint
-
-class Comments(Bcfg2.Server.Lint.ServerPlugin):
- """ check files for various required headers """
- def __init__(self, *args, **kwargs):
- Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
- self.config_cache = {}
-
- def Run(self):
- self.check_bundles()
- self.check_properties()
- self.check_metadata()
- self.check_cfg()
- self.check_infoxml()
- self.check_probes()
-
- def required_keywords(self, rtype):
- """ given a file type, fetch the list of required VCS keywords
- from the bcfg2-lint config """
- return self.required_items(rtype, "keyword")
-
- def required_comments(self, rtype):
- """ given a file type, fetch the list of required comments
- from the bcfg2-lint config """
- return self.required_items(rtype, "comment")
-
- def required_items(self, rtype, itype):
- """ given a file type and item type (comment or keyword),
- fetch the list of required items from the bcfg2-lint config """
- if itype not in self.config_cache:
- self.config_cache[itype] = {}
-
- if rtype not in self.config_cache[itype]:
- rv = []
- global_item = "global_%ss" % itype
- if global_item in self.config:
- rv.extend(self.config[global_item].split(","))
-
- item = "%s_%ss" % (rtype.lower(), itype)
- if item in self.config:
- if self.config[item]:
- rv.extend(self.config[item].split(","))
- else:
- # config explicitly specifies nothing
- rv = []
- self.config_cache[itype][rtype] = rv
- return self.config_cache[itype][rtype]
-
- def check_bundles(self):
- """ check bundle files for required headers """
- if 'Bundler' in self.core.plugins:
- for bundle in self.core.plugins['Bundler'].entries.values():
- xdata = None
- rtype = ""
- try:
- xdata = lxml.etree.XML(bundle.data)
- rtype = "bundler"
- except (lxml.etree.XMLSyntaxError, AttributeError):
- xdata = lxml.etree.parse(bundle.template.filepath).getroot()
- rtype = "sgenshi"
-
- self.check_xml(bundle.name, xdata, rtype)
-
- def check_properties(self):
- """ check properties files for required headers """
- if 'Properties' in self.core.plugins:
- props = self.core.plugins['Properties']
- for propfile, pdata in props.store.entries.items():
- if os.path.splitext(propfile)[1] == ".xml":
- self.check_xml(pdata.name, pdata.xdata, 'properties')
-
- def check_metadata(self):
- """ check metadata files for required headers """
- if self.has_all_xincludes("groups.xml"):
- self.check_xml(os.path.join(self.metadata.data, "groups.xml"),
- self.metadata.groups_xml.data,
- "metadata")
- if self.has_all_xincludes("clients.xml"):
- self.check_xml(os.path.join(self.metadata.data, "clients.xml"),
- self.metadata.clients_xml.data,
- "metadata")
-
- def check_cfg(self):
- """ check Cfg files for required headers """
- if 'Cfg' in self.core.plugins:
- for entryset in self.core.plugins['Cfg'].entries.values():
- for entry in entryset.entries.values():
- if entry.name.endswith(".genshi"):
- rtype = "tgenshi"
- else:
- rtype = "cfg"
- self.check_plaintext(entry.name, entry.data, rtype)
-
- def check_infoxml(self):
- """ check info.xml files for required headers """
- if 'Cfg' in self.core.plugins:
- for entryset in self.core.plugins['Cfg'].entries.items():
- if (hasattr(entryset, "infoxml") and
- entryset.infoxml is not None):
- self.check_xml(entryset.infoxml.name,
- entryset.infoxml.pnode.data,
- "infoxml")
-
- def check_probes(self):
- """ check probes for required headers """
- if 'Probes' in self.core.plugins:
- for probe in self.core.plugins['Probes'].probes.entries.values():
- self.check_plaintext(probe.name, probe.data, "probes")
-
- def check_xml(self, filename, xdata, rtype):
- """ check generic XML files for required headers """
- self.check_lines(filename,
- [str(el)
- for el in xdata.getiterator(lxml.etree.Comment)],
- rtype)
-
- def check_plaintext(self, filename, data, rtype):
- """ check generic plaintex files for required headers """
- self.check_lines(filename, data.splitlines(), rtype)
-
- def check_lines(self, filename, lines, rtype):
- """ generic header check for a set of lines """
- if self.HandlesFile(filename):
- # found is trivalent:
- # False == not found
- # None == found but not expanded
- # True == found and expanded
- found = dict((k, False) for k in self.required_keywords(rtype))
-
- for line in lines:
- # we check for both '$<keyword>:' and '$<keyword>$' to see
- # if the keyword just hasn't been expanded
- for (keyword, status) in found.items():
- if not status:
- if '$%s:' % keyword in line:
- found[keyword] = True
- elif '$%s$' % keyword in line:
- found[keyword] = None
-
- unexpanded = [keyword for (keyword, status) in found.items()
- if status is None]
- if unexpanded:
- self.LintError("unexpanded-keywords",
- "%s: Required keywords(s) found but not expanded: %s" %
- (filename, ", ".join(unexpanded)))
- missing = [keyword for (keyword, status) in found.items()
- if status is False]
- if missing:
- self.LintError("keywords-not-found",
- "%s: Required keywords(s) not found: $%s$" %
- (filename, "$, $".join(missing)))
-
- # next, check for required comments. found is just
- # boolean
- found = dict((k, False) for k in self.required_comments(rtype))
-
- for line in lines:
- for (comment, status) in found.items():
- if not status:
- found[comment] = comment in line
-
- missing = [comment for (comment, status) in found.items()
- if status is False]
- if missing:
- self.LintError("comments-not-found",
- "%s: Required comments(s) not found: %s" %
- (filename, ", ".join(missing)))
-
- def has_all_xincludes(self, mfile):
- """ return true if self.files includes all XIncludes listed in
- the specified metadata type, false otherwise"""
- if self.files is None:
- return True
- else:
- path = os.path.join(self.metadata.data, mfile)
- if path in self.files:
- xdata = lxml.etree.parse(path)
- for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'):
- if not self.has_all_xincludes(el.get('href')):
- self.LintError("broken-xinclude-chain",
- "Broken XInclude chain: could not include %s" % path)
- return False
-
- return True
-
diff --git a/src/lib/Server/Lint/Duplicates.py b/src/lib/Server/Lint/Duplicates.py
deleted file mode 100644
index 75f620603..000000000
--- a/src/lib/Server/Lint/Duplicates.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import os.path
-import lxml.etree
-import Bcfg2.Server.Lint
-
-class Duplicates(Bcfg2.Server.Lint.ServerPlugin):
- """ Find duplicate clients, groups, etc. """
- def __init__(self, *args, **kwargs):
- Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
- self.groups_xdata = None
- self.clients_xdata = None
- self.load_xdata()
-
- def Run(self):
- """ run plugin """
- # only run this plugin if we were not given a list of files.
- # not only is it marginally silly to run this plugin with a
- # partial list of files, it turns out to be really freaking
- # hard to get only a fragment of group or client metadata
- if self.groups_xdata is not None:
- self.duplicate_groups()
- self.duplicate_defaults()
- if self.clients_xdata is not None:
- self.duplicate_clients()
-
- def load_xdata(self):
- """ attempt to load XML data for groups and clients. only
- actually load data if all documents reference in XIncludes can
- be found in self.files"""
- if self.has_all_xincludes("groups.xml"):
- self.groups_xdata = self.metadata.clients_xml.xdata
- if self.has_all_xincludes("clients.xml"):
- self.clients_xdata = self.metadata.clients_xml.xdata
-
- def duplicate_groups(self):
- """ find duplicate groups """
- self.duplicate_entries(self.clients_xdata.xpath('//Groups/Group'),
- 'group')
-
- def duplicate_clients(self):
- """ find duplicate clients """
- self.duplicate_entries(self.clients_xdata.xpath('//Clients/Client'),
- 'client')
-
- def duplicate_entries(self, data, etype):
- """ generic duplicate entry finder """
- seen = {}
- for el in data:
- if el.get('name') not in seen:
- seen[el.get('name')] = el
- else:
- self.LintError("duplicate-%s" % etype,
- "Duplicate %s '%s':\n%s\n%s" %
- (etype, el.get('name'),
- self.RenderXML(seen[el.get('name')]),
- self.RenderXML(el)))
-
- def duplicate_defaults(self):
- """ check for multiple default group definitions """
- default_groups = [g for g in self.groups_xdata.findall('.//Group')
- if g.get('default') == 'true']
- if len(default_groups) > 1:
- self.LintError("multiple-default-groups",
- "Multiple default groups defined: %s" %
- ",".join(default_groups))
-
- def has_all_xincludes(self, mfile):
- """ return true if self.files includes all XIncludes listed in
- the specified metadata type, false otherwise"""
- if self.files is None:
- return True
- else:
- path = os.path.join(self.metadata.data, mfile)
- if path in self.files:
- xdata = lxml.etree.parse(path)
- for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'):
- if not self.has_all_xincludes(el.get('href')):
- self.LintError("broken-xinclude-chain",
- "Broken XInclude chain: could not include %s" % path)
- return False
-
- return True
diff --git a/src/lib/Server/Lint/Genshi.py b/src/lib/Server/Lint/Genshi.py
deleted file mode 100755
index 552c495b2..000000000
--- a/src/lib/Server/Lint/Genshi.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import genshi.template
-import Bcfg2.Server.Lint
-
-class Genshi(Bcfg2.Server.Lint.ServerPlugin):
- """ Check Genshi templates for syntax errors """
-
- def Run(self):
- """ run plugin """
- loader = genshi.template.TemplateLoader()
- for plugin in ['Cfg', 'TGenshi']:
- if plugin in self.core.plugins:
- self.check_files(self.core.plugins[plugin].entries,
- loader=loader)
-
- def check_files(self, entries, loader=None):
- if loader is None:
- loader = genshi.template.TemplateLoader()
-
- for eset in entries.values():
- for fname, sdata in list(eset.entries.items()):
- if fname.endswith(".genshi") or fname.endswith(".newtxt"):
- try:
- loader.load(sdata.name,
- cls=genshi.template.NewTextTemplate)
- except genshi.template.TemplateSyntaxError, err:
- self.LintError("genshi-syntax-error",
- "Genshi syntax error: %s" % err)
diff --git a/src/lib/Server/Lint/GroupPatterns.py b/src/lib/Server/Lint/GroupPatterns.py
deleted file mode 100644
index b69d7a5d8..000000000
--- a/src/lib/Server/Lint/GroupPatterns.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import sys
-import Bcfg2.Server.Lint
-from Bcfg2.Server.Plugins.GroupPatterns import PatternMap
-
-class GroupPatterns(Bcfg2.Server.Lint.ServerPlugin):
- """ Check Genshi templates for syntax errors """
-
- def Run(self):
- """ run plugin """
- if 'GroupPatterns' in self.core.plugins:
- cfg = self.core.plugins['GroupPatterns'].config
- for entry in cfg.xdata.xpath('//GroupPattern'):
- groups = [g.text for g in entry.findall('Group')]
- self.check(entry, groups, ptype='NamePattern')
- self.check(entry, groups, ptype='NameRange')
-
- def check(self, entry, groups, ptype="NamePattern"):
- if ptype == "NamePattern":
- pmap = lambda p: PatternMap(p, None, groups)
- else:
- pmap = lambda p: PatternMap(None, p, groups)
-
- for el in entry.findall(ptype):
- pat = el.text
- try:
- pmap(pat)
- except:
- err = sys.exc_info()[1]
- self.LintError("pattern-fails-to-initialize",
- "Failed to initialize %s %s for %s: %s" %
- (ptype, pat, entry.get('pattern'), err))
diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py
deleted file mode 100644
index 2054e23bf..000000000
--- a/src/lib/Server/Lint/InfoXML.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import os.path
-import Bcfg2.Options
-import Bcfg2.Server.Lint
-
-class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
- """ ensure that all config files have an info.xml file"""
-
- def Run(self):
- if 'Cfg' in self.core.plugins:
- for filename, entryset in self.core.plugins['Cfg'].entries.items():
- infoxml_fname = os.path.join(entryset.path, "info.xml")
- if self.HandlesFile(infoxml_fname):
- if (hasattr(entryset, "infoxml") and
- entryset.infoxml is not None):
- self.check_infoxml(infoxml_fname,
- entryset.infoxml.pnode.data)
- else:
- self.LintError("no-infoxml",
- "No info.xml found for %s" % filename)
-
- def check_infoxml(self, fname, xdata):
- for info in xdata.getroottree().findall("//Info"):
- required = []
- if "required_attrs" in self.config:
- required = self.config["required_attrs"].split(",")
-
- missing = [attr for attr in required if info.get(attr) is None]
- if missing:
- self.LintError("required-infoxml-attrs-missing",
- "Required attribute(s) %s not found in %s:%s" %
- (",".join(missing), fname, self.RenderXML(info)))
-
- if ((Bcfg2.Options.MDATA_PARANOID.value and
- info.get("paranoid") is not None and
- info.get("paranoid").lower() == "false") or
- (not Bcfg2.Options.MDATA_PARANOID.value and
- (info.get("paranoid") is None or
- info.get("paranoid").lower() != "true"))):
- self.LintError("paranoid-false",
- "Paranoid must be true in %s:%s" %
- (fname, self.RenderXML(info)))
-
diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py
deleted file mode 100644
index ff6e3449a..000000000
--- a/src/lib/Server/Lint/MergeFiles.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import os
-import copy
-from difflib import SequenceMatcher
-import Bcfg2.Server.Lint
-
-class MergeFiles(Bcfg2.Server.Lint.ServerPlugin):
- """ find Probes or Cfg files with multiple similar files that
- might be merged into one """
-
- def Run(self):
- if 'Cfg' in self.core.plugins:
- self.check_cfg()
- if 'Probes' in self.core.plugins:
- self.check_probes()
-
- def check_cfg(self):
- for filename, entryset in self.core.plugins['Cfg'].entries.items():
- for mset in self.get_similar(entryset.entries):
- self.LintError("merge-cfg",
- "The following files are similar: %s. "
- "Consider merging them into a single Genshi "
- "template." %
- ", ".join([os.path.join(filename, p)
- for p in mset]))
-
- def check_probes(self):
- probes = self.core.plugins['Probes'].probes.entries
- for mset in self.get_similar(probes):
- self.LintError("merge-cfg",
- "The following probes are similar: %s. "
- "Consider merging them into a single probe." %
- ", ".join([p for p in mset]))
-
- def get_similar(self, entries):
- if "threshold" in self.config:
- # accept threshold either as a percent (e.g., "threshold=75") or
- # as a ratio (e.g., "threshold=.75")
- threshold = float(self.config['threshold'])
- if threshold > 1:
- threshold /= 100
- else:
- threshold = 0.75
- rv = []
- elist = entries.items()
- while elist:
- result = self._find_similar(elist.pop(0), copy.copy(elist),
- threshold)
- if len(result) > 1:
- elist = [(fname, fdata)
- for fname, fdata in elist
- if fname not in result]
- rv.append(result)
- return rv
-
- def _find_similar(self, ftuple, others, threshold):
- fname, fdata = ftuple
- rv = [fname]
- while others:
- cname, cdata = others.pop(0)
- sm = SequenceMatcher(None, fdata.data, cdata.data)
- # perform progressively more expensive comparisons
- if (sm.real_quick_ratio() > threshold and
- sm.quick_ratio() > threshold and
- sm.ratio() > threshold):
- rv.extend(self._find_similar((cname, cdata), copy.copy(others),
- threshold))
- return rv
-
-
diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Server/Lint/Pkgmgr.py
deleted file mode 100644
index 8f099163a..000000000
--- a/src/lib/Server/Lint/Pkgmgr.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import glob
-import lxml.etree
-import Bcfg2.Server.Lint
-
-class Pkgmgr(Bcfg2.Server.Lint.ServerlessPlugin):
- """ find duplicate Pkgmgr entries with the same priority """
-
- def Run(self):
- pset = set()
- for pfile in glob.glob("%s/Pkgmgr/*.xml" % self.config['repo']):
- if self.HandlesFile(pfile):
- xdata = lxml.etree.parse(pfile).getroot()
- # get priority, type, group
- priority = xdata.get('priority')
- ptype = xdata.get('type')
- for pkg in xdata.xpath("//Package"):
- if pkg.getparent().tag == 'Group':
- grp = pkg.getparent().get('name')
- if (type(grp) is not str and
- grp.getparent().tag == 'Group'):
- pgrp = grp.getparent().get('name')
- else:
- pgrp = 'none'
- else:
- grp = 'none'
- pgrp = 'none'
- ptuple = (pkg.get('name'), priority, ptype, grp, pgrp)
- # check if package is already listed with same
- # priority, type, grp
- if ptuple in pset:
- self.LintError("duplicate-package",
- "Duplicate Package %s, priority:%s, type:%s" %
- (pkg.get('name'), priority, ptype))
- else:
- pset.add(ptuple)
diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Server/Lint/RequiredAttrs.py
deleted file mode 100644
index 55206d2ba..000000000
--- a/src/lib/Server/Lint/RequiredAttrs.py
+++ /dev/null
@@ -1,138 +0,0 @@
-import os.path
-import lxml.etree
-import Bcfg2.Server.Lint
-from Bcfg2.Server.Plugins.Packages import Apt, Yum
-
-class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
- """ verify attributes for configuration entries (as defined in
- doc/server/configurationentries) """
-
- def __init__(self, *args, **kwargs):
- Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
- self.required_attrs = {
- 'Path': {
- 'device': ['name', 'owner', 'group', 'dev_type'],
- 'directory': ['name', 'owner', 'group', 'perms'],
- 'file': ['name', 'owner', 'group', 'perms', '__text__'],
- 'hardlink': ['name', 'to'],
- 'symlink': ['name', 'to'],
- 'ignore': ['name'],
- 'nonexistent': ['name'],
- 'permissions': ['name', 'owner', 'group', 'perms'],
- 'vcs': ['vcstype', 'revision', 'sourceurl']},
- 'Service': {
- 'chkconfig': ['name'],
- 'deb': ['name'],
- 'rc-update': ['name'],
- 'smf': ['name', 'FMRI'],
- 'upstart': ['name']},
- 'Action': ['name', 'timing', 'when', 'status', 'command'],
- 'Package': ['name']}
-
- def Run(self):
- self.check_packages()
- if "Defaults" in self.core.plugins:
- self.logger.info("Defaults plugin enabled; skipping required "
- "attribute checks")
- else:
- self.check_rules()
- self.check_bundles()
-
- def check_packages(self):
- """ check package sources for Source entries with missing attrs """
- if 'Packages' in self.core.plugins:
- for source in self.core.plugins['Packages'].sources:
- if isinstance(source, Yum.YumSource):
- if (not source.pulp_id and not source.url and
- not source.rawurl):
- self.LintError("required-attrs-missing",
- "A %s source must have either a url, "
- "rawurl, or pulp_id attribute: %s" %
- (source.ptype,
- self.RenderXML(source.xsource)))
- elif not source.url and not source.rawurl:
- self.LintError("required-attrs-missing",
- "A %s source must have either a url or "
- "rawurl attribute: %s" %
- (source.ptype,
- self.RenderXML(source.xsource)))
-
- if (not isinstance(source, Apt.AptSource) and
- source.recommended):
- self.LintError("extra-attrs",
- "The recommended attribute is not "
- "supported on %s sources: %s" %
- (source.ptype,
- self.RenderXML(source.xsource)))
-
- def check_rules(self):
- """ check Rules for Path entries with missing attrs """
- if 'Rules' in self.core.plugins:
- for rules in self.core.plugins['Rules'].entries.values():
- xdata = rules.pnode.data
- for path in xdata.xpath("//Path"):
- self.check_entry(path, os.path.join(self.config['repo'],
- rules.name))
-
- def check_bundles(self):
- """ check bundles for BoundPath entries with missing attrs """
- if 'Bundler' in self.core.plugins:
- for bundle in self.core.plugins['Bundler'].entries.values():
- try:
- xdata = lxml.etree.XML(bundle.data)
- except (lxml.etree.XMLSyntaxError, AttributeError):
- xdata = lxml.etree.parse(bundle.template.filepath).getroot()
-
- for path in xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"):
- self.check_entry(path, bundle.name)
-
- def check_entry(self, entry, filename):
- """ generic entry check """
- if self.HandlesFile(filename):
- name = entry.get('name')
- tag = entry.tag
- if tag.startswith("Bound"):
- tag = tag[5:]
- if tag not in self.required_attrs:
- self.LintError("unknown-entry-tag",
- "Unknown entry tag '%s': %s" %
- (entry.tag, self.RenderXML(entry)))
-
- if isinstance(self.required_attrs[tag], dict):
- etype = entry.get('type')
- if etype in self.required_attrs[tag]:
- required_attrs = set(self.required_attrs[tag][etype] +
- ['type'])
- else:
- self.LintError("unknown-entry-type",
- "Unknown %s type %s: %s" %
- (tag, etype, self.RenderXML(entry)))
- return
- else:
- required_attrs = set(self.required_attrs[tag])
- attrs = set(entry.attrib.keys())
-
- if 'dev_type' in required_attrs:
- dev_type = entry.get('dev_type')
- if dev_type in ['block', 'char']:
- # check if major/minor are specified
- required_attrs |= set(['major', 'minor'])
-
- if '__text__' in required_attrs:
- required_attrs.remove('__text__')
- if (not entry.text and
- not entry.get('empty', 'false').lower() == 'true'):
- self.LintError("required-attrs-missing",
- "Text missing for %s %s in %s: %s" %
- (entry.tag, name, filename,
- self.RenderXML(entry)))
-
- if not attrs.issuperset(required_attrs):
- self.LintError("required-attrs-missing",
- "The following required attribute(s) are "
- "missing for %s %s in %s: %s\n%s" %
- (entry.tag, name, filename,
- ", ".join([attr
- for attr in
- required_attrs.difference(attrs)]),
- self.RenderXML(entry)))
diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py
deleted file mode 100644
index 952a65365..000000000
--- a/src/lib/Server/Lint/Validate.py
+++ /dev/null
@@ -1,200 +0,0 @@
-import fnmatch
-import glob
-import lxml.etree
-import os
-from subprocess import Popen, PIPE, STDOUT
-import sys
-
-import Bcfg2.Server.Lint
-
-class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
- """ Ensure that the repo validates """
-
- def __init__(self, *args, **kwargs):
- Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs)
- self.filesets = {"metadata:groups":"%s/metadata.xsd",
- "metadata:clients":"%s/clients.xsd",
- "info":"%s/info.xsd",
- "%s/Bundler/*.xml":"%s/bundle.xsd",
- "%s/Bundler/*.genshi":"%s/bundle.xsd",
- "%s/Pkgmgr/*.xml":"%s/pkglist.xsd",
- "%s/Base/*.xml":"%s/base.xsd",
- "%s/Rules/*.xml":"%s/rules.xsd",
- "%s/Defaults/*.xml":"%s/defaults.xsd",
- "%s/etc/report-configuration.xml":"%s/report-configuration.xsd",
- "%s/Svcmgr/*.xml":"%s/services.xsd",
- "%s/Deps/*.xml":"%s/deps.xsd",
- "%s/Decisions/*.xml":"%s/decisions.xsd",
- "%s/Packages/sources.xml":"%s/packages.xsd",
- "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd",
- "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd",
- "%s/FileProbes/config.xml":"%s/fileprobes.xsd",
- }
-
- self.filelists = {}
- self.get_filelists()
-
- def Run(self):
- schemadir = self.config['schema']
-
- for path, schemaname in self.filesets.items():
- try:
- filelist = self.filelists[path]
- except KeyError:
- filelist = []
-
- if filelist:
- # avoid loading schemas for empty file lists
- schemafile = schemaname % schemadir
- try:
- schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile))
- except IOError:
- e = sys.exc_info()[1]
- self.LintError("input-output-error", str(e))
- continue
- except lxml.etree.XMLSchemaParseError:
- e = sys.exc_info()[1]
- self.LintError("schema-failed-to-parse",
- "Failed to process schema %s: %s" %
- (schemafile, e))
- continue
- for filename in filelist:
- self.validate(filename, schemafile, schema=schema)
-
- self.check_properties()
-
- def check_properties(self):
- """ check Properties files against their schemas """
- for filename in self.filelists['props']:
- schemafile = "%s.xsd" % os.path.splitext(filename)[0]
- if os.path.exists(schemafile):
- self.validate(filename, schemafile)
- else:
- self.LintError("properties-schema-not-found",
- "No schema found for %s" % filename)
-
- def validate(self, filename, schemafile, schema=None):
- """validate a file against the given lxml.etree.Schema.
- return True on success, False on failure """
- if schema is None:
- # if no schema object was provided, instantiate one
- try:
- schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile))
- except:
- self.LintError("schema-failed-to-parse",
- "Failed to process schema %s" % schemafile)
- return False
-
- try:
- datafile = lxml.etree.parse(filename)
- except SyntaxError:
- lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT)
- self.LintError("xml-failed-to-parse",
- "%s fails to parse:\n%s" % (filename,
- lint.communicate()[0]))
- lint.wait()
- return False
- except IOError:
- self.LintError("xml-failed-to-read",
- "Failed to open file %s" % filename)
- return False
-
- if not schema.validate(datafile):
- cmd = ["xmllint"]
- if self.files is None:
- cmd.append("--xinclude")
- cmd.extend(["--noout", "--schema", schemafile, filename])
- lint = Popen(cmd, stdout=PIPE, stderr=STDOUT)
- output = lint.communicate()[0]
- if lint.wait():
- self.LintError("xml-failed-to-verify",
- "%s fails to verify:\n%s" % (filename, output))
- return False
- return True
-
- def get_filelists(self):
- """ get lists of different kinds of files to validate """
- if self.files is not None:
- listfiles = lambda p: fnmatch.filter(self.files, p % "*")
- else:
- listfiles = lambda p: glob.glob(p % self.config['repo'])
-
- for path in self.filesets.keys():
- if path.startswith("metadata:"):
- mtype = path.split(":")[1]
- self.filelists[path] = self.get_metadata_list(mtype)
- elif path == "info":
- if self.files is not None:
- self.filelists[path] = \
- [f for f in self.files
- if os.path.basename(f) == 'info.xml']
- else: # self.files is None
- self.filelists[path] = []
- for infodir in ['Cfg', 'TGenshi', 'TCheetah']:
- for root, dirs, files in os.walk('%s/%s' %
- (self.config['repo'],
- infodir)):
- self.filelists[path].extend([os.path.join(root, f)
- for f in files
- if f == 'info.xml'])
- else:
- self.filelists[path] = listfiles(path)
-
- self.filelists['props'] = listfiles("%s/Properties/*.xml")
- all_metadata = listfiles("%s/Metadata/*.xml")
-
- # if there are other files in Metadata that aren't xincluded
- # from clients.xml or groups.xml, we can't verify them. warn
- # about those.
- for fname in all_metadata:
- if (fname not in self.filelists['metadata:groups'] and
- fname not in self.filelists['metadata:clients']):
- self.LintError("broken-xinclude-chain",
- "Broken XInclude chain: Could not determine file type of %s" % fname)
-
- def get_metadata_list(self, mtype):
- """ get all metadata files for the specified type (clients or
- group) """
- if self.files is not None:
- rv = fnmatch.filter(self.files, "*/Metadata/%s.xml" % mtype)
- else:
- rv = glob.glob("%s/Metadata/%s.xml" % (self.config['repo'], mtype))
-
- # attempt to follow XIncludes. if the top-level files aren't
- # listed in self.files, though, there's really nothing we can
- # do to guess what a file in Metadata is
- if rv:
- try:
- rv.extend(self.follow_xinclude(rv[0]))
- except lxml.etree.XMLSyntaxError:
- e = sys.exc_info()[1]
- self.LintError("xml-failed-to-parse",
- "%s fails to parse:\n%s" % (rv[0], e))
-
-
- return rv
-
- def follow_xinclude(self, xfile):
- """ follow xincludes in the given file """
- xdata = lxml.etree.parse(xfile)
- included = set([ent.get('href') for ent in
- xdata.findall('./{http://www.w3.org/2001/XInclude}include')])
- rv = []
-
- while included:
- try:
- filename = included.pop()
- except KeyError:
- continue
-
- path = os.path.join(os.path.dirname(xfile), filename)
- if self.HandlesFile(path):
- rv.append(path)
- groupdata = lxml.etree.parse(path)
- [included.add(el.get('href'))
- for el in
- groupdata.findall('./{http://www.w3.org/2001/XInclude}include')]
- included.discard(filename)
-
- return rv
-
diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py
deleted file mode 100644
index 326f5b1dc..000000000
--- a/src/lib/Server/Lint/__init__.py
+++ /dev/null
@@ -1,215 +0,0 @@
-__all__ = ['Bundles',
- 'Comments',
- 'Duplicates',
- 'InfoXML',
- 'MergeFiles',
- 'Pkgmgr',
- 'RequiredAttrs',
- 'Validate',
- 'Genshi']
-
-import logging
-import os
-import sys
-from copy import copy
-import textwrap
-import lxml.etree
-import Bcfg2.Logger
-import fcntl
-import termios
-import struct
-
-def _ioctl_GWINSZ(fd):
- try:
- cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
- except:
- return None
- return cr
-
-def get_termsize():
- """ get a tuple of (width, height) giving the size of the terminal """
- if not sys.stdout.isatty():
- return None
- cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2)
- if not cr:
- try:
- fd = os.open(os.ctermid(), os.O_RDONLY)
- cr = _ioctl_GWINSZ(fd)
- os.close(fd)
- except:
- pass
- if not cr:
- try:
- cr = (os.environ['LINES'], os.environ['COLUMNS'])
- except KeyError:
- return None
- return int(cr[1]), int(cr[0])
-
-class Plugin (object):
- """ base class for ServerlessPlugin and ServerPlugin """
-
- def __init__(self, config, errorhandler=None, files=None):
- self.files = files
- self.config = config
- self.logger = logging.getLogger('bcfg2-lint')
- if errorhandler is None:
- self.errorHandler = ErrorHandler()
- else:
- self.errorHandler = errorhandler
-
- def Run(self):
- """ run the plugin. must be overloaded by child classes """
- pass
-
- def HandlesFile(self, fname):
- """ returns true if the given file should be handled by the
- plugin according to the files list, false otherwise """
- return (self.files is None or
- fname in self.files or
- os.path.join(self.config['repo'], fname) in self.files or
- os.path.abspath(fname) in self.files or
- os.path.abspath(os.path.join(self.config['repo'],
- fname)) in self.files)
-
- def LintError(self, err, msg):
- self.errorHandler.dispatch(err, msg)
-
- def RenderXML(self, element):
- """render an XML element for error output -- line number
- prefixed, no children"""
- xml = None
- if len(element) or element.text:
- el = copy(element)
- if el.text:
- el.text = '...'
- [el.remove(c) for c in el.iterchildren()]
- xml = lxml.etree.tostring(el).strip()
- else:
- xml = lxml.etree.tostring(element).strip()
- return " line %s: %s" % (element.sourceline, xml)
-
-
-class ErrorHandler (object):
- # how to handle different errors by default
- _errors = {"no-infoxml":"warning",
- "paranoid-false":"warning",
- "bundle-not-found":"error",
- "inconsistent-bundle-name":"warning",
- "group-tag-not-allowed":"error",
- "unexpanded-keywords":"warning",
- "keywords-not-found":"warning",
- "comments-not-found":"warning",
- "broken-xinclude-chain":"warning",
- "duplicate-client":"error",
- "duplicate-group":"error",
- "duplicate-package":"error",
- "multiple-default-groups":"error",
- "required-infoxml-attrs-missing":"error",
- "unknown-entry-type":"error",
- "required-attrs-missing":"error",
- "extra-attrs":"warning",
- "schema-failed-to-parse":"warning",
- "properties-schema-not-found":"warning",
- "xml-failed-to-parse":"error",
- "xml-failed-to-read":"error",
- "xml-failed-to-verify":"error",
- "merge-cfg":"warning",
- "merge-probes":"warning",
- "input-output-error":"error",
- "genshi-syntax-error":"error",
- "pattern-fails-to-initialize":"error"}
-
- def __init__(self, config=None):
- self.errors = 0
- self.warnings = 0
-
- self.logger = logging.getLogger('bcfg2-lint')
-
- termsize = get_termsize()
- if termsize is not None:
- self._wrapper = textwrap.TextWrapper(initial_indent=" ",
- subsequent_indent=" ",
- width=termsize[0])
- else:
- self._wrapper = None
-
- self._handlers = {}
- if config is not None:
- for err, action in config.items():
- if "warn" in action:
- self._handlers[err] = self.warn
- elif "err" in action:
- self._handlers[err] = self.error
- else:
- self._handlers[err] = self.debug
-
- for err, action in self._errors.items():
- if err not in self._handlers:
- if "warn" in action:
- self._handlers[err] = self.warn
- elif "err" in action:
- self._handlers[err] = self.error
- else:
- self._handlers[err] = self.debug
-
- def dispatch(self, err, msg):
- if err in self._handlers:
- self._handlers[err](msg)
- self.logger.debug(" (%s)" % err)
- else:
- # assume that it's an error, but complain
- self.error(msg)
- self.logger.warning("Unknown error %s" % err)
-
- def error(self, msg):
- """ log an error condition """
- self.errors += 1
- self._log(msg, self.logger.error, prefix="ERROR: ")
-
- def warn(self, msg):
- """ log a warning condition """
- self.warnings += 1
- self._log(msg, self.logger.warning, prefix="WARNING: ")
-
- def debug(self, msg):
- """ log a silent/debug condition """
- self._log(msg, self.logger.debug)
-
- def _log(self, msg, logfunc, prefix=""):
- # a message may itself consist of multiple lines. wrap() will
- # elide them all into a single paragraph, which we don't want.
- # so we split the message into its paragraphs and wrap each
- # paragraph individually. this means, unfortunately, that we
- # lose textwrap's built-in initial indent functionality,
- # because we want to only treat the very first line of the
- # first paragraph specially. so we do some silliness.
- rawlines = msg.splitlines()
- firstline = True
- for rawline in rawlines:
- if self._wrapper:
- lines = self._wrapper.wrap(rawline)
- else:
- lines = [rawline]
- for line in lines:
- if firstline:
- logfunc("%s%s" % (prefix, line.lstrip()))
- firstline = False
- else:
- logfunc(line)
-
-
-class ServerlessPlugin (Plugin):
- """ base class for plugins that are run before the server starts
- up (i.e., plugins that check things that may prevent the server
- from starting up) """
- pass
-
-
-class ServerPlugin (Plugin):
- """ base class for plugins that check things that require the
- running Bcfg2 server """
- def __init__(self, lintCore, config, **kwargs):
- Plugin.__init__(self, config, **kwargs)
- self.core = lintCore
- self.logger = self.core.logger
- self.metadata = self.core.metadata