Rewrote bcfg2-repo-validate as bcfg2-lint, which uses a plugin

interface to be lots more flexible and extensible.  Added several more
tests.  If bcfg2-lint is run as bcfg2-repo-validate, it roughly
emulates the functionality of that program.

TODO: Need to figure out correct way to symlink bcfg2-repo-validate to
bcfg2-lint on install.

1 files changed, 0 insertions, 328 deletions

diff --git a/src/sbin/bcfg2-repo-validate b/src/sbin/bcfg2-repo-validate
deleted file mode 100755
index e82b57659..000000000
--- a/src/sbin/bcfg2-repo-validate
+++ /dev/null
@@ -1,328 +0,0 @@
-#!/usr/bin/env python
-
-"""
-bcfg2-repo-validate checks all xml files in Bcfg2
-repos against their respective XML schemas.
-"""
-__revision__ = '$Revision$'
-
-import fnmatch
-import glob
-import lxml.etree
-import os
-import sys
-import fnmatch
-import logging
-import Bcfg2.Options
-from subprocess import Popen, PIPE, STDOUT
-
-def follow_xinclude(xfile, file_list=None):
-    """ follow xincludes in the given file """
-    xdata = lxml.etree.parse(xfile)
-    included = set([ent.get('href') for ent in
-                    xdata.findall('./{http://www.w3.org/2001/XInclude}include')])
-    rv = []
-        
-    while included:
-        try:
-            filename = included.pop()
-        except KeyError:
-            continue
-
-        path = os.path.join(os.path.dirname(xfile), filename)
-        if file_list is not None and path in file_list:
-            rv.append(path)
-            groupdata = lxml.etree.parse(path)
-            [included.add(el.get('href'))
-             for el in
-             groupdata.findall('./{http://www.w3.org/2001/XInclude}include')]
-        included.discard(filename)
-
-    return rv
-
-def validate(filename, schemafile, schema=None, xinclude=True):
-    """validate a fail against the given lxml.etree.Schema.  return
-    True on success, False on failure"""
-    if schema is None:
-        # if no schema object was provided, instantiate one
-        try:
-            schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile))
-        except:
-            logging.warn("Failed to process schema %s", schemafile)
-            return False
-
-    try:
-        datafile = lxml.etree.parse(filename)
-    except SyntaxError:
-        logging.warn("%s ***FAILS*** to parse \t\t<----", filename)
-        lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT)
-        logging.warn(lint.communicate()[0])
-        lint.wait()
-        return False
-    except IOError:
-        logging.warn("Failed to open file %s \t\t<---", filename)
-        return False
-    
-    if schema.validate(datafile):
-        logging.info("%s checks out", filename)
-    else:
-        cmd = ["xmllint"]
-        if xinclude:
-            cmd.append("--xinclude")
-        cmd.extend(["--noout", "--schema", schemafile, filename])
-        lint = Popen(cmd, stdout=PIPE, stderr=STDOUT)
-        output = lint.communicate()[0]
-        if lint.wait():
-            logging.warn("%s ***FAILS*** to verify \t\t<----", filename)
-            logging.warn(output)
-            return False
-        else:
-            logging.info("%s checks out", filename)
-    return True
-
-if __name__ == '__main__':
-    opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY,
-            'verbose': Bcfg2.Options.VERBOSE,
-            'configfile': Bcfg2.Options.CFILE,
-            'require-schema': Bcfg2.Options.REQUIRE_SCHEMA,
-            'schema': Bcfg2.Options.SCHEMA_PATH,
-            'stdin': Bcfg2.Options.FILES_ON_STDIN}
-    setup = Bcfg2.Options.OptionParser(opts)
-    setup.parse(sys.argv[1:])
-    verbose = setup['verbose']
-    cpath = setup['configfile']
-    schemadir = setup['schema']
-    os.chdir(schemadir)
-    repo = setup['repo']
-
-    # set up logging
-    level = logging.WARNING
-    if verbose:
-        level = logging.INFO
-    logging.basicConfig(level=level, format="%(message)s")
-
-    if setup['stdin']:
-        file_list = [s.strip() for s in sys.stdin.readlines()]
-        info_list = [f for f in file_list if os.path.basename(f) == 'info.xml']
-        metadata_list = fnmatch.filter(file_list, "*/Metadata/groups.xml")
-        clients_list = fnmatch.filter(file_list, "*/Metadata/clients.xml")
-        bundle_list = fnmatch.filter(file_list, "*/Bundler/*.xml")
-        genshibundle_list = fnmatch.filter(file_list, "*/Bundler/*.genshi")
-        pkg_list = fnmatch.filter(file_list, "*/Pkgmgr/*.xml")
-        base_list = fnmatch.filter(file_list, "*/Base/*.xml")
-        rules_list = fnmatch.filter(file_list, "*/Rules/*.xml")
-        imageinfo_list = fnmatch.filter(file_list,
-                                        "*/etc/report-configuration.xml")
-        services_list = fnmatch.filter(file_list, "*/Svcmgr/*.xml")
-        deps_list = fnmatch.filter(file_list, "*/Deps/*.xml")
-        dec_list = fnmatch.filter(file_list, "*/Decisions/*")
-        pkgcfg_list = fnmatch.filter(file_list, "*/Packages/config.xml")
-        gp_list = fnmatch.filter(file_list, "*/GroupPatterns/config.xml")
-        props_list = [f
-                      for f in fnmatch.filter(file_list, "*/Properties/*.xml")
-                      if "%s.xsd" % os.path.splitext(f)[0] in file_list]
-
-        # attempt to follow XIncludes in groups.xml and clients.xml.
-        # if those top-level files aren't listed in file_list, though,
-        # there's really nothing we can do to guess what a file in
-        # Metadata is
-        if metadata_list:
-            metadata_list.extend(follow_xinclude(metadata_list[0],
-                                                 file_list=file_list))
-        if clients_list:
-            clients_list.extend(follow_xinclude(clients_list[0],
-                                                file_list=file_list))
-
-        # if there are other files in Metadata in file_list that
-        # aren't listed in metadata_list or clients_list, we can't
-        # verify them.  warn about those.
-        for fname in fnmatch.filter(file_list, "*/Metadata/*.xml"):
-            if fname not in metadata_list and fname not in clients_list:
-                logging.warn("Broken XInclude chain: Could not determine file type of %s", fname)
-    else:
-        # not reading files from stdin
-
-        # Get a list of all info.xml files in the bcfg2 repository
-        info_list = []
-        for infodir in ['Cfg', 'TGenshi', 'TCheetah']:
-            for root, dirs, files in os.walk('%s/%s' % (repo, infodir)):
-                info_list.extend([os.path.join(root, f) for f in files
-                                  if f == 'info.xml'])
-
-        # get metadata list
-        metadata_list = glob.glob("%s/Metadata/groups.xml" % repo)
-
-        # get other file lists
-        clients_list = glob.glob("%s/Metadata/clients.xml" % repo)
-        bundle_list = glob.glob("%s/Bundler/*.xml" % repo)
-        genshibundle_list = glob.glob("%s/Bundler/*.genshi" % repo)
-        pkg_list = glob.glob("%s/Pkgmgr/*.xml" % repo)
-        base_list = glob.glob("%s/Base/*.xml" % repo)
-        rules_list = glob.glob("%s/Rules/*.xml" % repo)
-        imageinfo_list = glob.glob("%s/etc/report-configuration.xml" % repo)
-        services_list = glob.glob("%s/Svcmgr/*.xml" % repo)
-        deps_list = glob.glob("%s/Deps/*.xml" % repo)
-        dec_list = glob.glob("%s/Decisions/*" % repo)
-        pkgcfg_list = glob.glob("%s/Packages/config.xml" % repo)
-        gp_list = glob.glob('%s/GroupPatterns/config.xml' % repo)
-        props_list = glob.glob("%s/Properties/*.xml" % repo)
-
-        metadata_list.extend(follow_xinclude("%s/Metadata/groups.xml" % repo))
-        clients_list.extend(follow_xinclude("%s/Metadata/clients.xml" % repo))
-
-        # get all bundles
-        ref_bundles = set()
-        xdata = lxml.etree.parse("%s/Metadata/groups.xml" % repo)
-        xdata.xinclude()
-        for bundle in xdata.findall("//Bundle"):
-            ref_bundles.add("%s/Bundler/%s" % (repo, bundle.get('name')))
-        included = set([ent.get('href') for ent in
-                        xdata.findall('./{http://www.w3.org/2001/XInclude}include')])
-
-    # check for multiple default group definitions
-    if "%s/Metadata/groups.xml" % repo in metadata_list:
-        default_groups = [g for g in lxml.etree.parse("%s/Metadata/groups.xml" %
-                                                      repo).findall('.//Group')
-                          if g.get('default') == 'true']
-        if len(default_groups) > 1:
-            logging.warn("*** Warning: Multiple default groups defined")
-            for grp in default_groups:
-                logging.warn("  %s", grp.get('name'))
-
-    # verify attributes for configuration entries
-    # (as defined in doc/server/configurationentries)
-    # TODO: See if it is possible to do this in the schema instead
-    required_configuration_attrs = {
-        'device': ['name', 'owner', 'group', 'dev_type'],
-        'directory': ['name', 'owner', 'group', 'perms'],
-        'file': ['name', 'owner', 'group', 'perms'],
-        'hardlink': ['name', 'to'],
-        'symlink': ['name', 'to'],
-        'ignore': ['name'],
-        'nonexistent': ['name'],
-        'permissions': ['name', 'owner', 'group', 'perms']}
-    for rfile in rules_list:
-        try:
-            xdata = lxml.etree.parse(rfile)
-        except lxml.etree.XMLSyntaxError, e:
-            logging.warn("Failed to parse %s: %s", rfile, e)
-        for posixpath in xdata.findall("//Path"):
-            pathname = posixpath.get('name')
-            pathtype = posixpath.get('type')
-            pathset = set(posixpath.attrib.keys())
-            try:
-                required_attrs = set(required_configuration_attrs[pathtype] \
-                                     + ['type'])
-            except KeyError:
-                continue
-            if 'dev_type' in required_attrs:
-                dev_type = posixpath.get('dev_type')
-                if dev_type in ['block', 'char']:
-                    # check if major/minor are specified
-                    required_attrs |= set(['major', 'minor'])
-            if pathset.issuperset(required_attrs):
-                continue
-            else:
-                logging.warn("The following required attributes are missing for"
-                             " Path %s in %s: %s",
-                             pathname, rfile,
-                             [attr
-                              for attr in required_attrs.difference(pathset)])
-
-    # warn on duplicate Pkgmgr entries with the same priority
-    pset = set()
-    for plist in pkg_list:
-        try:
-            xdata = lxml.etree.parse(plist)
-        except lxml.etree.XMLSyntaxError, e:
-            logging.warn("Failed to parse %s: %s", plist, e)
-        # get priority, type, group
-        priority = xdata.getroot().get('priority')
-        ptype = xdata.getroot().get('type')
-        for pkg in xdata.findall("//Package"):
-            if pkg.getparent().tag == 'Group':
-                grp = pkg.getparent().get('name')
-                if type(grp) is not str and grp.getparent().tag == 'Group':
-                    pgrp = grp.getparent().get('name')
-                else:
-                    pgrp = 'none'
-            else:
-                grp = 'none'
-                pgrp = 'none'
-            ptuple = (pkg.get('name'), priority, ptype, grp, pgrp)
-            # check if package is already listed with same priority,
-            # type, grp
-            if ptuple in pset:
-                logging.warn("Duplicate Package %s, priority:%s, type:%s",
-                             pkg.get('name'), priority, ptype)
-            else:
-                pset.add(ptuple)
-
-    filesets = {"%s/metadata.xsd": metadata_list,
-                "%s/clients.xsd": clients_list,
-                "%s/info.xsd": info_list,
-                "%s/bundle.xsd": bundle_list + genshibundle_list,
-                "%s/pkglist.xsd": pkg_list,
-                "%s/base.xsd": base_list,
-                "%s/rules.xsd": rules_list,
-                "%s/report-configuration.xsd": imageinfo_list,
-                "%s/services.xsd": services_list,
-                "%s/deps.xsd": deps_list,
-                "%s/decisions.xsd": dec_list,
-                "%s/packages.xsd": pkgcfg_list,
-                "%s/grouppatterns.xsd": gp_list}
-
-    failures = 0
-    for schemaname, filelist in list(filesets.items()):
-        if filelist:
-            # avoid loading schemas for empty file lists
-            try:
-                schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname %
-                                                               schemadir))
-            except:
-                logging.warn("Failed to process schema %s",
-                             schemaname % schemadir)
-                failures = 1
-                continue
-            for filename in filelist:
-                if not validate(filename, schemaname % schemadir,
-                                schema=schema, xinclude=not setup['stdin']):
-                    failures = 1
-
-    # check Properties files against their schemas
-    for filename in props_list:
-        logging.info("checking %s" % filename)
-        schemafile = "%s.xsd" % os.path.splitext(filename)[0]
-        if os.path.exists(schemafile):
-            if not validate(filename, schemafile, xinclude=not setup['stdin']):
-                failures = 1
-        elif setup['require-schema']:
-            logging.warn("No schema found for %s", filename)
-            failures = 1
-        
-    # print out missing bundle information
-    logging.info("")
-    if not setup['stdin']:
-        # if we've taken a list of files on stdin, there's an
-        # excellent chance that referenced bundles do not exist, so
-        # skip this check
-        for bundle in ref_bundles:
-            # check for both regular and genshi bundles
-            xmlbundle = "%s.xml" % bundle
-            genshibundle = "%s.genshi" % bundle
-            allbundles = bundle_list + genshibundle_list
-            if xmlbundle not in allbundles and genshibundle not in allbundles:
-                logging.info("*** Warning: Bundle %s referenced, but does not "
-                             "exist.", bundle)
-                    
-    # verify bundle name attribute matches filename
-    for bundle in (bundle_list + genshibundle_list):
-        fname = bundle.split('Bundler/')[1].split('.')[0]
-        xdata = lxml.etree.parse(bundle)
-        bname = xdata.getroot().get('name')
-        if fname != bname:
-            logging.warn("The following names are inconsistent:")
-            logging.warn("  Filename is %s", fname)
-            logging.warn("  Bundle name found in %s is %s", fname, bname)
-
-    raise SystemExit(failures)