diff options
Diffstat (limited to 'src/lib')
33 files changed, 537 insertions, 222 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 11a82fcc0..b24b46dbc 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -161,7 +161,7 @@ class Frame(object): def promptFilter(self, msg, entries): """Filter a supplied list based on user input.""" ret = [] - entries.sort(cmpent) + entries.sort(key=lambda e: e.tag + ":" + e.get('name')) for entry in entries[:]: if entry in self.unhandled: # don't prompt for entries that can't be installed @@ -427,10 +427,12 @@ class Frame(object): # prune out unspecified bundles when running with -b continue if bundle in mbundles: - self.logger.debug("Bundle %s was modified" % bundle) + self.logger.debug("Bundle %s was modified" % + bundle.get('name')) func = "BundleUpdated" else: - self.logger.debug("Bundle %s was not modified" % bundle) + self.logger.debug("Bundle %s was not modified" % + bundle.get('name')) func = "BundleNotUpdated" for tool in self.tools: try: @@ -477,8 +479,8 @@ class Frame(object): self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) self.logger.info('Total managed entries: %d' % len(list(self.states.values()))) self.logger.info('Unmanaged entries: %d' % len(self.extra)) @@ -490,8 +492,8 @@ class Frame(object): self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) if ((list(self.states.values()).count(False) == 0) and not self.extra): self.logger.info('All entries correct.') diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index 57be34369..eda3a7fce 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -24,6 +24,7 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) +has_py32 = version >= (3, 2) __all__ = ["ComponentProxy", "RetryMethod", @@ -171,11 +172,14 @@ class SSLHTTPConnection(httplib.HTTPConnection): """ if not has_py26: httplib.HTTPConnection.__init__(self, host, port, strict) - else: + elif not has_py32: httplib.HTTPConnection.__init__(self, host, port, strict, timeout) + else: + # the strict parameter is deprecated. + # HTTP 0.9-style "Simple Responses" are not supported anymore. + httplib.HTTPConnection.__init__(self, host, port, timeout=timeout) self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, self.__class__.__name__)) - self.key = key self.cert = cert self.ca = ca diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index ec7f462b3..c3dcf7796 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -65,16 +65,19 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): self.cmd.run("/sbin/chkconfig --add %s" % (entry.attrib['name'])) self.logger.info("Installing Service %s" % (entry.get('name'))) rv = True - if entry.get('status') == 'off': + if (entry.get('status') == 'off' or + self.setup["servicemode"] == "build"): rv &= self.cmd.run((rcmd + " --level 0123456") % (entry.get('name'), entry.get('status'))).success - if entry.get("current_status") == "on": + if entry.get("current_status") == "on" and \ + self.setup["servicemode"] != "disabled": rv &= self.stop_service(entry).success else: rv &= self.cmd.run(rcmd % (entry.get('name'), entry.get('status'))).success - if entry.get("current_status") == "off": + if entry.get("current_status") == "off" and \ + self.setup["servicemode"] != "disabled": rv &= self.start_service(entry).success return rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 8ba1944d8..8f6bc5f37 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -154,7 +154,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get("current_exists", "true") == "true": # verify supplemental groups actual = [g[0] for g in self.user_supplementary_groups(entry)] - expected = [e.text for e in entry.findall("MemberOf")] + expected = [e.get("group", e.text).strip() + for e in entry.findall("MemberOf")] if set(expected) != set(actual): entry.set('qtext', "\n".join([entry.get('qtext', '')] + @@ -254,7 +255,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) - extras = [e.text for e in entry.findall("MemberOf")] + extras = [e.get("group", e.text).strip() + for e in entry.findall("MemberOf")] if extras: cmd.extend(['-G', ",".join(extras)]) cmd.extend(['-d', entry.get('home')]) diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index 5e319a414..e52da081b 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -73,10 +73,10 @@ class Portage(Bcfg2.Client.Tools.PkgTool): self.logger.debug('Running equery check on %s' % entry.get('name')) - for line in self.cmd.run(["/usr/bin/equery", "-N", "check", - '=%s-%s' % - (entry.get('name'), - version)]).stdout.splitlines(): + for line in self.cmd.run( + ["/usr/bin/equery", "-N", "check", + '=%s-%s' % (entry.get('name'), + entry.get('version'))]).stdout.splitlines(): if '!!!' in line and line.split()[1] not in modlist: return False diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index f38615062..92572ef1d 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -492,7 +492,8 @@ class SELinuxSeportHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding and modifying entries """ (port, proto) = entry.get("name").split("/") - return (port, proto, '', entry.get("selinuxtype")) + return (port, proto, entry.get("mlsrange", ""), + entry.get("selinuxtype")) def _deleteargs(self, entry): return tuple(entry.get("name").split("/")) @@ -565,7 +566,7 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler): """ argument list for adding, modifying, and deleting entries """ return (entry.get("name"), entry.get("selinuxtype"), self.filetypeargs[entry.get("filetype", "all")], - '', '') + entry.get("mlsrange", ""), '') def primarykey(self, entry): return ":".join([entry.tag, entry.get("name"), @@ -600,7 +601,7 @@ class SELinuxSenodeHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ (addr, netmask) = entry.get("name").split("/") - return (addr, netmask, entry.get("proto"), "", + return (addr, netmask, entry.get("proto"), entry.get("mlsrange", ""), entry.get("selinuxtype")) @@ -612,7 +613,8 @@ class SELinuxSeloginHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), entry.get("selinuxuser"), "") + return (entry.get("name"), entry.get("selinuxuser"), + entry.get("mlsrange", "")) class SELinuxSeuserHandler(SELinuxEntryHandler): @@ -652,15 +654,16 @@ class SELinuxSeuserHandler(SELinuxEntryHandler): # prefix. see the comment in Install() above for more # details. rv = [entry.get("name"), - entry.get("roles", "").replace(" ", ",").split(",")] + entry.get("roles", "").replace(" ", ",").split(","), + '', entry.get("mlsrange", "")] if self.needs_prefix: - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) else: key = self._key(entry) if key in self.all_records: attrs = self._key2attrs(key) if attrs['prefix'] != entry.get("prefix"): - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) return tuple(rv) @@ -672,7 +675,8 @@ class SELinuxSeinterfaceHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), '', entry.get("selinuxtype")) + return (entry.get("name"), entry.get("mlsrange", ""), + entry.get("selinuxtype")) class SELinuxSepermissiveHandler(SELinuxEntryHandler): diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 027d91c71..20a172d3d 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,6 +13,8 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} + conflicts = ['Chkconfig'] + def get_svc_command(self, service, action): return "/bin/systemctl %s %s.service" % (action, service.get('name')) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index e40ef750b..3bc261f2f 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -19,7 +19,7 @@ def prompt(msg): while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: os.read(sys.stdin.fileno(), 4096) try: - ans = input(msg.encode(sys.stdout.encoding, 'replace')) + ans = input(msg) return ans in ['y', 'Y'] except EOFError: # python 2.4.3 on CentOS doesn't like ^C for some reason diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 4e9239e26..d034c0777 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -89,11 +89,28 @@ def u_str(string, encoding=None): else: return unicode(string) +try: + from functools import wraps +except ImportError: + def wraps(wrapped): # pylint: disable=W0613 + """ implementation of functools.wraps() for python 2.4 """ + return lambda f: f + + # base64 compat if sys.hexversion >= 0x03000000: from base64 import b64encode as _b64encode, b64decode as _b64decode - b64encode = lambda s: _b64encode(s.encode('UTF-8')).decode('UTF-8') - b64decode = lambda s: _b64decode(s.encode('UTF-8')).decode('UTF-8') + + @wraps(_b64encode) + def b64encode(val, **kwargs): # pylint: disable=C0111 + try: + return _b64encode(val, **kwargs) + except TypeError: + return _b64encode(val.encode('UTF-8'), **kwargs).decode('UTF-8') + + @wraps(_b64decode) + def b64decode(val, **kwargs): # pylint: disable=C0111 + return _b64decode(val.encode('UTF-8'), **kwargs).decode('UTF-8') else: from base64 import b64encode, b64decode @@ -242,14 +259,6 @@ except ImportError: from md5 import md5 -try: - from functools import wraps -except ImportError: - def wraps(wrapped): # pylint: disable=W0613 - """ implementation of functools.wraps() for python 2.4 """ - return lambda f: f - - def oct_mode(mode): """ Convert a decimal number describing a POSIX permissions mode to a string giving the octal mode. In Python 2, this is a synonym diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py index 5c51dad1e..73bdd0b3a 100644 --- a/src/lib/Bcfg2/Reporting/Transport/__init__.py +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -2,11 +2,11 @@ Public transport routines """ -import traceback - +import sys from Bcfg2.Reporting.Transport.base import TransportError, \ TransportImportError + def load_transport(transport_name, setup): """ Try to load the transport. Raise TransportImportError on failure @@ -18,13 +18,14 @@ def load_transport(transport_name, setup): try: mod = __import__(transport_name) except: - raise TransportImportError("Unavailable") + raise TransportImportError("Error importing transport %s: %s" % + (transport_name, sys.exc_info()[1])) try: - cls = getattr(mod, transport_name) - return cls(setup) + return getattr(mod, transport_name)(setup) except: - raise TransportImportError("Transport unavailable: %s" % - traceback.format_exc().splitlines()[-1]) + raise TransportImportError("Error instantiating transport %s: %s" % + (transport_name, sys.exc_info()[1])) + def load_transport_from_config(setup): """Load the transport in the config... eventually""" @@ -32,4 +33,3 @@ def load_transport_from_config(setup): return load_transport(setup['reporting_transport'], setup) except KeyError: raise TransportImportError('Transport missing in config') - diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index a6f66eb13..28c38e741 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -3,7 +3,7 @@ import sys from django.core.exceptions import ImproperlyConfigured try: - from django.db import models + from django.db import models, backend, connection except ImproperlyConfigured: e = sys.exc_info()[1] print("Reports: unable to import django models: %s" % e) @@ -49,6 +49,20 @@ def hash_entry(entry_dict): return hash(cPickle.dumps(dataset)) +_our_backend = None +def _quote(value): + """ + Quote a string to use as a table name or column + + Newer versions and various drivers require an argument + https://code.djangoproject.com/ticket/13630 + """ + global _our_backend + if not _our_backend: + _our_backend = backend.DatabaseOperations(connection) + return _our_backend.quote_name(value) + + class Client(models.Model): """Object representing every client we have seen stats for.""" creation = models.DateTimeField(auto_now_add=True) @@ -77,16 +91,20 @@ class InteractionManager(models.Manager): cursor = connection.cursor() cfilter = "expiration is null" - sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \ - 'as timer from Reporting_interaction' + sql = 'select ri.id, x.client_id from ' + \ + '(select client_id, MAX(timestamp) as timer from ' + \ + _quote('Reporting_interaction') if maxdate: if not isinstance(maxdate, datetime): raise ValueError('Expected a datetime object') sql = sql + " where timestamp <= '%s' " % maxdate cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) - sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \ - 'ri.client_id = x.client_id AND ri.timestamp = x.timer' - sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter + sql = sql + ' GROUP BY client_id) x, ' + \ + _quote('Reporting_interaction') + \ + ' ri where ri.client_id = x.client_id AND' + \ + ' ri.timestamp = x.timer and x.client_id in' + \ + ' (select id from %s where %s)' % \ + (_quote('Reporting_client'), cfilter) try: cursor.execute(sql) return [item[0] for item in cursor.fetchall()] diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index d317cc03b..06a419354 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -128,6 +128,7 @@ class MetadataCore(Mode): except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed: %s" % msg) + self.bcore.load_plugins() self.bcore.fam.handle_event_set() self.metadata = self.bcore.metadata diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 48455819d..b05ad9d41 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -118,11 +118,11 @@ class Core(BaseCore): self.logger.error("Server startup failed: %s" % err) self.context.close() return False - self.server.register_instance(self) return True def _block(self): """ Enter the blocking infinite loop. """ + self.server.register_instance(self) try: self.server.serve_forever() finally: diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 8bfb76461..7c3b2d9cc 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -1,8 +1,9 @@ -""" check files for various required comments """ +""" Check files for various required comments. """ import os import lxml.etree import Bcfg2.Server.Lint +from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator @@ -11,7 +12,10 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML class Comments(Bcfg2.Server.Lint.ServerPlugin): - """ check files for various required headers """ + """ The Comments lint plugin checks files for header comments that + give information about the files. For instance, you can require + SVN keywords in a comment, or require the name of the maintainer + of a Genshi template, and so on. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.config_cache = {} @@ -27,21 +31,43 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def Errors(cls): return {"unexpanded-keywords": "warning", "keywords-not-found": "warning", - "comments-not-found": "warning"} + "comments-not-found": "warning", + "broken-xinclude-chain": "warning"} def required_keywords(self, rtype): - """ given a file type, fetch the list of required VCS keywords - from the bcfg2-lint config """ + """ Given a file type, fetch the list of required VCS keywords + from the bcfg2-lint config. Valid file types are documented + in :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :returns: list - the required items + """ return self.required_items(rtype, "keyword") def required_comments(self, rtype): - """ given a file type, fetch the list of required comments - from the bcfg2-lint config """ + """ Given a file type, fetch the list of required comments + from the bcfg2-lint config. Valid file types are documented + in :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :returns: list - the required items + """ return self.required_items(rtype, "comment") def required_items(self, rtype, itype): - """ given a file type and item type (comment or keyword), - fetch the list of required items from the bcfg2-lint config """ + """ Given a file type and item type (``comment`` or + ``keyword``), fetch the list of required items from the + bcfg2-lint config. Valid file types are documented in + :manpage:`bcfg2-lint.conf(5)`. + + :param rtype: The file type + :type rtype: string + :param itype: The item type (``comment`` or ``keyword``) + :type itype: string + :returns: list - the required items + """ if itype not in self.config_cache: self.config_cache[itype] = {} @@ -62,7 +88,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): return self.config_cache[itype][rtype] def check_bundles(self): - """ check bundle files for required headers """ + """ Check bundle files for required comments. """ if 'Bundler' in self.core.plugins: for bundle in self.core.plugins['Bundler'].entries.values(): xdata = None @@ -78,15 +104,41 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_xml(bundle.name, xdata, rtype) def check_properties(self): - """ check properties files for required headers """ + """ Check Properties files for required comments. """ if 'Properties' in self.core.plugins: props = self.core.plugins['Properties'] - for propfile, pdata in props.store.entries.items(): + for propfile, pdata in props.entries.items(): if os.path.splitext(propfile)[1] == ".xml": self.check_xml(pdata.name, pdata.xdata, 'properties') + def has_all_xincludes(self, mfile): + """ Return True if :attr:`Bcfg2.Server.Lint.Plugin.files` + includes all XIncludes listed in the specified metadata type, + false otherwise. In other words, this returns True if + bcfg2-lint is dealing with complete metadata. + + :param mfile: The metadata file ("clients.xml" or + "groups.xml") to check for XIncludes + :type mfile: string + :returns: bool + """ + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./%sinclude' % XI_NAMESPACE): + if not self.has_all_xincludes(el.get('href')): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not " + "include %s" % path) + return False + + return True + def check_metadata(self): - """ check metadata files for required headers """ + """ Check Metadata files for required comments. """ if self.has_all_xincludes("groups.xml"): self.check_xml(os.path.join(self.metadata.data, "groups.xml"), self.metadata.groups_xml.data, @@ -97,7 +149,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): "metadata") def check_cfg(self): - """ check Cfg files and info.xml files for required headers """ + """ Check Cfg files and ``info.xml`` files for required + comments. """ if 'Cfg' in self.core.plugins: for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): @@ -117,29 +170,57 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_plaintext(entry.name, entry.data, rtype) def check_probes(self): - """ check probes for required headers """ + """ Check Probes for required comments """ if 'Probes' in self.core.plugins: for probe in self.core.plugins['Probes'].probes.entries.values(): self.check_plaintext(probe.name, probe.data, "probes") def check_xml(self, filename, xdata, rtype): - """ check generic XML files for required headers """ + """ Generic check to check an XML file for required comments. + + :param filename: The filename + :type filename: string + :param xdata: The file data + :type xdata: lxml.etree._Element + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ self.check_lines(filename, [str(el) for el in xdata.getiterator(lxml.etree.Comment)], rtype) def check_plaintext(self, filename, data, rtype): - """ check generic plaintext files for required headers """ + """ Generic check to check a plain text file for required + comments. + + :param filename: The filename + :type filename: string + :param data: The file data + :type data: string + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ self.check_lines(filename, data.splitlines(), rtype) def check_lines(self, filename, lines, rtype): - """ generic header check for a set of lines """ + """ Generic header check for a set of lines. + + :param filename: The filename + :type filename: string + :param lines: The data to check + :type lines: list of strings + :param rtype: The type of file. Available types are + documented in :manpage:`bcfg2-lint.conf(5)`. + :type rtype: string + """ if self.HandlesFile(filename): # found is trivalent: - # False == not found - # None == found but not expanded - # True == found and expanded + # False == keyword not found + # None == keyword found but not expanded + # True == keyword found and expanded found = dict((k, False) for k in self.required_keywords(rtype)) for line in lines: diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index caee238bc..a1d0b7fa1 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,4 +1,4 @@ -""" Check Genshi templates for syntax errors """ +""" Check Genshi templates for syntax errors. """ import sys import Bcfg2.Server.Lint @@ -8,10 +8,9 @@ from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator class Genshi(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ + """ Check Genshi templates for syntax errors. """ def Run(self): - """ run plugin """ if 'Cfg' in self.core.plugins: self.check_cfg() if 'Bundler' in self.core.plugins: @@ -22,7 +21,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): return {"genshi-syntax-error": "error"} def check_cfg(self): - """ Check genshi templates in Cfg for syntax errors """ + """ Check genshi templates in Cfg for syntax errors. """ for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): if (self.HandlesFile(entry.name) and @@ -37,7 +36,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): "Genshi syntax error: %s" % err) def check_bundler(self): - """ Check templates in Bundler for syntax errors """ + """ Check templates in Bundler for syntax errors. """ loader = TemplateLoader() for entry in self.core.plugins['Bundler'].entries.values(): diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index e41ed867e..730f32750 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -1,4 +1,4 @@ -""" ensure that all named groups are valid group names """ +""" Ensure that all named groups are valid group names. """ import os import re @@ -6,8 +6,15 @@ import Bcfg2.Server.Lint class GroupNames(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all named groups are valid group names """ + """ Ensure that all named groups are valid group names. """ + + #: A string regex that matches only valid group names. Currently, + #: a group name is considered valid if it contains only + #: non-whitespace characters. pattern = r'\S+$' + + #: A compiled regex for + #: :attr:`Bcfg2.Server.Lint.GroupNames.GroupNames.pattern` valid = re.compile(r'^' + pattern) def Run(self): @@ -26,7 +33,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): return {"invalid-group-name": "error"} def check_rules(self): - """ Check groups used in the Rules plugin for validity """ + """ Check groups used in the Rules plugin for validity. """ for rules in self.core.plugins['Rules'].entries.values(): if not self.HandlesFile(rules.name): continue @@ -35,7 +42,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): os.path.join(self.config['repo'], rules.name)) def check_bundles(self): - """ Check groups used in the Bundler plugin for validity """ + """ Check groups used in the Bundler plugin for validity. """ for bundle in self.core.plugins['Bundler'].entries.values(): if self.HandlesFile(bundle.name) and bundle.template is None: self.check_entries(bundle.xdata.xpath("//Group"), @@ -43,7 +50,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_metadata(self): """ Check groups used or declared in the Metadata plugin for - validity """ + validity. """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), os.path.join(self.config['repo'], self.metadata.groups_xml.name)) @@ -61,7 +68,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_cfg(self): """ Check groups used in group-specific files in the Cfg - plugin for validity """ + plugin for validity. """ for root, _, files in os.walk(self.core.plugins['Cfg'].data): for fname in files: basename = os.path.basename(root) @@ -74,7 +81,14 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): def check_entries(self, entries, fname): """ Check a generic list of XML entries for <Group> tags with - invalid name attributes """ + invalid name attributes. + + :param entries: A list of XML <Group> tags whose ``name`` + attributes will be validated. + :type entries: list of lxml.etree._Element + :param fname: The filename the entry list came from + :type fname: string + """ for grp in entries: if not self.valid.search(grp.get("name")): self.LintError("invalid-group-name", diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index f2349f861..184f657b7 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -1,4 +1,4 @@ -""" ensure that all config files have an info.xml file""" +""" Ensure that all config files have a valid info.xml file. """ import os import Bcfg2.Options @@ -7,7 +7,14 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML class InfoXML(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all config files have an info.xml file""" + """ Ensure that all config files have a valid info.xml file. This + plugin can check for: + + * Missing ``info.xml`` files; + * Use of deprecated ``info``/``:info`` files; + * Paranoid mode disabled in an ``info.xml`` file; + * Required attributes missing from ``info.xml`` + """ def Run(self): if 'Cfg' not in self.core.plugins: return @@ -29,11 +36,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): def Errors(cls): return {"no-infoxml": "warning", "paranoid-false": "warning", - "broken-xinclude-chain": "warning", "required-infoxml-attrs-missing": "error"} def check_infoxml(self, fname, xdata): - """ verify that info.xml contains everything it should """ + """ Verify that info.xml contains everything it should. """ for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 44d02c2ff..2419c3d43 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -57,7 +57,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): else: threshold = 0.75 rv = [] - elist = entries.items() + elist = list(entries.items()) while elist: result = self._find_similar(elist.pop(0), copy.copy(elist), threshold) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 7a2fd3fe9..f2464b585 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -1,5 +1,5 @@ -""" verify attributes for configuration entries that cannot be -verified with an XML schema alone""" +""" Verify attributes for configuration entries that cannot be +verified with an XML schema alone. """ import os import re @@ -14,7 +14,8 @@ except ImportError: HAS_GENSHI = False -# format verifying functions +# format verifying functions. TODO: These should be moved into XML +# schemas where possible. def is_filename(val): """ Return True if val is a string describing a valid full path """ @@ -52,8 +53,8 @@ def is_device_mode(val): class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): - """ verify attributes for configuration entries that cannot be - verified with an XML schema alone """ + """ Verify attributes for configuration entries that cannot be + verified with an XML schema alone. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = dict( @@ -114,8 +115,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): SEInterface={None: dict(name=None, selinuxtype=is_selinux_type)}, SEPermissive={None: dict(name=is_selinux_type)}, POSIXGroup={None: dict(name=is_username)}, - POSIXUser={None: dict(name=is_username)}, - MemberOf={None: dict(__text__=is_username)}) + POSIXUser={None: dict(name=is_username)}) def Run(self): self.check_packages() @@ -135,7 +135,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "extra-attrs": "warning"} def check_packages(self): - """ check package sources for Source entries with missing attrs """ + """ Check Packages sources for Source entries with missing + attributes. """ if 'Packages' not in self.core.plugins: return @@ -175,7 +176,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): rules.name)) def check_bundles(self): - """ check bundles for BoundPath entries with missing attrs """ + """ Check bundles for BoundPath entries with missing + attrs. """ if 'Bundler' not in self.core.plugins: return @@ -186,7 +188,13 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): self.check_entry(path, bundle.name) def check_entry(self, entry, filename): - """ generic entry check """ + """ Generic entry check. + + :param entry: The XML entry to check for missing attributes. + :type entry: lxml.etree._Element + :param filename: The filename the entry came from + :type filename: string + """ if self.HandlesFile(filename): name = entry.get('name') tag = entry.tag diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 946ef8270..ca9f138ef 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -1,4 +1,5 @@ -""" Ensure that the repo validates """ +""" Ensure that all XML files in the Bcfg2 repository validate +according to their respective schemas. """ import os import sys @@ -10,10 +11,19 @@ from Bcfg2.Utils import Executor class Validate(Bcfg2.Server.Lint.ServerlessPlugin): - """ Ensure that the repo validates """ + """ Ensure that all XML files in the Bcfg2 repository validate + according to their respective schemas. """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A dict of <file glob>: <schema file> that maps files in the + #: Bcfg2 specification to their schemas. The globs are + #: extended :mod:`fnmatch` globs that also support ``**``, + #: which matches any number of any characters, including + #: forward slashes. The schema files are relative to the + #: schema directory, which can be controlled by the + #: ``bcfg2-lint --schema`` option. self.filesets = \ {"Metadata/groups.xml": "metadata.xsd", "Metadata/clients.xml": "clients.xsd", @@ -76,7 +86,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "input-output-error": "error"} def check_properties(self): - """ check Properties files against their schemas """ + """ Check Properties files against their schemas. """ for filename in self.filelists['props']: schemafile = "%s.xsd" % os.path.splitext(filename)[0] if os.path.exists(schemafile): @@ -90,7 +100,11 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): def parse(self, filename): """ Parse an XML file, raising the appropriate LintErrors if it can't be parsed or read. Return the - lxml.etree._ElementTree parsed from the file. """ + lxml.etree._ElementTree parsed from the file. + + :param filename: The full path to the file to parse + :type filename: string + :returns: lxml.etree._ElementTree - the parsed data""" try: return lxml.etree.parse(filename) except SyntaxError: @@ -105,8 +119,20 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): return False def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ + """ Validate a file against the given schema. + + :param filename: The full path to the file to validate + :type filename: string + :param schemafile: The full path to the schema file to + validate against + :type schemafile: string + :param schema: The loaded schema to validate against. This + can be used to avoid parsing a single schema + file for every file that needs to be validate + against it. + :type schema: lxml.etree.Schema + :returns: bool - True if the file validates, false otherwise + """ if schema is None: # if no schema object was provided, instantiate one schema = self._load_schema(schemafile) @@ -127,7 +153,14 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): return True def get_filelists(self): - """ get lists of different kinds of files to validate """ + """ Get lists of different kinds of files to validate. This + doesn't return anything, but it sets + :attr:`Bcfg2.Server.Lint.Validate.Validate.filelists` to a + dict whose keys are path globs given in + :attr:`Bcfg2.Server.Lint.Validate.Validate.filesets` and whose + values are lists of the full paths to all files in the Bcfg2 + repository (or given with ``bcfg2-lint --stdin``) that match + the glob.""" if self.files is not None: listfiles = lambda p: fnmatch.filter(self.files, os.path.join('*', p)) @@ -154,7 +187,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filelists['props'] = listfiles("Properties/*.xml") def _load_schema(self, filename): - """ load an XML schema document, returning the Schema object """ + """ Load an XML schema document, returning the Schema object + and raising appropriate lint errors on failure. + + :param filename: The full path to the schema file to load. + :type filename: string + :returns: lxml.etree.Schema - The loaded schema data + """ try: return lxml.etree.XMLSchema(lxml.etree.parse(filename)) except IOError: diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 11afdd75d..28644263f 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -9,10 +9,9 @@ import lxml.etree import fcntl import termios import struct -from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Compat import walk_packages -__all__ = [m[1] for m in walk_packages(path=__path__)] +plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -45,30 +44,56 @@ def get_termsize(): class Plugin(object): - """ base class for ServerlessPlugin and ServerPlugin """ + """ Base class for all bcfg2-lint plugins """ def __init__(self, config, errorhandler=None, files=None): + """ + :param config: A :mod:`Bcfg2.Options` setup dict + :type config: dict + :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` + that will be used to handle lint errors. + If one is not provided, a new one will be + instantiated. + :type errorhandler: Bcfg2.Server.Lint.ErrorHandler + :param files: A list of files to run bcfg2-lint against. (See + the bcfg2-lint ``--stdin`` option.) + :type files: list of strings + """ + + #: The list of files that bcfg2-lint should be run against self.files = files + + #: The Bcfg2.Options setup dict self.config = config + self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: + #: The error handler self.errorhandler = ErrorHandler() else: self.errorhandler = errorhandler self.errorhandler.RegisterErrors(self.Errors()) def Run(self): - """ run the plugin. must be overloaded by child classes """ - pass + """ Run the plugin. Must be overloaded by child classes. """ + raise NotImplementedError @classmethod def Errors(cls): - """ returns a dict of errors the plugin supplies. must be - overloaded by child classes """ + """ Returns a dict of errors the plugin supplies, in a format + suitable for passing to + :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`. + + Must be overloaded by child classes. + + :returns: dict + """ + raise NotImplementedError def HandlesFile(self, fname): - """ returns true if the given file should be handled by the - plugin according to the files list, false otherwise """ + """ Returns True if the given file should be handled by the + plugin according to :attr:`Bcfg2.Server.Lint.Plugin.files`, + False otherwise. """ return (self.files is None or fname in self.files or os.path.join(self.config['repo'], fname) in self.files or @@ -77,12 +102,27 @@ class Plugin(object): fname)) in self.files) def LintError(self, err, msg): - """ record an error in the lint process """ + """ Raise an error from the lint process. + + :param err: The name of the error being raised. This name + must be a key in the dict returned by + :func:`Bcfg2.Server.Lint.Plugin.Errors`. + :type err: string + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.errorhandler.dispatch(err, msg) def RenderXML(self, element, keep_text=False): - """render an XML element for error output -- line number - prefixed, no children""" + """ Render an XML element for error output. This prefixes the + line number and removes children for nicer display. + + :param element: The element to render + :type element: lxml.etree._Element + :param keep_text: Do not discard text content from the element + for display + :type keep_text: boolean + """ xml = None if len(element) or element.text: el = copy(element) @@ -100,11 +140,18 @@ class Plugin(object): return " line %s: %s" % (element.sourceline, xml) -class ErrorHandler (object): - """ a class to handle errors for bcfg2-lint plugins """ +class ErrorHandler(object): + """ A class to handle errors for bcfg2-lint plugins """ - def __init__(self, config=None): + def __init__(self, errors=None): + """ + :param config: An initial dict of errors to register + :type config: dict + """ + #: The number of errors passed to this error handler self.errors = 0 + + #: The number of warnings passed to this error handler self.warnings = 0 self.logger = logging.getLogger('bcfg2-lint') @@ -114,17 +161,25 @@ class ErrorHandler (object): twrap = textwrap.TextWrapper(initial_indent=" ", subsequent_indent=" ", width=termsize[0]) + #: A function to wrap text to the width of the terminal self._wrapper = twrap.wrap else: self._wrapper = lambda s: [s] + #: A dict of registered errors self.errortypes = dict() - if config is not None: - self.RegisterErrors(dict(config.items())) + if errors is not None: + self.RegisterErrors(dict(errors.items())) def RegisterErrors(self, errors): - """ Register a dict of errors (name: default level) that a - plugin may raise """ + """ Register a dict of errors that a plugin may raise. The + keys of the dict are short strings that describe each error; + the values are the default error handling for that error + ("error", "warning", or "silent"). + + :param errors: The error dict + :type errors: dict + """ for err, action in errors.items(): if err not in self.errortypes: if "warn" in action: @@ -135,7 +190,16 @@ class ErrorHandler (object): self.errortypes[err] = self.debug def dispatch(self, err, msg): - """ Dispatch an error to the correct handler """ + """ Dispatch an error to the correct handler. + + :param err: The name of the error being raised. This name + must be a key in + :attr:`Bcfg2.Server.Lint.ErrorHandler.errortypes`, + the dict of registered errors. + :type err: string + :param msg: The freeform message to display to the end user. + :type msg: string + """ if err in self.errortypes: self.errortypes[err](msg) self.logger.debug(" (%s)" % err) @@ -145,22 +209,34 @@ class ErrorHandler (object): self.logger.warning("Unknown error %s" % err) def error(self, msg): - """ log an error condition """ + """ Log an error condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.errors += 1 self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): - """ log a warning condition """ + """ Log a warning condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self.warnings += 1 self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): - """ log a silent/debug condition """ + """ Log a silent/debug condition. + + :param msg: The freeform message to display to the end user. + :type msg: string + """ self._log(msg, self.logger.debug) def _log(self, msg, logfunc, prefix=""): """ Generic log function that logs a message with the given - function after wrapping it for the terminal width """ + function after wrapping it for the terminal width. """ # a message may itself consist of multiple lines. wrap() will # elide them all into a single paragraph, which we don't want. # so we split the message into its paragraphs and wrap each @@ -180,37 +256,37 @@ class ErrorHandler (object): logfunc(line) -class ServerlessPlugin (Plugin): - """ base class for plugins that are run before the server starts - up (i.e., plugins that check things that may prevent the server - from starting up) """ +class ServerlessPlugin(Plugin): # pylint: disable=W0223 + """ Base class for bcfg2-lint plugins that are run before the + server starts up (i.e., plugins that check things that may prevent + the server from starting up). """ pass -class ServerPlugin (Plugin): - """ base class for plugins that check things that require the - running Bcfg2 server """ - def __init__(self, core, config, **kwargs): - Plugin.__init__(self, config, **kwargs) +class ServerPlugin(Plugin): # pylint: disable=W0223 + """ Base class for bcfg2-lint plugins that check things that + require the running Bcfg2 server. """ + + def __init__(self, core, config, errorhandler=None, files=None): + """ + :param core: The Bcfg2 server core + :type core: Bcfg2.Server.Core.BaseCore + :param config: A :mod:`Bcfg2.Options` setup dict + :type config: dict + :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` + that will be used to handle lint errors. + If one is not provided, a new one will be + instantiated. + :type errorhandler: Bcfg2.Server.Lint.ErrorHandler + :param files: A list of files to run bcfg2-lint against. (See + the bcfg2-lint ``--stdin`` option.) + :type files: list of strings + """ + Plugin.__init__(self, config, errorhandler=errorhandler, files=files) + + #: The server core self.core = core self.logger = self.core.logger - self.metadata = self.core.metadata - self.errorhandler.RegisterErrors({"broken-xinclude-chain": "warning"}) - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./%sinclude' % XI_NAMESPACE): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not " - "include %s" % path) - return False - - return True + #: The metadata plugin + self.metadata = self.core.metadata diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index d8290d844..d114b0873 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -126,10 +126,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ + """ Perform various :ref:`Bundler + <server-plugins-structures-bundler-index>` checks. """ def Run(self): - """ run plugin """ self.missing_bundles() for bundle in self.core.plugins['Bundler'].entries.values(): if self.HandlesFile(bundle.name): @@ -143,7 +143,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): "genshi-extension-bundle": "error"} def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ + """ Find bundles listed in Metadata but not implemented in + Bundler. """ if self.files is None: # when given a list of files on stdin, this check is # useless, so skip it diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 9042a979e..3e5508160 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -125,7 +125,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for GroupPatterns """ + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + <server-plugins-grouping-grouppatterns>` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" def Run(self): cfg = self.core.plugins['GroupPatterns'].config diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 49e36f72b..2bc82caa9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) @@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) + self.set_profile(client, pgroup, (None, None), + require_public=False) profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client @@ -1477,7 +1479,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ bcfg2-lint plugin for Metadata """ + """ ``bcfg2-lint`` plugin for :ref:`Metadata + <server-plugins-grouping-metadata>`. This checks for several things: + + * ``<Client>`` tags nested inside other ``<Client>`` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ def Run(self): self.nested_clients() @@ -1500,8 +1511,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "default-is-not-profile": "error"} def deprecated_options(self): - """ check for the location='floating' option, which has been - deprecated in favor of floating='true' """ + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1519,8 +1530,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (loc, floating, self.RenderXML(el))) def nested_clients(self): - """ check for a Client tag inside a Client tag, which doesn't - make any sense """ + """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, + which is either redundant or will never match. """ groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", @@ -1528,8 +1539,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (el.get("name"), self.RenderXML(el))) def bogus_profiles(self): - """ check for clients that have profiles that are either not - flagged as public groups in groups.xml, or don't exist """ + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1547,20 +1558,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (profile, client.get("name"), profile, self.RenderXML(client))) - def duplicate_groups(self): - """ check for groups that are defined twice. We count a group - tag as a definition if it a) has profile or public set; or b) - has any children. """ - self.duplicate_entries( - self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"), - "group", - include=lambda g: (g.get("profile") or - g.get("public") or - g.getchildren())) - def duplicate_default_groups(self): - """ check for multiple default groups """ + """ Check for multiple default groups. """ defaults = [] for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): @@ -1572,7 +1571,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "\n".join(defaults)) def duplicate_clients(self): - """ check for clients that are defined twice. """ + """ Check for clients that are defined more than once. """ if not hasattr(self.metadata, "clients_xml"): # using metadata database return @@ -1580,17 +1579,34 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.metadata.clients_xml.xdata.xpath("//Client"), "client") - def duplicate_entries(self, allentries, etype, include=None): - """ generic duplicate entry finder """ - if include is None: - include = lambda e: True + def duplicate_groups(self): + """ Check for groups that are defined more than once. We + count a group tag as a definition if it a) has profile or + public set; or b) has any children.""" + allgroups = [ + g + for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") + if g.get("profile") or g.get("public") or g.getchildren()] + self.duplicate_entries(allgroups, "group") + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-<etype>``) and for + display to the end user. + :type etype: string + """ entries = dict() for el in allentries: - if include(el): - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] for ename, els in entries.items(): if len(els) > 1: self.LintError("duplicate-%s" % etype, @@ -1598,7 +1614,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (etype.title(), ename, "\n".join(els))) def default_is_profile(self): - """ ensure that the default group is a profile group """ + """ Ensure that the default group is a profile group. """ if (self.metadata.default and not self.metadata.groups[self.metadata.default].is_profile): xdata = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 48c580be1..dba56eed2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -39,6 +39,11 @@ class AptCollection(Collection): else: lines.append("deb %s %s %s" % (source.url, source.version, " ".join(source.components))) + if source.debsrc: + lines.append("deb-src %s %s %s" % + (source.url, + source.version, + " ".join(source.components))) lines.append("") return "\n".join(lines) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 3319dda78..767ac13ac 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -152,6 +152,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: this source self.whitelist = [item.text for item in xsource.findall('Whitelist')] + #: Whether or not to include deb-src lines in the generated APT + #: configuration + self.debsrc = xsource.get('debsrc', 'false') == 'true' + #: A dict of repository options that will be included in the #: configuration generated on the server side (if such is #: applicable; most backends do not generate any sort of diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 052c362ab..567a16c40 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,7 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -445,7 +445,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, try: open(localfile, 'w').write(urlopen(key).read()) keys.append(key) - except HTTPError: + except (URLError, HTTPError): err = sys.exc_info()[1] self.logger.error("Packages: Error downloading %s: %s" % (key, err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 7dac907e1..a1dcb575f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ Find duplicate :ref:`Pkgmgr + <server-plugins-generators-pkgmgr>` entries with the same + priority. """ + def Run(self): pset = set() for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', @@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): # check if package is already listed with same # priority, type, grp if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) else: pset.add(ptuple) - + @classmethod def Errors(cls): - return {"duplicate-packages":"error"} + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7e4935d74..e97607093 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -65,7 +65,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data.encode('utf-8')) + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -224,15 +224,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - try: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str( - self.probedata[client][probe]).decode('utf-8')) - except AttributeError: - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self.probedata[client][probe]) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 1264fd1cf..84dcf2780 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -173,8 +173,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) - except PluginExecutionError: + newips.update(self.get_ipcache_entry(name)[0]) + except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) names[cmeta.hostname].update(cmeta.addresses) @@ -290,7 +290,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([info[4][0] + for info in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index e834759c2..050ba3b3e 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -96,7 +96,18 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ find duplicate Pkgmgr entries with the same priority """ + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + <server-plugins-connectors-templatehelper>` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.reserved_keywords = dir(HelperModule("foo.py")) @@ -107,7 +118,11 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): self.check_helper(helper.name) def check_helper(self, helper): - """ check a helper module for export errors """ + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ module_name = MODULE_RE.search(helper).group(1) try: diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 13c756049..eea2183f7 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -346,7 +346,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, :param register: Presence should be reported to service-location :type register: bool :param allow_none: Allow None values in XML-RPC - :type allow_non: bool + :type allow_none: bool :param encoding: Encoding to use for XML-RPC """ diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 68282331e..69c3264f9 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,6 +2,7 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ +import shlex import fcntl import logging import threading @@ -217,6 +218,7 @@ class Executor(object): """ if isinstance(command, str): cmdstr = command + command = shlex.split(cmdstr) else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) |