summaryrefslogtreecommitdiffstats
path: root/src/lib
diff options
context:
space:
mode:
authorChris St. Pierre <chris.a.st.pierre@gmail.com>2013-05-17 15:07:45 -0400
committerChris St. Pierre <chris.a.st.pierre@gmail.com>2013-05-17 15:07:45 -0400
commitb7e2699f5156607258ed0d500c5d6d6b92c35f26 (patch)
tree24bde5be78c5d9920d21b3f79de06474ff3d282f /src/lib
parent044213c789c4f1ee214da3d70f02352b1aaa8673 (diff)
parent7e93fe741c17203fa63f60a7d1f66bfcdfb90d03 (diff)
downloadbcfg2-b7e2699f5156607258ed0d500c5d6d6b92c35f26.tar.gz
bcfg2-b7e2699f5156607258ed0d500c5d6d6b92c35f26.tar.bz2
bcfg2-b7e2699f5156607258ed0d500c5d6d6b92c35f26.zip
Merge branch 'maint'
Conflicts: doc/appendix/guides/centos.txt doc/server/plugins/grouping/metadata.txt setup.py src/lib/Bcfg2/Client/Frame.py src/lib/Bcfg2/Client/Proxy.py src/lib/Bcfg2/Server/Lint/Genshi.py src/lib/Bcfg2/Server/Lint/Validate.py src/lib/Bcfg2/Server/Plugins/Bundler.py src/lib/Bcfg2/Server/Plugins/SSHbase.py src/sbin/bcfg2-lint
Diffstat (limited to 'src/lib')
-rw-r--r--src/lib/Bcfg2/Client/Frame.py16
-rw-r--r--src/lib/Bcfg2/Client/Proxy.py8
-rw-r--r--src/lib/Bcfg2/Client/Tools/Chkconfig.py9
-rw-r--r--src/lib/Bcfg2/Client/Tools/POSIXUsers.py6
-rw-r--r--src/lib/Bcfg2/Client/Tools/Portage.py8
-rw-r--r--src/lib/Bcfg2/Client/Tools/SELinux.py20
-rw-r--r--src/lib/Bcfg2/Client/Tools/Systemd.py2
-rw-r--r--src/lib/Bcfg2/Client/__init__.py2
-rw-r--r--src/lib/Bcfg2/Compat.py29
-rw-r--r--src/lib/Bcfg2/Reporting/Transport/__init__.py16
-rw-r--r--src/lib/Bcfg2/Reporting/models.py30
-rw-r--r--src/lib/Bcfg2/Server/Admin/__init__.py1
-rw-r--r--src/lib/Bcfg2/Server/BuiltinCore.py2
-rw-r--r--src/lib/Bcfg2/Server/Lint/Comments.py123
-rwxr-xr-xsrc/lib/Bcfg2/Server/Lint/Genshi.py9
-rw-r--r--src/lib/Bcfg2/Server/Lint/GroupNames.py28
-rw-r--r--src/lib/Bcfg2/Server/Lint/InfoXML.py14
-rw-r--r--src/lib/Bcfg2/Server/Lint/MergeFiles.py2
-rw-r--r--src/lib/Bcfg2/Server/Lint/RequiredAttrs.py28
-rw-r--r--src/lib/Bcfg2/Server/Lint/Validate.py55
-rw-r--r--src/lib/Bcfg2/Server/Lint/__init__.py180
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Bundler.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/GroupPatterns.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py84
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py5
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Pkgmgr.py16
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Probes.py14
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSHbase.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/TemplateHelper.py19
-rw-r--r--src/lib/Bcfg2/Server/SSLServer.py2
-rw-r--r--src/lib/Bcfg2/Utils.py2
33 files changed, 537 insertions, 222 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py
index 11a82fcc0..b24b46dbc 100644
--- a/src/lib/Bcfg2/Client/Frame.py
+++ b/src/lib/Bcfg2/Client/Frame.py
@@ -161,7 +161,7 @@ class Frame(object):
def promptFilter(self, msg, entries):
"""Filter a supplied list based on user input."""
ret = []
- entries.sort(cmpent)
+ entries.sort(key=lambda e: e.tag + ":" + e.get('name'))
for entry in entries[:]:
if entry in self.unhandled:
# don't prompt for entries that can't be installed
@@ -427,10 +427,12 @@ class Frame(object):
# prune out unspecified bundles when running with -b
continue
if bundle in mbundles:
- self.logger.debug("Bundle %s was modified" % bundle)
+ self.logger.debug("Bundle %s was modified" %
+ bundle.get('name'))
func = "BundleUpdated"
else:
- self.logger.debug("Bundle %s was not modified" % bundle)
+ self.logger.debug("Bundle %s was not modified" %
+ bundle.get('name'))
func = "BundleNotUpdated"
for tool in self.tools:
try:
@@ -477,8 +479,8 @@ class Frame(object):
self.logger.info("%s:%s:%s" % (entry.tag, etype,
entry.get('name')))
else:
- self.logger.info(" %s:%s" % (entry.tag,
- entry.get('name')))
+ self.logger.info("%s:%s" % (entry.tag,
+ entry.get('name')))
self.logger.info('Total managed entries: %d' %
len(list(self.states.values())))
self.logger.info('Unmanaged entries: %d' % len(self.extra))
@@ -490,8 +492,8 @@ class Frame(object):
self.logger.info("%s:%s:%s" % (entry.tag, etype,
entry.get('name')))
else:
- self.logger.info(" %s:%s" % (entry.tag,
- entry.get('name')))
+ self.logger.info("%s:%s" % (entry.tag,
+ entry.get('name')))
if ((list(self.states.values()).count(False) == 0) and not self.extra):
self.logger.info('All entries correct.')
diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py
index 57be34369..eda3a7fce 100644
--- a/src/lib/Bcfg2/Client/Proxy.py
+++ b/src/lib/Bcfg2/Client/Proxy.py
@@ -24,6 +24,7 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus
version = sys.version_info[:2]
has_py26 = version >= (2, 6)
+has_py32 = version >= (3, 2)
__all__ = ["ComponentProxy",
"RetryMethod",
@@ -171,11 +172,14 @@ class SSLHTTPConnection(httplib.HTTPConnection):
"""
if not has_py26:
httplib.HTTPConnection.__init__(self, host, port, strict)
- else:
+ elif not has_py32:
httplib.HTTPConnection.__init__(self, host, port, strict, timeout)
+ else:
+ # the strict parameter is deprecated.
+ # HTTP 0.9-style "Simple Responses" are not supported anymore.
+ httplib.HTTPConnection.__init__(self, host, port, timeout=timeout)
self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__,
self.__class__.__name__))
-
self.key = key
self.cert = cert
self.ca = ca
diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py
index ec7f462b3..c3dcf7796 100644
--- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py
+++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py
@@ -65,16 +65,19 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool):
self.cmd.run("/sbin/chkconfig --add %s" % (entry.attrib['name']))
self.logger.info("Installing Service %s" % (entry.get('name')))
rv = True
- if entry.get('status') == 'off':
+ if (entry.get('status') == 'off' or
+ self.setup["servicemode"] == "build"):
rv &= self.cmd.run((rcmd + " --level 0123456") %
(entry.get('name'),
entry.get('status'))).success
- if entry.get("current_status") == "on":
+ if entry.get("current_status") == "on" and \
+ self.setup["servicemode"] != "disabled":
rv &= self.stop_service(entry).success
else:
rv &= self.cmd.run(rcmd % (entry.get('name'),
entry.get('status'))).success
- if entry.get("current_status") == "off":
+ if entry.get("current_status") == "off" and \
+ self.setup["servicemode"] != "disabled":
rv &= self.start_service(entry).success
return rv
diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
index 8ba1944d8..8f6bc5f37 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
@@ -154,7 +154,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
if entry.get("current_exists", "true") == "true":
# verify supplemental groups
actual = [g[0] for g in self.user_supplementary_groups(entry)]
- expected = [e.text for e in entry.findall("MemberOf")]
+ expected = [e.get("group", e.text).strip()
+ for e in entry.findall("MemberOf")]
if set(expected) != set(actual):
entry.set('qtext',
"\n".join([entry.get('qtext', '')] +
@@ -254,7 +255,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
if entry.get('uid'):
cmd.extend(['-u', entry.get('uid')])
cmd.extend(['-g', entry.get('group')])
- extras = [e.text for e in entry.findall("MemberOf")]
+ extras = [e.get("group", e.text).strip()
+ for e in entry.findall("MemberOf")]
if extras:
cmd.extend(['-G', ",".join(extras)])
cmd.extend(['-d', entry.get('home')])
diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py
index 5e319a414..e52da081b 100644
--- a/src/lib/Bcfg2/Client/Tools/Portage.py
+++ b/src/lib/Bcfg2/Client/Tools/Portage.py
@@ -73,10 +73,10 @@ class Portage(Bcfg2.Client.Tools.PkgTool):
self.logger.debug('Running equery check on %s' %
entry.get('name'))
- for line in self.cmd.run(["/usr/bin/equery", "-N", "check",
- '=%s-%s' %
- (entry.get('name'),
- version)]).stdout.splitlines():
+ for line in self.cmd.run(
+ ["/usr/bin/equery", "-N", "check",
+ '=%s-%s' % (entry.get('name'),
+ entry.get('version'))]).stdout.splitlines():
if '!!!' in line and line.split()[1] not in modlist:
return False
diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py
index f38615062..92572ef1d 100644
--- a/src/lib/Bcfg2/Client/Tools/SELinux.py
+++ b/src/lib/Bcfg2/Client/Tools/SELinux.py
@@ -492,7 +492,8 @@ class SELinuxSeportHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding and modifying entries """
(port, proto) = entry.get("name").split("/")
- return (port, proto, '', entry.get("selinuxtype"))
+ return (port, proto, entry.get("mlsrange", ""),
+ entry.get("selinuxtype"))
def _deleteargs(self, entry):
return tuple(entry.get("name").split("/"))
@@ -565,7 +566,7 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler):
""" argument list for adding, modifying, and deleting entries """
return (entry.get("name"), entry.get("selinuxtype"),
self.filetypeargs[entry.get("filetype", "all")],
- '', '')
+ entry.get("mlsrange", ""), '')
def primarykey(self, entry):
return ":".join([entry.tag, entry.get("name"),
@@ -600,7 +601,7 @@ class SELinuxSenodeHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding, modifying, and deleting entries """
(addr, netmask) = entry.get("name").split("/")
- return (addr, netmask, entry.get("proto"), "",
+ return (addr, netmask, entry.get("proto"), entry.get("mlsrange", ""),
entry.get("selinuxtype"))
@@ -612,7 +613,8 @@ class SELinuxSeloginHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding, modifying, and deleting entries """
- return (entry.get("name"), entry.get("selinuxuser"), "")
+ return (entry.get("name"), entry.get("selinuxuser"),
+ entry.get("mlsrange", ""))
class SELinuxSeuserHandler(SELinuxEntryHandler):
@@ -652,15 +654,16 @@ class SELinuxSeuserHandler(SELinuxEntryHandler):
# prefix. see the comment in Install() above for more
# details.
rv = [entry.get("name"),
- entry.get("roles", "").replace(" ", ",").split(",")]
+ entry.get("roles", "").replace(" ", ",").split(","),
+ '', entry.get("mlsrange", "")]
if self.needs_prefix:
- rv.extend(['', '', entry.get("prefix")])
+ rv.append(entry.get("prefix"))
else:
key = self._key(entry)
if key in self.all_records:
attrs = self._key2attrs(key)
if attrs['prefix'] != entry.get("prefix"):
- rv.extend(['', '', entry.get("prefix")])
+ rv.append(entry.get("prefix"))
return tuple(rv)
@@ -672,7 +675,8 @@ class SELinuxSeinterfaceHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding, modifying, and deleting entries """
- return (entry.get("name"), '', entry.get("selinuxtype"))
+ return (entry.get("name"), entry.get("mlsrange", ""),
+ entry.get("selinuxtype"))
class SELinuxSepermissiveHandler(SELinuxEntryHandler):
diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py
index 027d91c71..20a172d3d 100644
--- a/src/lib/Bcfg2/Client/Tools/Systemd.py
+++ b/src/lib/Bcfg2/Client/Tools/Systemd.py
@@ -13,6 +13,8 @@ class Systemd(Bcfg2.Client.Tools.SvcTool):
__handles__ = [('Service', 'systemd')]
__req__ = {'Service': ['name', 'status']}
+ conflicts = ['Chkconfig']
+
def get_svc_command(self, service, action):
return "/bin/systemctl %s %s.service" % (action, service.get('name'))
diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py
index e40ef750b..3bc261f2f 100644
--- a/src/lib/Bcfg2/Client/__init__.py
+++ b/src/lib/Bcfg2/Client/__init__.py
@@ -19,7 +19,7 @@ def prompt(msg):
while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0:
os.read(sys.stdin.fileno(), 4096)
try:
- ans = input(msg.encode(sys.stdout.encoding, 'replace'))
+ ans = input(msg)
return ans in ['y', 'Y']
except EOFError:
# python 2.4.3 on CentOS doesn't like ^C for some reason
diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py
index 4e9239e26..d034c0777 100644
--- a/src/lib/Bcfg2/Compat.py
+++ b/src/lib/Bcfg2/Compat.py
@@ -89,11 +89,28 @@ def u_str(string, encoding=None):
else:
return unicode(string)
+try:
+ from functools import wraps
+except ImportError:
+ def wraps(wrapped): # pylint: disable=W0613
+ """ implementation of functools.wraps() for python 2.4 """
+ return lambda f: f
+
+
# base64 compat
if sys.hexversion >= 0x03000000:
from base64 import b64encode as _b64encode, b64decode as _b64decode
- b64encode = lambda s: _b64encode(s.encode('UTF-8')).decode('UTF-8')
- b64decode = lambda s: _b64decode(s.encode('UTF-8')).decode('UTF-8')
+
+ @wraps(_b64encode)
+ def b64encode(val, **kwargs): # pylint: disable=C0111
+ try:
+ return _b64encode(val, **kwargs)
+ except TypeError:
+ return _b64encode(val.encode('UTF-8'), **kwargs).decode('UTF-8')
+
+ @wraps(_b64decode)
+ def b64decode(val, **kwargs): # pylint: disable=C0111
+ return _b64decode(val.encode('UTF-8'), **kwargs).decode('UTF-8')
else:
from base64 import b64encode, b64decode
@@ -242,14 +259,6 @@ except ImportError:
from md5 import md5
-try:
- from functools import wraps
-except ImportError:
- def wraps(wrapped): # pylint: disable=W0613
- """ implementation of functools.wraps() for python 2.4 """
- return lambda f: f
-
-
def oct_mode(mode):
""" Convert a decimal number describing a POSIX permissions mode
to a string giving the octal mode. In Python 2, this is a synonym
diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py
index 5c51dad1e..73bdd0b3a 100644
--- a/src/lib/Bcfg2/Reporting/Transport/__init__.py
+++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py
@@ -2,11 +2,11 @@
Public transport routines
"""
-import traceback
-
+import sys
from Bcfg2.Reporting.Transport.base import TransportError, \
TransportImportError
+
def load_transport(transport_name, setup):
"""
Try to load the transport. Raise TransportImportError on failure
@@ -18,13 +18,14 @@ def load_transport(transport_name, setup):
try:
mod = __import__(transport_name)
except:
- raise TransportImportError("Unavailable")
+ raise TransportImportError("Error importing transport %s: %s" %
+ (transport_name, sys.exc_info()[1]))
try:
- cls = getattr(mod, transport_name)
- return cls(setup)
+ return getattr(mod, transport_name)(setup)
except:
- raise TransportImportError("Transport unavailable: %s" %
- traceback.format_exc().splitlines()[-1])
+ raise TransportImportError("Error instantiating transport %s: %s" %
+ (transport_name, sys.exc_info()[1]))
+
def load_transport_from_config(setup):
"""Load the transport in the config... eventually"""
@@ -32,4 +33,3 @@ def load_transport_from_config(setup):
return load_transport(setup['reporting_transport'], setup)
except KeyError:
raise TransportImportError('Transport missing in config')
-
diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py
index e63c180a8..2e75c1d1a 100644
--- a/src/lib/Bcfg2/Reporting/models.py
+++ b/src/lib/Bcfg2/Reporting/models.py
@@ -3,7 +3,7 @@ import sys
from django.core.exceptions import ImproperlyConfigured
try:
- from django.db import models
+ from django.db import models, backend, connection
except ImproperlyConfigured:
e = sys.exc_info()[1]
print("Reports: unable to import django models: %s" % e)
@@ -49,6 +49,20 @@ def hash_entry(entry_dict):
return hash(cPickle.dumps(dataset))
+_our_backend = None
+def _quote(value):
+ """
+ Quote a string to use as a table name or column
+
+ Newer versions and various drivers require an argument
+ https://code.djangoproject.com/ticket/13630
+ """
+ global _our_backend
+ if not _our_backend:
+ _our_backend = backend.DatabaseOperations(connection)
+ return _our_backend.quote_name(value)
+
+
class Client(models.Model):
"""Object representing every client we have seen stats for."""
creation = models.DateTimeField(auto_now_add=True)
@@ -77,16 +91,20 @@ class InteractionManager(models.Manager):
cursor = connection.cursor()
cfilter = "expiration is null"
- sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \
- 'as timer from Reporting_interaction'
+ sql = 'select ri.id, x.client_id from ' + \
+ '(select client_id, MAX(timestamp) as timer from ' + \
+ _quote('Reporting_interaction')
if maxdate:
if not isinstance(maxdate, datetime):
raise ValueError('Expected a datetime object')
sql = sql + " where timestamp <= '%s' " % maxdate
cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate)
- sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \
- 'ri.client_id = x.client_id AND ri.timestamp = x.timer'
- sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter
+ sql = sql + ' GROUP BY client_id) x, ' + \
+ _quote('Reporting_interaction') + \
+ ' ri where ri.client_id = x.client_id AND' + \
+ ' ri.timestamp = x.timer and x.client_id in' + \
+ ' (select id from %s where %s)' % \
+ (_quote('Reporting_client'), cfilter)
try:
cursor.execute(sql)
return [item[0] for item in cursor.fetchall()]
diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py
index d317cc03b..06a419354 100644
--- a/src/lib/Bcfg2/Server/Admin/__init__.py
+++ b/src/lib/Bcfg2/Server/Admin/__init__.py
@@ -128,6 +128,7 @@ class MetadataCore(Mode):
except Bcfg2.Server.Core.CoreInitError:
msg = sys.exc_info()[1]
self.errExit("Core load failed: %s" % msg)
+ self.bcore.load_plugins()
self.bcore.fam.handle_event_set()
self.metadata = self.bcore.metadata
diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py
index 48455819d..b05ad9d41 100644
--- a/src/lib/Bcfg2/Server/BuiltinCore.py
+++ b/src/lib/Bcfg2/Server/BuiltinCore.py
@@ -118,11 +118,11 @@ class Core(BaseCore):
self.logger.error("Server startup failed: %s" % err)
self.context.close()
return False
- self.server.register_instance(self)
return True
def _block(self):
""" Enter the blocking infinite loop. """
+ self.server.register_instance(self)
try:
self.server.serve_forever()
finally:
diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py
index 8bfb76461..7c3b2d9cc 100644
--- a/src/lib/Bcfg2/Server/Lint/Comments.py
+++ b/src/lib/Bcfg2/Server/Lint/Comments.py
@@ -1,8 +1,9 @@
-""" check files for various required comments """
+""" Check files for various required comments. """
import os
import lxml.etree
import Bcfg2.Server.Lint
+from Bcfg2.Server import XI_NAMESPACE
from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \
import CfgPlaintextGenerator
from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
@@ -11,7 +12,10 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML
class Comments(Bcfg2.Server.Lint.ServerPlugin):
- """ check files for various required headers """
+ """ The Comments lint plugin checks files for header comments that
+ give information about the files. For instance, you can require
+ SVN keywords in a comment, or require the name of the maintainer
+ of a Genshi template, and so on. """
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
self.config_cache = {}
@@ -27,21 +31,43 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
def Errors(cls):
return {"unexpanded-keywords": "warning",
"keywords-not-found": "warning",
- "comments-not-found": "warning"}
+ "comments-not-found": "warning",
+ "broken-xinclude-chain": "warning"}
def required_keywords(self, rtype):
- """ given a file type, fetch the list of required VCS keywords
- from the bcfg2-lint config """
+ """ Given a file type, fetch the list of required VCS keywords
+ from the bcfg2-lint config. Valid file types are documented
+ in :manpage:`bcfg2-lint.conf(5)`.
+
+ :param rtype: The file type
+ :type rtype: string
+ :returns: list - the required items
+ """
return self.required_items(rtype, "keyword")
def required_comments(self, rtype):
- """ given a file type, fetch the list of required comments
- from the bcfg2-lint config """
+ """ Given a file type, fetch the list of required comments
+ from the bcfg2-lint config. Valid file types are documented
+ in :manpage:`bcfg2-lint.conf(5)`.
+
+ :param rtype: The file type
+ :type rtype: string
+ :returns: list - the required items
+ """
return self.required_items(rtype, "comment")
def required_items(self, rtype, itype):
- """ given a file type and item type (comment or keyword),
- fetch the list of required items from the bcfg2-lint config """
+ """ Given a file type and item type (``comment`` or
+ ``keyword``), fetch the list of required items from the
+ bcfg2-lint config. Valid file types are documented in
+ :manpage:`bcfg2-lint.conf(5)`.
+
+ :param rtype: The file type
+ :type rtype: string
+ :param itype: The item type (``comment`` or ``keyword``)
+ :type itype: string
+ :returns: list - the required items
+ """
if itype not in self.config_cache:
self.config_cache[itype] = {}
@@ -62,7 +88,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
return self.config_cache[itype][rtype]
def check_bundles(self):
- """ check bundle files for required headers """
+ """ Check bundle files for required comments. """
if 'Bundler' in self.core.plugins:
for bundle in self.core.plugins['Bundler'].entries.values():
xdata = None
@@ -78,15 +104,41 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
self.check_xml(bundle.name, xdata, rtype)
def check_properties(self):
- """ check properties files for required headers """
+ """ Check Properties files for required comments. """
if 'Properties' in self.core.plugins:
props = self.core.plugins['Properties']
- for propfile, pdata in props.store.entries.items():
+ for propfile, pdata in props.entries.items():
if os.path.splitext(propfile)[1] == ".xml":
self.check_xml(pdata.name, pdata.xdata, 'properties')
+ def has_all_xincludes(self, mfile):
+ """ Return True if :attr:`Bcfg2.Server.Lint.Plugin.files`
+ includes all XIncludes listed in the specified metadata type,
+ false otherwise. In other words, this returns True if
+ bcfg2-lint is dealing with complete metadata.
+
+ :param mfile: The metadata file ("clients.xml" or
+ "groups.xml") to check for XIncludes
+ :type mfile: string
+ :returns: bool
+ """
+ if self.files is None:
+ return True
+ else:
+ path = os.path.join(self.metadata.data, mfile)
+ if path in self.files:
+ xdata = lxml.etree.parse(path)
+ for el in xdata.findall('./%sinclude' % XI_NAMESPACE):
+ if not self.has_all_xincludes(el.get('href')):
+ self.LintError("broken-xinclude-chain",
+ "Broken XInclude chain: could not "
+ "include %s" % path)
+ return False
+
+ return True
+
def check_metadata(self):
- """ check metadata files for required headers """
+ """ Check Metadata files for required comments. """
if self.has_all_xincludes("groups.xml"):
self.check_xml(os.path.join(self.metadata.data, "groups.xml"),
self.metadata.groups_xml.data,
@@ -97,7 +149,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
"metadata")
def check_cfg(self):
- """ check Cfg files and info.xml files for required headers """
+ """ Check Cfg files and ``info.xml`` files for required
+ comments. """
if 'Cfg' in self.core.plugins:
for entryset in self.core.plugins['Cfg'].entries.values():
for entry in entryset.entries.values():
@@ -117,29 +170,57 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
self.check_plaintext(entry.name, entry.data, rtype)
def check_probes(self):
- """ check probes for required headers """
+ """ Check Probes for required comments """
if 'Probes' in self.core.plugins:
for probe in self.core.plugins['Probes'].probes.entries.values():
self.check_plaintext(probe.name, probe.data, "probes")
def check_xml(self, filename, xdata, rtype):
- """ check generic XML files for required headers """
+ """ Generic check to check an XML file for required comments.
+
+ :param filename: The filename
+ :type filename: string
+ :param xdata: The file data
+ :type xdata: lxml.etree._Element
+ :param rtype: The type of file. Available types are
+ documented in :manpage:`bcfg2-lint.conf(5)`.
+ :type rtype: string
+ """
self.check_lines(filename,
[str(el)
for el in xdata.getiterator(lxml.etree.Comment)],
rtype)
def check_plaintext(self, filename, data, rtype):
- """ check generic plaintext files for required headers """
+ """ Generic check to check a plain text file for required
+ comments.
+
+ :param filename: The filename
+ :type filename: string
+ :param data: The file data
+ :type data: string
+ :param rtype: The type of file. Available types are
+ documented in :manpage:`bcfg2-lint.conf(5)`.
+ :type rtype: string
+ """
self.check_lines(filename, data.splitlines(), rtype)
def check_lines(self, filename, lines, rtype):
- """ generic header check for a set of lines """
+ """ Generic header check for a set of lines.
+
+ :param filename: The filename
+ :type filename: string
+ :param lines: The data to check
+ :type lines: list of strings
+ :param rtype: The type of file. Available types are
+ documented in :manpage:`bcfg2-lint.conf(5)`.
+ :type rtype: string
+ """
if self.HandlesFile(filename):
# found is trivalent:
- # False == not found
- # None == found but not expanded
- # True == found and expanded
+ # False == keyword not found
+ # None == keyword found but not expanded
+ # True == keyword found and expanded
found = dict((k, False) for k in self.required_keywords(rtype))
for line in lines:
diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py
index caee238bc..a1d0b7fa1 100755
--- a/src/lib/Bcfg2/Server/Lint/Genshi.py
+++ b/src/lib/Bcfg2/Server/Lint/Genshi.py
@@ -1,4 +1,4 @@
-""" Check Genshi templates for syntax errors """
+""" Check Genshi templates for syntax errors. """
import sys
import Bcfg2.Server.Lint
@@ -8,10 +8,9 @@ from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
class Genshi(Bcfg2.Server.Lint.ServerPlugin):
- """ Check Genshi templates for syntax errors """
+ """ Check Genshi templates for syntax errors. """
def Run(self):
- """ run plugin """
if 'Cfg' in self.core.plugins:
self.check_cfg()
if 'Bundler' in self.core.plugins:
@@ -22,7 +21,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin):
return {"genshi-syntax-error": "error"}
def check_cfg(self):
- """ Check genshi templates in Cfg for syntax errors """
+ """ Check genshi templates in Cfg for syntax errors. """
for entryset in self.core.plugins['Cfg'].entries.values():
for entry in entryset.entries.values():
if (self.HandlesFile(entry.name) and
@@ -37,7 +36,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin):
"Genshi syntax error: %s" % err)
def check_bundler(self):
- """ Check templates in Bundler for syntax errors """
+ """ Check templates in Bundler for syntax errors. """
loader = TemplateLoader()
for entry in self.core.plugins['Bundler'].entries.values():
diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py
index e41ed867e..730f32750 100644
--- a/src/lib/Bcfg2/Server/Lint/GroupNames.py
+++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py
@@ -1,4 +1,4 @@
-""" ensure that all named groups are valid group names """
+""" Ensure that all named groups are valid group names. """
import os
import re
@@ -6,8 +6,15 @@ import Bcfg2.Server.Lint
class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
- """ ensure that all named groups are valid group names """
+ """ Ensure that all named groups are valid group names. """
+
+ #: A string regex that matches only valid group names. Currently,
+ #: a group name is considered valid if it contains only
+ #: non-whitespace characters.
pattern = r'\S+$'
+
+ #: A compiled regex for
+ #: :attr:`Bcfg2.Server.Lint.GroupNames.GroupNames.pattern`
valid = re.compile(r'^' + pattern)
def Run(self):
@@ -26,7 +33,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
return {"invalid-group-name": "error"}
def check_rules(self):
- """ Check groups used in the Rules plugin for validity """
+ """ Check groups used in the Rules plugin for validity. """
for rules in self.core.plugins['Rules'].entries.values():
if not self.HandlesFile(rules.name):
continue
@@ -35,7 +42,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
os.path.join(self.config['repo'], rules.name))
def check_bundles(self):
- """ Check groups used in the Bundler plugin for validity """
+ """ Check groups used in the Bundler plugin for validity. """
for bundle in self.core.plugins['Bundler'].entries.values():
if self.HandlesFile(bundle.name) and bundle.template is None:
self.check_entries(bundle.xdata.xpath("//Group"),
@@ -43,7 +50,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
def check_metadata(self):
""" Check groups used or declared in the Metadata plugin for
- validity """
+ validity. """
self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"),
os.path.join(self.config['repo'],
self.metadata.groups_xml.name))
@@ -61,7 +68,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
def check_cfg(self):
""" Check groups used in group-specific files in the Cfg
- plugin for validity """
+ plugin for validity. """
for root, _, files in os.walk(self.core.plugins['Cfg'].data):
for fname in files:
basename = os.path.basename(root)
@@ -74,7 +81,14 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
def check_entries(self, entries, fname):
""" Check a generic list of XML entries for <Group> tags with
- invalid name attributes """
+ invalid name attributes.
+
+ :param entries: A list of XML <Group> tags whose ``name``
+ attributes will be validated.
+ :type entries: list of lxml.etree._Element
+ :param fname: The filename the entry list came from
+ :type fname: string
+ """
for grp in entries:
if not self.valid.search(grp.get("name")):
self.LintError("invalid-group-name",
diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py
index f2349f861..184f657b7 100644
--- a/src/lib/Bcfg2/Server/Lint/InfoXML.py
+++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py
@@ -1,4 +1,4 @@
-""" ensure that all config files have an info.xml file"""
+""" Ensure that all config files have a valid info.xml file. """
import os
import Bcfg2.Options
@@ -7,7 +7,14 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML
class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
- """ ensure that all config files have an info.xml file"""
+ """ Ensure that all config files have a valid info.xml file. This
+ plugin can check for:
+
+ * Missing ``info.xml`` files;
+ * Use of deprecated ``info``/``:info`` files;
+ * Paranoid mode disabled in an ``info.xml`` file;
+ * Required attributes missing from ``info.xml``
+ """
def Run(self):
if 'Cfg' not in self.core.plugins:
return
@@ -29,11 +36,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
def Errors(cls):
return {"no-infoxml": "warning",
"paranoid-false": "warning",
- "broken-xinclude-chain": "warning",
"required-infoxml-attrs-missing": "error"}
def check_infoxml(self, fname, xdata):
- """ verify that info.xml contains everything it should """
+ """ Verify that info.xml contains everything it should. """
for info in xdata.getroottree().findall("//Info"):
required = []
if "required_attrs" in self.config:
diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py
index 44d02c2ff..2419c3d43 100644
--- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py
+++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py
@@ -57,7 +57,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin):
else:
threshold = 0.75
rv = []
- elist = entries.items()
+ elist = list(entries.items())
while elist:
result = self._find_similar(elist.pop(0), copy.copy(elist),
threshold)
diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
index 7a2fd3fe9..f2464b585 100644
--- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
+++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
@@ -1,5 +1,5 @@
-""" verify attributes for configuration entries that cannot be
-verified with an XML schema alone"""
+""" Verify attributes for configuration entries that cannot be
+verified with an XML schema alone. """
import os
import re
@@ -14,7 +14,8 @@ except ImportError:
HAS_GENSHI = False
-# format verifying functions
+# format verifying functions. TODO: These should be moved into XML
+# schemas where possible.
def is_filename(val):
""" Return True if val is a string describing a valid full path
"""
@@ -52,8 +53,8 @@ def is_device_mode(val):
class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
- """ verify attributes for configuration entries that cannot be
- verified with an XML schema alone """
+ """ Verify attributes for configuration entries that cannot be
+ verified with an XML schema alone. """
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
self.required_attrs = dict(
@@ -114,8 +115,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
SEInterface={None: dict(name=None, selinuxtype=is_selinux_type)},
SEPermissive={None: dict(name=is_selinux_type)},
POSIXGroup={None: dict(name=is_username)},
- POSIXUser={None: dict(name=is_username)},
- MemberOf={None: dict(__text__=is_username)})
+ POSIXUser={None: dict(name=is_username)})
def Run(self):
self.check_packages()
@@ -135,7 +135,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
"extra-attrs": "warning"}
def check_packages(self):
- """ check package sources for Source entries with missing attrs """
+ """ Check Packages sources for Source entries with missing
+ attributes. """
if 'Packages' not in self.core.plugins:
return
@@ -175,7 +176,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
rules.name))
def check_bundles(self):
- """ check bundles for BoundPath entries with missing attrs """
+ """ Check bundles for BoundPath entries with missing
+ attrs. """
if 'Bundler' not in self.core.plugins:
return
@@ -186,7 +188,13 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
self.check_entry(path, bundle.name)
def check_entry(self, entry, filename):
- """ generic entry check """
+ """ Generic entry check.
+
+ :param entry: The XML entry to check for missing attributes.
+ :type entry: lxml.etree._Element
+ :param filename: The filename the entry came from
+ :type filename: string
+ """
if self.HandlesFile(filename):
name = entry.get('name')
tag = entry.tag
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 946ef8270..ca9f138ef 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -1,4 +1,5 @@
-""" Ensure that the repo validates """
+""" Ensure that all XML files in the Bcfg2 repository validate
+according to their respective schemas. """
import os
import sys
@@ -10,10 +11,19 @@ from Bcfg2.Utils import Executor
class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
- """ Ensure that the repo validates """
+ """ Ensure that all XML files in the Bcfg2 repository validate
+ according to their respective schemas. """
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs)
+
+ #: A dict of <file glob>: <schema file> that maps files in the
+ #: Bcfg2 specification to their schemas. The globs are
+ #: extended :mod:`fnmatch` globs that also support ``**``,
+ #: which matches any number of any characters, including
+ #: forward slashes. The schema files are relative to the
+ #: schema directory, which can be controlled by the
+ #: ``bcfg2-lint --schema`` option.
self.filesets = \
{"Metadata/groups.xml": "metadata.xsd",
"Metadata/clients.xml": "clients.xsd",
@@ -76,7 +86,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"input-output-error": "error"}
def check_properties(self):
- """ check Properties files against their schemas """
+ """ Check Properties files against their schemas. """
for filename in self.filelists['props']:
schemafile = "%s.xsd" % os.path.splitext(filename)[0]
if os.path.exists(schemafile):
@@ -90,7 +100,11 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
def parse(self, filename):
""" Parse an XML file, raising the appropriate LintErrors if
it can't be parsed or read. Return the
- lxml.etree._ElementTree parsed from the file. """
+ lxml.etree._ElementTree parsed from the file.
+
+ :param filename: The full path to the file to parse
+ :type filename: string
+ :returns: lxml.etree._ElementTree - the parsed data"""
try:
return lxml.etree.parse(filename)
except SyntaxError:
@@ -105,8 +119,20 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
return False
def validate(self, filename, schemafile, schema=None):
- """validate a file against the given lxml.etree.Schema.
- return True on success, False on failure """
+ """ Validate a file against the given schema.
+
+ :param filename: The full path to the file to validate
+ :type filename: string
+ :param schemafile: The full path to the schema file to
+ validate against
+ :type schemafile: string
+ :param schema: The loaded schema to validate against. This
+ can be used to avoid parsing a single schema
+ file for every file that needs to be validate
+ against it.
+ :type schema: lxml.etree.Schema
+ :returns: bool - True if the file validates, false otherwise
+ """
if schema is None:
# if no schema object was provided, instantiate one
schema = self._load_schema(schemafile)
@@ -127,7 +153,14 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
return True
def get_filelists(self):
- """ get lists of different kinds of files to validate """
+ """ Get lists of different kinds of files to validate. This
+ doesn't return anything, but it sets
+ :attr:`Bcfg2.Server.Lint.Validate.Validate.filelists` to a
+ dict whose keys are path globs given in
+ :attr:`Bcfg2.Server.Lint.Validate.Validate.filesets` and whose
+ values are lists of the full paths to all files in the Bcfg2
+ repository (or given with ``bcfg2-lint --stdin``) that match
+ the glob."""
if self.files is not None:
listfiles = lambda p: fnmatch.filter(self.files,
os.path.join('*', p))
@@ -154,7 +187,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
self.filelists['props'] = listfiles("Properties/*.xml")
def _load_schema(self, filename):
- """ load an XML schema document, returning the Schema object """
+ """ Load an XML schema document, returning the Schema object
+ and raising appropriate lint errors on failure.
+
+ :param filename: The full path to the schema file to load.
+ :type filename: string
+ :returns: lxml.etree.Schema - The loaded schema data
+ """
try:
return lxml.etree.XMLSchema(lxml.etree.parse(filename))
except IOError:
diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py
index 11afdd75d..28644263f 100644
--- a/src/lib/Bcfg2/Server/Lint/__init__.py
+++ b/src/lib/Bcfg2/Server/Lint/__init__.py
@@ -9,10 +9,9 @@ import lxml.etree
import fcntl
import termios
import struct
-from Bcfg2.Server import XI_NAMESPACE
from Bcfg2.Compat import walk_packages
-__all__ = [m[1] for m in walk_packages(path=__path__)]
+plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103
def _ioctl_GWINSZ(fd): # pylint: disable=C0103
@@ -45,30 +44,56 @@ def get_termsize():
class Plugin(object):
- """ base class for ServerlessPlugin and ServerPlugin """
+ """ Base class for all bcfg2-lint plugins """
def __init__(self, config, errorhandler=None, files=None):
+ """
+ :param config: A :mod:`Bcfg2.Options` setup dict
+ :type config: dict
+ :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler`
+ that will be used to handle lint errors.
+ If one is not provided, a new one will be
+ instantiated.
+ :type errorhandler: Bcfg2.Server.Lint.ErrorHandler
+ :param files: A list of files to run bcfg2-lint against. (See
+ the bcfg2-lint ``--stdin`` option.)
+ :type files: list of strings
+ """
+
+ #: The list of files that bcfg2-lint should be run against
self.files = files
+
+ #: The Bcfg2.Options setup dict
self.config = config
+
self.logger = logging.getLogger('bcfg2-lint')
if errorhandler is None:
+ #: The error handler
self.errorhandler = ErrorHandler()
else:
self.errorhandler = errorhandler
self.errorhandler.RegisterErrors(self.Errors())
def Run(self):
- """ run the plugin. must be overloaded by child classes """
- pass
+ """ Run the plugin. Must be overloaded by child classes. """
+ raise NotImplementedError
@classmethod
def Errors(cls):
- """ returns a dict of errors the plugin supplies. must be
- overloaded by child classes """
+ """ Returns a dict of errors the plugin supplies, in a format
+ suitable for passing to
+ :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`.
+
+ Must be overloaded by child classes.
+
+ :returns: dict
+ """
+ raise NotImplementedError
def HandlesFile(self, fname):
- """ returns true if the given file should be handled by the
- plugin according to the files list, false otherwise """
+ """ Returns True if the given file should be handled by the
+ plugin according to :attr:`Bcfg2.Server.Lint.Plugin.files`,
+ False otherwise. """
return (self.files is None or
fname in self.files or
os.path.join(self.config['repo'], fname) in self.files or
@@ -77,12 +102,27 @@ class Plugin(object):
fname)) in self.files)
def LintError(self, err, msg):
- """ record an error in the lint process """
+ """ Raise an error from the lint process.
+
+ :param err: The name of the error being raised. This name
+ must be a key in the dict returned by
+ :func:`Bcfg2.Server.Lint.Plugin.Errors`.
+ :type err: string
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self.errorhandler.dispatch(err, msg)
def RenderXML(self, element, keep_text=False):
- """render an XML element for error output -- line number
- prefixed, no children"""
+ """ Render an XML element for error output. This prefixes the
+ line number and removes children for nicer display.
+
+ :param element: The element to render
+ :type element: lxml.etree._Element
+ :param keep_text: Do not discard text content from the element
+ for display
+ :type keep_text: boolean
+ """
xml = None
if len(element) or element.text:
el = copy(element)
@@ -100,11 +140,18 @@ class Plugin(object):
return " line %s: %s" % (element.sourceline, xml)
-class ErrorHandler (object):
- """ a class to handle errors for bcfg2-lint plugins """
+class ErrorHandler(object):
+ """ A class to handle errors for bcfg2-lint plugins """
- def __init__(self, config=None):
+ def __init__(self, errors=None):
+ """
+ :param config: An initial dict of errors to register
+ :type config: dict
+ """
+ #: The number of errors passed to this error handler
self.errors = 0
+
+ #: The number of warnings passed to this error handler
self.warnings = 0
self.logger = logging.getLogger('bcfg2-lint')
@@ -114,17 +161,25 @@ class ErrorHandler (object):
twrap = textwrap.TextWrapper(initial_indent=" ",
subsequent_indent=" ",
width=termsize[0])
+ #: A function to wrap text to the width of the terminal
self._wrapper = twrap.wrap
else:
self._wrapper = lambda s: [s]
+ #: A dict of registered errors
self.errortypes = dict()
- if config is not None:
- self.RegisterErrors(dict(config.items()))
+ if errors is not None:
+ self.RegisterErrors(dict(errors.items()))
def RegisterErrors(self, errors):
- """ Register a dict of errors (name: default level) that a
- plugin may raise """
+ """ Register a dict of errors that a plugin may raise. The
+ keys of the dict are short strings that describe each error;
+ the values are the default error handling for that error
+ ("error", "warning", or "silent").
+
+ :param errors: The error dict
+ :type errors: dict
+ """
for err, action in errors.items():
if err not in self.errortypes:
if "warn" in action:
@@ -135,7 +190,16 @@ class ErrorHandler (object):
self.errortypes[err] = self.debug
def dispatch(self, err, msg):
- """ Dispatch an error to the correct handler """
+ """ Dispatch an error to the correct handler.
+
+ :param err: The name of the error being raised. This name
+ must be a key in
+ :attr:`Bcfg2.Server.Lint.ErrorHandler.errortypes`,
+ the dict of registered errors.
+ :type err: string
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
if err in self.errortypes:
self.errortypes[err](msg)
self.logger.debug(" (%s)" % err)
@@ -145,22 +209,34 @@ class ErrorHandler (object):
self.logger.warning("Unknown error %s" % err)
def error(self, msg):
- """ log an error condition """
+ """ Log an error condition.
+
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self.errors += 1
self._log(msg, self.logger.error, prefix="ERROR: ")
def warn(self, msg):
- """ log a warning condition """
+ """ Log a warning condition.
+
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self.warnings += 1
self._log(msg, self.logger.warning, prefix="WARNING: ")
def debug(self, msg):
- """ log a silent/debug condition """
+ """ Log a silent/debug condition.
+
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self._log(msg, self.logger.debug)
def _log(self, msg, logfunc, prefix=""):
""" Generic log function that logs a message with the given
- function after wrapping it for the terminal width """
+ function after wrapping it for the terminal width. """
# a message may itself consist of multiple lines. wrap() will
# elide them all into a single paragraph, which we don't want.
# so we split the message into its paragraphs and wrap each
@@ -180,37 +256,37 @@ class ErrorHandler (object):
logfunc(line)
-class ServerlessPlugin (Plugin):
- """ base class for plugins that are run before the server starts
- up (i.e., plugins that check things that may prevent the server
- from starting up) """
+class ServerlessPlugin(Plugin): # pylint: disable=W0223
+ """ Base class for bcfg2-lint plugins that are run before the
+ server starts up (i.e., plugins that check things that may prevent
+ the server from starting up). """
pass
-class ServerPlugin (Plugin):
- """ base class for plugins that check things that require the
- running Bcfg2 server """
- def __init__(self, core, config, **kwargs):
- Plugin.__init__(self, config, **kwargs)
+class ServerPlugin(Plugin): # pylint: disable=W0223
+ """ Base class for bcfg2-lint plugins that check things that
+ require the running Bcfg2 server. """
+
+ def __init__(self, core, config, errorhandler=None, files=None):
+ """
+ :param core: The Bcfg2 server core
+ :type core: Bcfg2.Server.Core.BaseCore
+ :param config: A :mod:`Bcfg2.Options` setup dict
+ :type config: dict
+ :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler`
+ that will be used to handle lint errors.
+ If one is not provided, a new one will be
+ instantiated.
+ :type errorhandler: Bcfg2.Server.Lint.ErrorHandler
+ :param files: A list of files to run bcfg2-lint against. (See
+ the bcfg2-lint ``--stdin`` option.)
+ :type files: list of strings
+ """
+ Plugin.__init__(self, config, errorhandler=errorhandler, files=files)
+
+ #: The server core
self.core = core
self.logger = self.core.logger
- self.metadata = self.core.metadata
- self.errorhandler.RegisterErrors({"broken-xinclude-chain": "warning"})
- def has_all_xincludes(self, mfile):
- """ return true if self.files includes all XIncludes listed in
- the specified metadata type, false otherwise"""
- if self.files is None:
- return True
- else:
- path = os.path.join(self.metadata.data, mfile)
- if path in self.files:
- xdata = lxml.etree.parse(path)
- for el in xdata.findall('./%sinclude' % XI_NAMESPACE):
- if not self.has_all_xincludes(el.get('href')):
- self.LintError("broken-xinclude-chain",
- "Broken XInclude chain: could not "
- "include %s" % path)
- return False
-
- return True
+ #: The metadata plugin
+ self.metadata = self.core.metadata
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index d8290d844..d114b0873 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -126,10 +126,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
- """ Perform various bundle checks """
+ """ Perform various :ref:`Bundler
+ <server-plugins-structures-bundler-index>` checks. """
def Run(self):
- """ run plugin """
self.missing_bundles()
for bundle in self.core.plugins['Bundler'].entries.values():
if self.HandlesFile(bundle.name):
@@ -143,7 +143,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
"genshi-extension-bundle": "error"}
def missing_bundles(self):
- """ find bundles listed in Metadata but not implemented in Bundler """
+ """ Find bundles listed in Metadata but not implemented in
+ Bundler. """
if self.files is None:
# when given a list of files on stdin, this check is
# useless, so skip it
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
index 9042a979e..3e5508160 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -125,7 +125,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin,
class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin):
- """ bcfg2-lint plugin for GroupPatterns """
+ """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns
+ <server-plugins-grouping-grouppatterns>` patterns for validity.
+ This is simply done by trying to create a
+ :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for
+ each pattern, and catching exceptions and presenting them as
+ ``bcfg2-lint`` errors."""
def Run(self):
cfg = self.core.plugins['GroupPatterns'].config
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 49e36f72b..2bc82caa9 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.debug_log("Client %s set as nonexistent group %s"
% (client, group))
- def set_profile(self, client, profile, addresspair):
+ def set_profile(self, client, profile, # pylint: disable=W0221
+ addresspair, require_public=True):
"""Set group parameter for provided client."""
self.logger.info("Asserting client %s profile to %s" % (client,
profile))
@@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.logger.error(msg)
raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
group = self.groups[profile]
- if not group.is_public:
+ if require_public and not group.is_public:
msg = "Cannot set client %s to private group %s" % (client,
profile)
self.logger.error(msg)
@@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
pgroup = self.default
if pgroup:
- self.set_profile(client, pgroup, (None, None))
+ self.set_profile(client, pgroup, (None, None),
+ require_public=False)
profile = _add_group(pgroup)
else:
msg = "Cannot add new client %s; no default group set" % client
@@ -1477,7 +1479,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
- """ bcfg2-lint plugin for Metadata """
+ """ ``bcfg2-lint`` plugin for :ref:`Metadata
+ <server-plugins-grouping-metadata>`. This checks for several things:
+
+ * ``<Client>`` tags nested inside other ``<Client>`` tags;
+ * Deprecated options (like ``location="floating"``);
+ * Profiles that don't exist, or that aren't profile groups;
+ * Groups or clients that are defined multiple times;
+ * Multiple default groups or a default group that isn't a profile
+ group.
+ """
def Run(self):
self.nested_clients()
@@ -1500,8 +1511,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
"default-is-not-profile": "error"}
def deprecated_options(self):
- """ check for the location='floating' option, which has been
- deprecated in favor of floating='true' """
+ """ Check for the ``location='floating'`` option, which has
+ been deprecated in favor of ``floating='true'``. """
if not hasattr(self.metadata, "clients_xml"):
# using metadata database
return
@@ -1519,8 +1530,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(loc, floating, self.RenderXML(el)))
def nested_clients(self):
- """ check for a Client tag inside a Client tag, which doesn't
- make any sense """
+ """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag,
+ which is either redundant or will never match. """
groupdata = self.metadata.groups_xml.xdata
for el in groupdata.xpath("//Client//Client"):
self.LintError("nested-client-tags",
@@ -1528,8 +1539,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(el.get("name"), self.RenderXML(el)))
def bogus_profiles(self):
- """ check for clients that have profiles that are either not
- flagged as public groups in groups.xml, or don't exist """
+ """ Check for clients that have profiles that are either not
+ flagged as profile groups in ``groups.xml``, or don't exist. """
if not hasattr(self.metadata, "clients_xml"):
# using metadata database
return
@@ -1547,20 +1558,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(profile, client.get("name"), profile,
self.RenderXML(client)))
- def duplicate_groups(self):
- """ check for groups that are defined twice. We count a group
- tag as a definition if it a) has profile or public set; or b)
- has any children. """
- self.duplicate_entries(
- self.metadata.groups_xml.xdata.xpath("//Groups/Group") +
- self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"),
- "group",
- include=lambda g: (g.get("profile") or
- g.get("public") or
- g.getchildren()))
-
def duplicate_default_groups(self):
- """ check for multiple default groups """
+ """ Check for multiple default groups. """
defaults = []
for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \
self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"):
@@ -1572,7 +1571,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
"\n".join(defaults))
def duplicate_clients(self):
- """ check for clients that are defined twice. """
+ """ Check for clients that are defined more than once. """
if not hasattr(self.metadata, "clients_xml"):
# using metadata database
return
@@ -1580,17 +1579,34 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
self.metadata.clients_xml.xdata.xpath("//Client"),
"client")
- def duplicate_entries(self, allentries, etype, include=None):
- """ generic duplicate entry finder """
- if include is None:
- include = lambda e: True
+ def duplicate_groups(self):
+ """ Check for groups that are defined more than once. We
+ count a group tag as a definition if it a) has profile or
+ public set; or b) has any children."""
+ allgroups = [
+ g
+ for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") +
+ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group")
+ if g.get("profile") or g.get("public") or g.getchildren()]
+ self.duplicate_entries(allgroups, "group")
+
+ def duplicate_entries(self, allentries, etype):
+ """ Generic duplicate entry finder.
+
+ :param allentries: A list of all entries to check for
+ duplicates.
+ :type allentries: list of lxml.etree._Element
+ :param etype: The entry type. This will be used to determine
+ the error name (``duplicate-<etype>``) and for
+ display to the end user.
+ :type etype: string
+ """
entries = dict()
for el in allentries:
- if include(el):
- if el.get("name") in entries:
- entries[el.get("name")].append(self.RenderXML(el))
- else:
- entries[el.get("name")] = [self.RenderXML(el)]
+ if el.get("name") in entries:
+ entries[el.get("name")].append(self.RenderXML(el))
+ else:
+ entries[el.get("name")] = [self.RenderXML(el)]
for ename, els in entries.items():
if len(els) > 1:
self.LintError("duplicate-%s" % etype,
@@ -1598,7 +1614,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(etype.title(), ename, "\n".join(els)))
def default_is_profile(self):
- """ ensure that the default group is a profile group """
+ """ Ensure that the default group is a profile group. """
if (self.metadata.default and
not self.metadata.groups[self.metadata.default].is_profile):
xdata = \
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 48c580be1..dba56eed2 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -39,6 +39,11 @@ class AptCollection(Collection):
else:
lines.append("deb %s %s %s" % (source.url, source.version,
" ".join(source.components)))
+ if source.debsrc:
+ lines.append("deb-src %s %s %s" %
+ (source.url,
+ source.version,
+ " ".join(source.components)))
lines.append("")
return "\n".join(lines)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 3319dda78..767ac13ac 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -152,6 +152,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
#: this source
self.whitelist = [item.text for item in xsource.findall('Whitelist')]
+ #: Whether or not to include deb-src lines in the generated APT
+ #: configuration
+ self.debsrc = xsource.get('debsrc', 'false') == 'true'
+
#: A dict of repository options that will be included in the
#: configuration generated on the server side (if such is
#: applicable; most backends do not generate any sort of
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 052c362ab..567a16c40 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -9,7 +9,7 @@ import shutil
import lxml.etree
import Bcfg2.Logger
import Bcfg2.Server.Plugin
-from Bcfg2.Compat import ConfigParser, urlopen, HTTPError
+from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError
from Bcfg2.Server.Plugins.Packages.Collection import Collection, \
get_collection_class
from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
@@ -445,7 +445,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
try:
open(localfile, 'w').write(urlopen(key).read())
keys.append(key)
- except HTTPError:
+ except (URLError, HTTPError):
err = sys.exc_info()[1]
self.logger.error("Packages: Error downloading %s: %s"
% (key, err))
diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
index 7dac907e1..a1dcb575f 100644
--- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
+++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
@@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir):
class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin):
- """ find duplicate Pkgmgr entries with the same priority """
+ """ Find duplicate :ref:`Pkgmgr
+ <server-plugins-generators-pkgmgr>` entries with the same
+ priority. """
+
def Run(self):
pset = set()
for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr',
@@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin):
# check if package is already listed with same
# priority, type, grp
if ptuple in pset:
- self.LintError("duplicate-package",
- "Duplicate Package %s, priority:%s, type:%s" %
- (pkg.get('name'), priority, ptype))
+ self.LintError(
+ "duplicate-package",
+ "Duplicate Package %s, priority:%s, type:%s" %
+ (pkg.get('name'), priority, ptype))
else:
pset.add(ptuple)
-
+
@classmethod
def Errors(cls):
- return {"duplicate-packages":"error"}
+ return {"duplicate-packages": "error"}
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 7e4935d74..e97607093 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -65,7 +65,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924
.json, and .yaml properties to provide convenient ways to use
ProbeData objects as XML, JSON, or YAML data """
def __new__(cls, data):
- return str.__new__(cls, data.encode('utf-8'))
+ return str.__new__(cls, data)
def __init__(self, data): # pylint: disable=W0613
str.__init__(self)
@@ -224,15 +224,9 @@ class Probes(Bcfg2.Server.Plugin.Probing,
lxml.etree.SubElement(top, 'Client', name=client,
timestamp=str(int(probedata.timestamp)))
for probe in sorted(probedata):
- try:
- lxml.etree.SubElement(
- ctag, 'Probe', name=probe,
- value=str(
- self.probedata[client][probe]).decode('utf-8'))
- except AttributeError:
- lxml.etree.SubElement(
- ctag, 'Probe', name=probe,
- value=str(self.probedata[client][probe]))
+ lxml.etree.SubElement(
+ ctag, 'Probe', name=probe,
+ value=self.probedata[client][probe])
for group in sorted(self.cgroups[client]):
lxml.etree.SubElement(ctag, "Group", name=group)
try:
diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
index 1264fd1cf..84dcf2780 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
@@ -173,8 +173,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
for name in names[cmeta.hostname]:
newnames.add(name.split('.')[0])
try:
- newips.add(self.get_ipcache_entry(name)[0])
- except PluginExecutionError:
+ newips.update(self.get_ipcache_entry(name)[0])
+ except: # pylint: disable=W0702
continue
names[cmeta.hostname].update(newnames)
names[cmeta.hostname].update(cmeta.addresses)
@@ -290,7 +290,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
else:
# need to add entry
try:
- ipaddr = socket.gethostbyname(client)
+ ipaddr = set([info[4][0]
+ for info in socket.getaddrinfo(client, None)])
self.ipcache[client] = (ipaddr, client)
return (ipaddr, client)
except socket.gaierror:
diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
index e834759c2..050ba3b3e 100644
--- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
+++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
@@ -96,7 +96,18 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin,
class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin):
- """ find duplicate Pkgmgr entries with the same priority """
+ """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper
+ <server-plugins-connectors-templatehelper>` modules are valid.
+ This can check for:
+
+ * A TemplateHelper module that cannot be imported due to syntax or
+ other compile-time errors;
+ * A TemplateHelper module that does not have an ``__export__``
+ attribute, or whose ``__export__`` is not a list;
+ * Bogus symbols listed in ``__export__``, including symbols that
+ don't exist, that are reserved, or that start with underscores.
+ """
+
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
self.reserved_keywords = dir(HelperModule("foo.py"))
@@ -107,7 +118,11 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin):
self.check_helper(helper.name)
def check_helper(self, helper):
- """ check a helper module for export errors """
+ """ Check a single helper module.
+
+ :param helper: The filename of the helper module
+ :type helper: string
+ """
module_name = MODULE_RE.search(helper).group(1)
try:
diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py
index 13c756049..eea2183f7 100644
--- a/src/lib/Bcfg2/Server/SSLServer.py
+++ b/src/lib/Bcfg2/Server/SSLServer.py
@@ -346,7 +346,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer,
:param register: Presence should be reported to service-location
:type register: bool
:param allow_none: Allow None values in XML-RPC
- :type allow_non: bool
+ :type allow_none: bool
:param encoding: Encoding to use for XML-RPC
"""
diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py
index 68282331e..69c3264f9 100644
--- a/src/lib/Bcfg2/Utils.py
+++ b/src/lib/Bcfg2/Utils.py
@@ -2,6 +2,7 @@
used by both client and server. Stuff that doesn't fit anywhere
else. """
+import shlex
import fcntl
import logging
import threading
@@ -217,6 +218,7 @@ class Executor(object):
"""
if isinstance(command, str):
cmdstr = command
+ command = shlex.split(cmdstr)
else:
cmdstr = " ".join(command)
self.logger.debug("Running: %s" % cmdstr)