diff options
24 files changed, 621 insertions, 530 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py index f40df38f3..d5aaf069d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py @@ -1,12 +1,12 @@ +""" Handle <Path type='nonexistent' ...> entries """ + import os import sys -try: - from base import POSIXTool, device_map -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool, device_map") +from Bcfg2.Client.Tools.POSIX.base import POSIXTool, device_map + class POSIXDevice(POSIXTool): + """ Handle <Path type='nonexistent' ...> entries """ __req__ = ['name', 'dev_type', 'mode', 'owner', 'group'] def fully_specified(self, entry): @@ -22,7 +22,7 @@ class POSIXDevice(POSIXTool): ondisk = self._exists(entry) if not ondisk: return False - + # attempt to verify device properties as specified in config rv = True dev_type = entry.get('dev_type') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py index d2d383f66..9aa8e7fa1 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py @@ -1,15 +1,15 @@ +""" Handle <Path type='directory' ...> entries """ + import os import sys import stat import shutil import Bcfg2.Client.XML -try: - from base import POSIXTool -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool") +from Bcfg2.Client.Tools.POSIX.base import POSIXTool + class POSIXDirectory(POSIXTool): + """ Handle <Path type='directory' ...> entries """ __req__ = ['name', 'perms', 'owner', 'group'] def verify(self, entry, modlist): @@ -18,10 +18,11 @@ class POSIXDirectory(POSIXTool): return False if not stat.S_ISDIR(ondisk[stat.ST_MODE]): - self.logger.info("POSIX: %s is not a directory" % entry.get('name')) + self.logger.info("POSIX: %s is not a directory" % + entry.get('name')) return False - - pruneTrue = True + + prune = True if entry.get('prune', 'false').lower() == 'true': # check for any extra entries when prune='true' attribute is set try: @@ -30,7 +31,7 @@ class POSIXDirectory(POSIXTool): if os.path.join(entry.get('name'), ent) not in modlist] if extras: - pruneTrue = False + prune = False msg = "Directory %s contains extra entries: %s" % \ (entry.get('name'), "; ".join(extras)) self.logger.info("POSIX: " + msg) @@ -38,9 +39,9 @@ class POSIXDirectory(POSIXTool): for extra in extras: Bcfg2.Client.XML.SubElement(entry, 'Prune', path=extra) except OSError: - pruneTrue = True + prune = True - return POSIXTool.verify(self, entry, modlist) and pruneTrue + return POSIXTool.verify(self, entry, modlist) and prune def install(self, entry): """Install device entries.""" @@ -60,7 +61,7 @@ class POSIXDirectory(POSIXTool): elif fmode: self.logger.debug("POSIX: Found a pre-existing directory at %s" % entry.get('name')) - + rv = True if not fmode: rv &= self._makedirs(entry) @@ -71,12 +72,12 @@ class POSIXDirectory(POSIXTool): pname = pent.get('path') ulfailed = False if os.path.isdir(pname): - rm = shutil.rmtree + remove = shutil.rmtree else: - rm = os.unlink + remove = os.unlink try: self.logger.debug("POSIX: Removing %s" % pname) - rm(pname) + remove(pname) except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to unlink %s: %s" % diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 99f0ed804..40aade818 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -1,17 +1,17 @@ +""" Handle <Path type='file' ...> entries """ + import os import sys import stat import time import difflib import tempfile -try: - from base import POSIXTool -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool") -from Bcfg2.Compat import unicode, b64encode, b64decode +from Bcfg2.Client.Tools.POSIX.base import POSIXTool +from Bcfg2.Compat import unicode, b64encode, b64decode # pylint: disable=W0622 + class POSIXFile(POSIXTool): + """ Handle <Path type='file' ...> entries """ __req__ = ['name', 'perms', 'owner', 'group'] def fully_specified(self, entry): @@ -29,15 +29,16 @@ class POSIXFile(POSIXTool): try: strng.decode(encoding) return True - except: + except: # pylint: disable=W0702 return False def _get_data(self, entry): + """ Get a tuple of (<file data>, <is binary>) for the given entry """ is_binary = False if entry.get('encoding', 'ascii') == 'base64': tempdata = b64decode(entry.text) is_binary = True - + elif entry.get('empty', 'false') == 'true': tempdata = '' else: @@ -89,6 +90,7 @@ class POSIXFile(POSIXTool): return POSIXTool.verify(self, entry, modlist) and not different def _write_tmpfile(self, entry): + """ Write the file data to a temp file """ filedata, _ = self._get_data(entry) # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions @@ -115,16 +117,17 @@ class POSIXFile(POSIXTool): return newfile def _rename_tmpfile(self, newfile, entry): + """ Rename the given file to the appropriate filename for entry """ try: os.rename(newfile, entry.get('name')) return True except OSError: err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to rename temp file %s to %s: %s" % - (newfile, entry.get('name'), err)) + self.logger.error("POSIX: Failed to rename temp file %s to %s: %s" + % (newfile, entry.get('name'), err)) try: os.unlink(newfile) - except: + except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Could not remove temp file %s: %s" % (newfile, err)) @@ -147,6 +150,7 @@ class POSIXFile(POSIXTool): def _get_diffs(self, entry, interactive=False, sensitive=False, is_binary=False, content=None): + """ generate the necessary diffs for entry """ if not interactive and sensitive: return @@ -201,6 +205,8 @@ class POSIXFile(POSIXTool): entry.set(attr, val) def _diff(self, content1, content2, difffunc, filename=None): + """ Return a diff of the two strings, as produced by difffunc. + warns after 5 seconds and times out after 30 seconds. """ rv = [] start = time.time() longtime = False @@ -217,8 +223,8 @@ class POSIXFile(POSIXTool): longtime = True elif now - start > 30: if filename: - self.logger.error("POSIX: Diff of %s took too long; giving " - "up" % filename) + self.logger.error("POSIX: Diff of %s took too long; " + "giving up" % filename) else: self.logger.error("POSIX: Diff took too long; giving up") return False diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py index ca7a23717..64a0b1e15 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py @@ -1,43 +1,15 @@ -import os -import sys -try: - from base import POSIXTool -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool") +""" Handle <Path type="hardlink" ...> entries """ -class POSIXHardlink(POSIXTool): - __req__ = ['name', 'to'] +import os +from Bcfg2.Client.Tools.POSIX.base import POSIXLinkTool - def verify(self, entry, modlist): - rv = True - try: - if not os.path.samefile(entry.get('name'), entry.get('to')): - msg = "Hardlink %s is incorrect" % entry.get('name') - self.logger.debug("POSIX: " + msg) - entry.set('qtext', "\n".join([entry.get('qtext', ''), msg])) - rv = False - except OSError: - self.logger.debug("POSIX: %s %s does not exist" % - (entry.tag, entry.get("name"))) - entry.set('current_exists', 'false') - return False +class POSIXHardlink(POSIXLinkTool): + """ Handle <Path type="hardlink" ...> entries """ + __linktype__ = "hardlink" - return POSIXTool.verify(self, entry, modlist) and rv - - def install(self, entry): - ondisk = self._exists(entry, remove=True) - if ondisk: - self.logger.info("POSIX: Hardlink %s cleanup failed" % - entry.get('name')) - try: - os.link(entry.get('to'), entry.get('name')) - rv = True - except OSError: - err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to create hardlink %s to %s: %s" % - (entry.get('name'), entry.get('to'), err)) - rv = False - return POSIXTool.install(self, entry) and rv + def _verify(self, entry): + return os.path.samefile(entry.get('name'), entry.get('to')) + def _link(self, entry): + return os.link(entry.get('to'), entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py index c870ca0ed..5f1fbbe7c 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py @@ -1,13 +1,13 @@ +""" Handle <Path type='nonexistent' ...> entries """ + import os import sys import shutil -try: - from base import POSIXTool -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool") +from Bcfg2.Client.Tools.POSIX.base import POSIXTool + class POSIXNonexistent(POSIXTool): + """ Handle <Path type='nonexistent' ...> entries """ __req__ = ['name'] def verify(self, entry, _): @@ -16,7 +16,7 @@ class POSIXNonexistent(POSIXTool): entry.get("name")) return False return True - + def install(self, entry): ename = entry.get('name') if entry.get('recursive', '').lower() == 'true': @@ -31,13 +31,13 @@ class POSIXNonexistent(POSIXTool): 'specified in your configuration.' % ename) return False - rm = shutil.rmtree + remove = shutil.rmtree elif os.path.isdir(ename): - rm = os.rmdir + remove = os.rmdir else: - rm = os.remove + remove = os.remove try: - rm(ename) + remove(ename) return True except OSError: err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py b/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py index 321376b98..5859f844a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py @@ -1,11 +1,8 @@ -import os -import sys -try: - from base import POSIXTool -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool") +""" Handle <Path type='permissions' ...> entries """ + +from Bcfg2.Client.Tools.POSIX.base import POSIXTool + class POSIXPermissions(POSIXTool): + """ Handle <Path type='permissions' ...> entries """ __req__ = ['name', 'perms', 'owner', 'group'] - diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py index fb303bdbe..5f4fa6ad7 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py @@ -1,46 +1,19 @@ +""" Handle <Path type="symlink" ...> entries """ + import os -import sys -try: - from base import POSIXTool -except ImportError: - # py3k, incompatible syntax with py2.4 - exec("from .base import POSIXTool") +from Bcfg2.Client.Tools.POSIX.base import POSIXLinkTool -class POSIXSymlink(POSIXTool): - __req__ = ['name', 'to'] - def verify(self, entry, modlist): - rv = True +class POSIXSymlink(POSIXLinkTool): + """ Handle <Path type="symlink" ...> entries """ + __linktype__ = "symlink" - try: - sloc = os.readlink(entry.get('name')) - if sloc != entry.get('to'): - entry.set('current_to', sloc) - msg = ("Symlink %s points to %s, should be %s" % - (entry.get('name'), sloc, entry.get('to'))) - self.logger.debug("POSIX: " + msg) - entry.set('qtext', "\n".join([entry.get('qtext', ''), msg])) - rv = False - except OSError: - self.logger.debug("POSIX: %s %s does not exist" % - (entry.tag, entry.get("name"))) - entry.set('current_exists', 'false') + def _verify(self, entry): + sloc = os.readlink(entry.get('name')) + if sloc != entry.get('to'): + entry.set('current_to', sloc) return False + return True - return POSIXTool.verify(self, entry, modlist) and rv - - def install(self, entry): - ondisk = self._exists(entry, remove=True) - if ondisk: - self.logger.info("POSIX: Symlink %s cleanup failed" % - entry.get('name')) - try: - os.symlink(entry.get('to'), entry.get('name')) - rv = True - except OSError: - err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to create symlink %s to %s: %s" % - (entry.get('name'), entry.get('to'), err)) - rv = False - return POSIXTool.install(self, entry) and rv - + def _link(self, entry): + return os.symlink(entry.get('to'), entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 15acd2b3d..7708c4f72 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -18,7 +18,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) self.ppath = setup['ppath'] self.max_copies = setup['max_copies'] - self._load_handlers() + self._handlers = self._load_handlers() self.logger.debug("POSIX: Handlers loaded: %s" % (", ".join(self._handlers.keys()))) self.__req__ = dict(Path=dict()) @@ -39,21 +39,22 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.handlesEntry(e))]) def _load_handlers(self): - # this must be called at run-time, not at compile-time, or we - # get wierd circular import issues. - self._handlers = dict() + """ load available POSIX tool handlers. this must be called + at run-time, not at compile-time, or we get wierd circular + import issues. """ + rv = dict() for submodule in walk_packages(path=__path__, prefix=__name__ + "."): mname = submodule[1].rsplit('.', 1)[-1] if mname == 'base': continue - module = getattr(__import__(submodule[1]).Client.Tools.POSIX, mname) + module = getattr(__import__(submodule[1]).Client.Tools.POSIX, + mname) hdlr = getattr(module, "POSIX" + mname) if POSIXTool in hdlr.__mro__: # figure out what entry type this handler handles etype = hdlr.__name__[5:].lower() - self._handlers[etype] = hdlr(self.logger, - self.setup, - self.config) + rv[etype] = hdlr(self.logger, self.setup, self.config) + return rv def canVerify(self, entry): if not Bcfg2.Client.Tools.Tool.canVerify(self, entry): @@ -96,9 +97,10 @@ class POSIX(Bcfg2.Client.Tools.Tool): return ret def _prune_old_backups(self, entry): + """ Remove old paranoid backup files """ bkupnam = entry.get('name').replace('/', '_') - bkup_re = re.compile(bkupnam + \ - r'_\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}$') + bkup_re = re.compile( + bkupnam + r'_\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}$') # current list of backups for this file try: bkuplist = [f for f in os.listdir(self.ppath) if @@ -121,6 +123,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): (os.path.join(self.ppath, oldest), err)) def _paranoid_backup(self, entry): + """ Take a backup of the specified entry for paranoid mode """ if (entry.get("paranoid", 'false').lower() == 'true' and self.setup.get("paranoid", False) and entry.get('current_exists', 'true') == 'true' and @@ -135,5 +138,5 @@ class POSIX(Bcfg2.Client.Tools.Tool): (entry.get('name'), bfile)) except IOError: err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to create backup file for %s: " - "%s" % (entry.get('name'), err)) + self.logger.error("POSIX: Failed to create backup file for " + "%s: %s" % (entry.get('name'), err)) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 2fe6791ae..6e7eb70b0 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -1,3 +1,5 @@ +""" Base class for tools that handle POSIX (Path) entries """ + import os import sys import pwd @@ -9,47 +11,51 @@ import Bcfg2.Client.XML try: import selinux - has_selinux = True + HAS_SELINUX = True except ImportError: - has_selinux = False + HAS_SELINUX = False try: import posix1e - has_acls = True + HAS_ACLS = True # map between permissions characters and numeric ACL constants - acl_map = dict(r=posix1e.ACL_READ, + ACL_MAP = dict(r=posix1e.ACL_READ, w=posix1e.ACL_WRITE, x=posix1e.ACL_EXECUTE) except ImportError: - has_acls = False - acl_map = dict(r=4, w=2, x=1) + HAS_ACLS = False + ACL_MAP = dict(r=4, w=2, x=1) # map between dev_type attribute and stat constants -device_map = dict(block=stat.S_IFBLK, +device_map = dict(block=stat.S_IFBLK, # pylint: disable=C0103 char=stat.S_IFCHR, fifo=stat.S_IFIFO) class POSIXTool(Bcfg2.Client.Tools.Tool): - def fully_specified(self, entry): + """ Base class for tools that handle POSIX (Path) entries """ + def fully_specified(self, entry): # pylint: disable=W0613 + """ return True if the entry is fully specified """ # checking is done by __req__ return True - def verify(self, entry, modlist): + def verify(self, entry, modlist): # pylint: disable=W0613 + """ return True if the entry is correct on disk """ if not self._verify_metadata(entry): return False if entry.get('recursive', 'false').lower() == 'true': # verify ownership information recursively for root, dirs, files in os.walk(entry.get('name')): - for p in dirs + files: + for path in dirs + files: if not self._verify_metadata(entry, - path=os.path.join(root, p)): + path=os.path.join(root, + path)): return False return True def install(self, entry): - plist = [entry.get('name')] + """ Install the given entry. Return True on success. """ rv = True rv &= self._set_perms(entry) if entry.get('recursive', 'false').lower() == 'true': @@ -60,28 +66,31 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): return rv def _exists(self, entry, remove=False): + """ check for existing paths and optionally remove them. if + the path exists, return the lstat of it """ try: - # check for existing paths and optionally remove them ondisk = os.lstat(entry.get('name')) if remove: if os.path.isdir(entry.get('name')): - rm = shutil.rmtree + remove = shutil.rmtree else: - rm = os.unlink + remove = os.unlink try: - rm(entry.get('name')) - return False + remove(entry.get('name')) + return None except OSError: err = sys.exc_info()[1] self.logger.warning('POSIX: Failed to unlink %s: %s' % (entry.get('name'), err)) - return ondisk # probably still exists + return ondisk # probably still exists else: return ondisk except OSError: - return False + return None def _set_perms(self, entry, path=None): + """ set permissions on the given entry, or on the given path + according to the given entry """ if path is None: path = entry.get("name") @@ -105,13 +114,13 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): rv = False if entry.get("perms"): - configPerms = int(entry.get('perms'), 8) + wanted_perms = int(entry.get('perms'), 8) if entry.get('dev_type'): - configPerms |= device_map[entry.get('dev_type')] + wanted_perms |= device_map[entry.get('dev_type')] try: self.logger.debug("POSIX: Setting permissions on %s to %s" % - (path, oct(configPerms))) - os.chmod(path, configPerms) + (path, oct(wanted_perms))) + os.chmod(path, wanted_perms) except (OSError, KeyError): self.logger.error('POSIX: Failed to change permissions on %s' % path) @@ -129,10 +138,35 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): rv &= self._set_acls(entry, path=path) return rv + def _apply_acl(self, acl, path, atype=posix1e.ACL_TYPE_ACCESS): + """ Apply the given ACL to the given path """ + if atype == posix1e.ACL_TYPE_ACCESS: + atype_str = "access" + else: + atype_str = "default" + if acl.valid(): + self.logger.debug("POSIX: Applying %s ACL to %s:" % (atype_str, + path)) + for line in str(acl).splitlines(): + self.logger.debug(" " + line) + try: + acl.applyto(path, atype) + return True + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to set ACLs on %s: %s" % + (path, err)) + return False + else: + self.logger.warning("POSIX: %s ACL created for %s was invalid:" + % (atype_str.title(), path)) + for line in str(acl).splitlines(): + self.logger.warning(" " + line) + return False - def _set_acls(self, entry, path=None): + def _set_acls(self, entry, path=None): # pylint: disable=R0912 """ set POSIX ACLs on the file on disk according to the config """ - if not has_acls: + if not HAS_ACLS: if entry.findall("ACL"): self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " "library installed" % entry.get('name')) @@ -182,20 +216,20 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): self.logger.warning("POSIX: Cannot set default ACLs on " "non-directory %s" % path) continue - entry = posix1e.Entry(defacl) + aclentry = posix1e.Entry(defacl) else: - entry = posix1e.Entry(acl) - for perm in acl_map.values(): + aclentry = posix1e.Entry(acl) + for perm in ACL_MAP.values(): if perm & perms: - entry.permset.add(perm) - entry.tag_type = scope + aclentry.permset.add(perm) + aclentry.tag_type = scope try: if scope == posix1e.ACL_USER: scopename = "user" - entry.qualifier = self._norm_uid(qualifier) + aclentry.qualifier = self._norm_uid(qualifier) elif scope == posix1e.ACL_GROUP: scopename = "group" - entry.qualifier = self._norm_gid(qualifier) + aclentry.qualifier = self._norm_gid(qualifier) except (OSError, KeyError): err = sys.exc_info()[1] self.logger.error("POSIX: Could not resolve %s %s: %s" % @@ -203,41 +237,16 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): continue acl.calc_mask() - def _apply_acl(acl, path, atype=posix1e.ACL_TYPE_ACCESS): - if atype == posix1e.ACL_TYPE_ACCESS: - atype_str = "access" - else: - atype_str = "default" - if acl.valid(): - self.logger.debug("POSIX: Applying %s ACL to %s:" % (atype_str, - path)) - for line in str(acl).splitlines(): - self.logger.debug(" " + line) - try: - acl.applyto(path, atype) - return True - except: - err = sys.exc_info()[1] - self.logger.error("POSIX: Failed to set ACLs on %s: %s" % - (path, err)) - return False - else: - self.logger.warning("POSIX: %s ACL created for %s was invalid:" - % (atype_str.title(), path)) - for line in str(acl).splitlines(): - self.logger.warning(" " + line) - return False - - rv = _apply_acl(acl, path) + rv = self._apply_acl(acl, path) if defacl: defacl.calc_mask() - rv &= _apply_acl(defacl, path, posix1e.ACL_TYPE_DEFAULT) + rv &= self._apply_acl(defacl, path, posix1e.ACL_TYPE_DEFAULT) return rv def _set_secontext(self, entry, path=None): """ set the SELinux context of the file on disk according to the config""" - if not has_selinux: + if not HAS_SELINUX: return True if path is None: @@ -251,7 +260,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: selinux.restorecon(path) rv = True - except: + except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to restore SELinux context " "for %s: %s" % (path, err)) @@ -259,7 +268,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: try: rv = selinux.lsetfilecon(path, context) == 0 - except: + except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to restore SELinux context " "for %s: %s" % (path, err)) @@ -275,12 +284,15 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): return int(grp.getgrnam(gid)[2]) def _norm_entry_gid(self, entry): + """ Given an entry, return the GID number of the desired group """ try: return self._norm_gid(entry.get('group')) except (OSError, KeyError): err = sys.exc_info()[1] - self.logger.error('POSIX: GID normalization failed for %s on %s: %s' - % (entry.get('group'), entry.get('name'), err)) + self.logger.error('POSIX: GID normalization failed for %s on %s: ' + '%s' % (entry.get('group'), + entry.get('name'), + err)) return 0 def _norm_uid(self, uid): @@ -292,12 +304,15 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): return int(pwd.getpwnam(uid)[2]) def _norm_entry_uid(self, entry): + """ Given an entry, return the UID number of the desired owner """ try: return self._norm_uid(entry.get("owner")) except (OSError, KeyError): err = sys.exc_info()[1] - self.logger.error('POSIX: UID normalization failed for %s on %s: %s' - % (entry.get('owner'), entry.get('name'), err)) + self.logger.error('POSIX: UID normalization failed for %s on %s: ' + '%s' % (entry.get('owner'), + entry.get('name'), + err)) return 0 def _norm_acl_perms(self, perms): @@ -307,7 +322,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): 'w', 'x', or '-' characters, or a posix1e.Permset object""" if hasattr(perms, 'test'): # Permset object - return sum([p for p in acl_map.values() + return sum([p for p in ACL_MAP.values() if perms.test(p)]) try: @@ -329,17 +344,19 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): for char in perms: if char == '-': continue - elif char not in acl_map: + elif char not in ACL_MAP: self.logger.warning("POSIX: Unknown permissions character " "in ACL: %s" % char) - elif rv & acl_map[char]: + elif rv & ACL_MAP[char]: self.logger.warning("POSIX: Duplicate permissions " "character in ACL: %s" % perms) else: - rv |= acl_map[char] + rv |= ACL_MAP[char] return rv def _acl2string(self, aclkey, perms): + """ Get a string representation of the given ACL. aclkey must + be a tuple of (<acl type>, <acl scope>, <qualifier>) """ atype, scope, qualifier = aclkey acl_str = [] if atype == 'default': @@ -353,15 +370,19 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): return ":".join(acl_str) def _acl_perm2string(self, perm): + """ Turn an octal permissions integer into a string suitable + for use with ACLs """ rv = [] for char in 'rwx': - if acl_map[char] & perm: + if ACL_MAP[char] & perm: rv.append(char) else: rv.append('-') return ''.join(rv) def _gather_data(self, path): + """ Get data on the existing state of <path> -- e.g., whether + or not it exists, owner, group, permissions, etc. """ try: ondisk = os.stat(path) except OSError: @@ -394,11 +415,11 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): perms = oct(ondisk[stat.ST_MODE])[-4:] except (OSError, KeyError, TypeError): err = sys.exc_info()[1] - self.logger.debug("POSIX: Could not get current permissions of %s: " - "%s" % (path, err)) + self.logger.debug("POSIX: Could not get current permissions of " + "%s: %s" % (path, err)) perms = None - if has_selinux: + if HAS_SELINUX: try: secontext = selinux.getfilecon(path)[1].split(":")[2] except (OSError, KeyError): @@ -409,13 +430,13 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if has_acls: + if HAS_ACLS: acls = self._list_file_acls(path) else: acls = None return (ondisk, owner, group, perms, secontext, acls) - def _verify_metadata(self, entry, path=None): + def _verify_metadata(self, entry, path=None): # pylint: disable=R0912 """ generic method to verify perms, owner, group, secontext, acls, and mtime """ # allow setting an alternate path for recursive permissions checking @@ -423,9 +444,9 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): path = entry.get('name') attrib = dict() ondisk, attrib['current_owner'], attrib['current_group'], \ - attrib['current_perms'], attrib['current_secontext'], acls = \ - self._gather_data(path) - + attrib['current_perms'], attrib['current_secontext'] = \ + self._gather_data(path)[0:5] + if not ondisk: entry.set('current_exists', 'false') return False @@ -438,31 +459,31 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): # symlink and hardlink entries, which have SELinux contexts # but not other permissions, optional secontext and mtime # attrs, and so on. - configOwner, configGroup, configPerms, mtime = None, None, None, -1 + wanted_owner, wanted_group, wanted_perms, mtime = None, None, None, -1 if entry.get('mtime', '-1') != '-1': mtime = str(ondisk[stat.ST_MTIME]) if entry.get("owner"): - configOwner = str(self._norm_entry_uid(entry)) + wanted_owner = str(self._norm_entry_uid(entry)) if entry.get("group"): - configGroup = str(self._norm_entry_gid(entry)) + wanted_group = str(self._norm_entry_gid(entry)) if entry.get("perms"): while len(entry.get('perms', '')) < 4: entry.set('perms', '0' + entry.get('perms', '')) - configPerms = int(entry.get('perms'), 8) + wanted_perms = int(entry.get('perms'), 8) errors = [] - if configOwner and attrib['current_owner'] != configOwner: + if wanted_owner and attrib['current_owner'] != wanted_owner: errors.append("Owner for path %s is incorrect. " "Current owner is %s but should be %s" % (path, attrib['current_owner'], entry.get('owner'))) - if configGroup and attrib['current_group'] != configGroup: + if wanted_group and attrib['current_group'] != wanted_group: errors.append("Group for path %s is incorrect. " "Current group is %s but should be %s" % (path, attrib['current_group'], entry.get('group'))) - if (configPerms and - oct(int(attrib['current_perms'], 8)) != oct(configPerms)): + if (wanted_perms and + oct(int(attrib['current_perms'], 8)) != oct(wanted_perms)): errors.append("Permissions for path %s are incorrect. " "Current permissions are %s but should be %s" % (path, attrib['current_perms'], entry.get('perms'))) @@ -474,16 +495,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): "Current mtime is %s but should be %s" % (path, mtime, entry.get('mtime'))) - if has_selinux and entry.get("secontext"): + if HAS_SELINUX and entry.get("secontext"): if entry.get("secontext") == "__default__": - configContext = selinux.matchpathcon(path, 0)[1].split(":")[2] + wanted_secontext = \ + selinux.matchpathcon(path, 0)[1].split(":")[2] else: - configContext = entry.get("secontext") - if attrib['current_secontext'] != configContext: + wanted_secontext = entry.get("secontext") + if attrib['current_secontext'] != wanted_secontext: errors.append("SELinux context for path %s is incorrect. " "Current context is %s but should be %s" % (path, attrib['current_secontext'], - configContext)) + wanted_secontext)) if errors: for error in errors: @@ -494,10 +516,11 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if val is not None: entry.set(attr, str(val)) - aclVerifies = self._verify_acls(entry, path=path) - return aclVerifies and len(errors) == 0 + return self._verify_acls(entry, path=path) and len(errors) == 0 def _list_entry_acls(self, entry): + """ Given an entry, get a dict of POSIX ACLs described in that + entry. """ wanted = dict() for acl in entry.findall("ACL"): if acl.get("scope") == "user": @@ -513,7 +536,14 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): return wanted def _list_file_acls(self, path): + """ Given a path, get a dict of existing POSIX ACLs on that + path. The dict keys are a tuple of (<acl type (access or + default)>, <acl scope (user or group)>, <acl qualifer (the + user or group it applies to)>. values are the permissions of + the described ACL. """ def _process_acl(acl, atype): + """ Given an ACL object, process it appropriately and add + it to the return value """ try: if acl.tag_type == posix1e.ACL_USER: qual = pwd.getpwuid(acl.qualifier)[0] @@ -550,7 +580,9 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): return existing def _verify_acls(self, entry, path=None): - if not has_acls: + """ verify POSIX ACLs on the given entry. return True if all + ACLS are correct, false otherwise """ + if not HAS_ACLS: if entry.findall("ACL"): self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " "library installed" % entry.get('name')) @@ -593,7 +625,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): for aclkey, perms in existing.items(): if aclkey not in wanted: extra.append(self._acl2string(aclkey, perms)) - + msg = [] if missing: msg.append("%s ACLs are missing: %s" % (len(missing), @@ -640,3 +672,51 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): for cpath in created: rv &= self._set_perms(entry, path=cpath) return rv + + +class POSIXLinkTool(POSIXTool): + """ Base handler for link (symbolic and hard) entries """ + __req__ = ['name', 'to'] + __linktype__ = None + + def verify(self, entry, modlist): + rv = True + + try: + if not self._verify(entry): + msg = "%s %s is incorrect" % (self.__linktype__.title(), + entry.get('name')) + self.logger.debug("POSIX: " + msg) + entry.set('qtext', "\n".join([entry.get('qtext', ''), msg])) + rv = False + except OSError: + self.logger.debug("POSIX: %s %s does not exist" % + (entry.tag, entry.get("name"))) + entry.set('current_exists', 'false') + return False + + return POSIXTool.verify(self, entry, modlist) and rv + + def _verify(self, entry): + """ perform actual verification of the link entry """ + raise NotImplementedError + + def install(self, entry): + ondisk = self._exists(entry, remove=True) + if ondisk: + self.logger.info("POSIX: %s %s cleanup failed" % + (self.__linktype__.title(), entry.get('name'))) + try: + self._link(entry) + rv = True + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to create %s %s to %s: %s" % + (self.__linktype__, entry.get('name'), + entry.get('to'), err)) + rv = False + return POSIXTool.install(self, entry) and rv + + def _link(self, entry): + """ create the link """ + raise NotImplementedError diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 9cd8a5531..4daba5b8c 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -249,3 +249,9 @@ try: except ImportError: import simplejson as json + +try: + from functools import wraps +except ImportError: + def wraps(wrapped): + return lambda f: f diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 5ceee36b7..ebd426802 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -1,23 +1,16 @@ """ the core of the builtin bcfg2 server """ -import os import sys import time import socket import daemon -import logging -from Bcfg2.Server.Core import BaseCore +from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.SSLServer import XMLRPCServer -logger = logging.getLogger() - - -class NoExposedMethod (Exception): - """There is no method exposed with the given name.""" - class Core(BaseCore): + """ The built-in server core """ name = 'bcfg2-server' def __init__(self, setup): @@ -25,21 +18,6 @@ class Core(BaseCore): self.server = None self.context = daemon.DaemonContext() - def _resolve_exposed_method(self, method_name): - """Resolve an exposed method. - - Arguments: - method_name -- name of the method to resolve - - """ - try: - func = getattr(self, method_name) - except AttributeError: - raise NoExposedMethod(method_name) - if not getattr(func, "exposed", False): - raise NoExposedMethod(method_name) - return func - def _dispatch(self, method, args, dispatch_dict): """Custom XML-RPC dispatcher for components. @@ -66,10 +44,10 @@ class Core(BaseCore): except xmlrpclib.Fault: raise except Exception: - e = sys.exc_info()[1] - if getattr(e, "log", True): - self.logger.error(e, exc_info=True) - raise xmlrpclib.Fault(getattr(e, "fault_code", 1), str(e)) + err = sys.exc_info()[1] + if getattr(err, "log", True): + self.logger.error(err, exc_info=True) + raise xmlrpclib.Fault(getattr(err, "fault_code", 1), str(err)) return result def _daemonize(self): @@ -91,10 +69,10 @@ class Core(BaseCore): timeout=1, ca=self.setup['ca'], protocol=self.setup['protocol']) - except: + except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Server startup failed: %s" % err) - os._exit(1) + self.context.close() self.server.register_instance(self) def _block(self): @@ -104,10 +82,3 @@ class Core(BaseCore): self.server.server_close() self.context.close() self.shutdown() - - def methodHelp(self, method_name): - try: - func = self._resolve_exposed_method(method_name) - except NoExposedMethod: - return "" - return func.__doc__ diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py index 5faed6f6c..684ea4556 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -3,14 +3,14 @@ import sys import time import cherrypy -import Bcfg2.Options from Bcfg2.Compat import urlparse, xmlrpclib, b64decode from Bcfg2.Server.Core import BaseCore from cherrypy.lib import xmlrpcutil from cherrypy._cptools import ErrorTool from cherrypy.process.plugins import Daemonizer -def on_error(*args, **kwargs): + +def on_error(*args, **kwargs): # pylint: disable=W0613 """ define our own error handler that handles xmlrpclib.Fault objects and so allows for the possibility of returning proper error codes. this obviates the need to use the builtin CherryPy @@ -18,12 +18,14 @@ def on_error(*args, **kwargs): err = sys.exc_info()[1] if not isinstance(err, xmlrpclib.Fault): err = xmlrpclib.Fault(xmlrpclib.INTERNAL_ERROR, str(err)) - xmlrpcutil._set_response(xmlrpclib.dumps(err)) + xmlrpcutil._set_response(xmlrpclib.dumps(err)) # pylint: disable=W0212 cherrypy.tools.xmlrpc_error = ErrorTool(on_error) class Core(BaseCore): + """ The CherryPy-based server core """ + _cp_config = {'tools.xmlrpc_error.on': True, 'tools.bcfg2_authn.on': True} @@ -37,34 +39,36 @@ class Core(BaseCore): cherrypy.engine.subscribe('stop', self.shutdown) def do_authn(self): + """ perform authentication """ try: header = cherrypy.request.headers['Authorization'] except KeyError: self.critical_error("No authentication data presented") - auth_type, auth_content = header.split() + auth_content = header.split()[1] auth_content = b64decode(auth_content) try: username, password = auth_content.split(":") except ValueError: username = auth_content password = "" - + # FIXME: Get client cert cert = None address = (cherrypy.request.remote.ip, cherrypy.request.remote.name) return self.authenticate(cert, username, password, address) @cherrypy.expose - def default(self, *vpath, **params): - # needed to make enough changes to the stock XMLRPCController - # to support plugin.__rmi__ and prepending client address that - # we just rewrote. it clearly wasn't written with inheritance - # in mind :( + def default(self, *args, **params): # pylint: disable=W0613 + """ needed to make enough changes to the stock + XMLRPCController to support plugin.__rmi__ and prepending + client address that we just rewrote. it clearly wasn't + written with inheritance in mind :( """ rpcparams, rpcmethod = xmlrpcutil.process_body() if rpcmethod == 'ERRORMETHOD': raise Exception("Unknown error processing XML-RPC request body") elif "." not in rpcmethod: - address = (cherrypy.request.remote.ip, cherrypy.request.remote.name) + address = (cherrypy.request.remote.ip, + cherrypy.request.remote.name) rpcparams = (address, ) + rpcparams handler = getattr(self, rpcmethod, None) @@ -81,7 +85,7 @@ class Core(BaseCore): body = handler(*rpcparams, **params) finally: self.stats.add_value(rpcmethod, time.time() - method_start) - + xmlrpcutil.respond(body, 'utf-8', True) return cherrypy.serving.response.body @@ -108,26 +112,3 @@ class Core(BaseCore): def _block(self): cherrypy.engine.block() - - -def parse_opts(argv=None): - if argv is None: - argv = sys.argv[1:] - optinfo = dict() - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.DAEMON_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo, argv=argv) - setup.parse(argv) - return setup - -def application(environ, start_response): - """ running behind Apache as a WSGI app is not currently - supported, but I'm keeping this code here because I hope for it to - be supported some day. we'll need to set up an AMQP task queue - and related magic for that to happen, though. """ - cherrypy.config.update({'environment': 'embedded'}) - setup = parse_opts(argv=['-C', environ['config']]) - root = Core(setup) - cherrypy.tree.mount(root) - return cherrypy.tree(environ, start_response) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 95d8173c6..0e5a88f79 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -16,7 +16,7 @@ import Bcfg2.Logger import Bcfg2.Server.FileMonitor from Bcfg2.Cache import Cache from Bcfg2.Statistics import Statistics -from Bcfg2.Compat import xmlrpclib, reduce # pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, reduce, wraps # pylint: disable=W0622 from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError try: @@ -29,11 +29,13 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' def exposed(func): + """ decorator that sets the 'exposed' attribute of a function to + expose it via XML-RPC """ func.exposed = True return func -class track_statistics(object): +class track_statistics(object): # pylint: disable=C0103 """ decorator that tracks execution time for the given function """ @@ -44,7 +46,9 @@ class track_statistics(object): if self.name is None: self.name = func.__name__ + @wraps(func) def inner(obj, *args, **kwargs): + """ The decorated function """ name = "%s:%s" % (obj.__class__.__name__, self.name) start = time.time() @@ -57,6 +61,7 @@ class track_statistics(object): def sort_xml(node, key=None): + """ sort XML in a deterministic fashion """ for child in node: sort_xml(child, key) @@ -72,12 +77,21 @@ class CoreInitError(Exception): pass +class NoExposedMethod (Exception): + """There is no method exposed with the given name.""" + + +# pylint: disable=W0702 +# in core we frequently want to catch all exceptions, regardless of +# type, so disable the pylint rule that catches that. + + class BaseCore(object): """The Core object is the container for all Bcfg2 Server logic and modules. """ - def __init__(self, setup): + def __init__(self, setup): # pylint: disable=R0912,R0915 self.datastore = setup['repo'] if setup['debug']: @@ -99,18 +113,19 @@ class BaseCore(object): self.logger = logging.getLogger('bcfg2-server') try: - fm = Bcfg2.Server.FileMonitor.available[setup['filemonitor']] + filemonitor = \ + Bcfg2.Server.FileMonitor.available[setup['filemonitor']] except KeyError: self.logger.error("File monitor driver %s not available; " "forcing to default" % setup['filemonitor']) - fm = Bcfg2.Server.FileMonitor.available['default'] + filemonitor = Bcfg2.Server.FileMonitor.available['default'] famargs = dict(ignore=[], debug=False) if 'ignore' in setup: famargs['ignore'] = setup['ignore'] if 'debug' in setup: famargs['debug'] = setup['debug'] try: - self.fam = fm(**famargs) + self.fam = filemonitor(**famargs) except IOError: msg = "Failed to instantiate fam driver %s" % setup['filemonitor'] self.logger.error(msg, exc_info=1) @@ -135,7 +150,8 @@ class BaseCore(object): self._database_available = False # verify our database schema try: - from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError + from Bcfg2.Server.SchemaUpdater import update_database, \ + UpdaterError try: update_database() self._database_available = True @@ -159,21 +175,21 @@ class BaseCore(object): if not plugin in self.plugins: self.init_plugins(plugin) # Remove blacklisted plugins - for p, bl in list(self.plugin_blacklist.items()): - if len(bl) > 0: + for plugin, blacklist in list(self.plugin_blacklist.items()): + if len(blacklist) > 0: self.logger.error("The following plugins conflict with %s;" - "Unloading %s" % (p, bl)) - for plug in bl: + "Unloading %s" % (plugin, blacklist)) + for plug in blacklist: del self.plugins[plug] # This section logs the experimental plugins - expl = [plug for (name, plug) in list(self.plugins.items()) + expl = [plug for plug in list(self.plugins.values()) if plug.experimental] if expl: self.logger.info("Loading experimental plugin(s): %s" % (" ".join([x.name for x in expl]))) self.logger.info("NOTE: Interfaces subject to change") # This section logs the deprecated plugins - depr = [plug for (name, plug) in list(self.plugins.items()) + depr = [plug for plug in list(self.plugins.values()) if plug.deprecated] if depr: self.logger.info("Loading deprecated plugin(s): %s" % @@ -187,7 +203,8 @@ class BaseCore(object): "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) - self.pull_sources = self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) + self.pull_sources = \ + self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) @@ -239,14 +256,16 @@ class BaseCore(object): (plugin)).Server.Plugins, plugin) except ImportError: try: - mod = __import__(plugin, globals(), locals(), [plugin.split('.')[-1]]) + mod = __import__(plugin, globals(), locals(), + [plugin.split('.')[-1]]) except: self.logger.error("Failed to load plugin %s" % plugin) return try: plug = getattr(mod, plugin.split('.')[-1]) except AttributeError: - self.logger.error("Failed to load plugin %s (AttributeError)" % plugin) + self.logger.error("Failed to load plugin %s (AttributeError)" % + plugin) return # Blacklist conflicting plugins cplugs = [conflict for conflict in plug.conflicts @@ -258,8 +277,8 @@ class BaseCore(object): self.logger.error("Failed to instantiate plugin %s" % plugin, exc_info=1) except: - self.logger.error("Unexpected instantiation failure for plugin %s" % - plugin, exc_info=1) + self.logger.error("Unexpected instantiation failure for plugin %s" + % plugin, exc_info=1) def shutdown(self): """Shutting down the plugins.""" @@ -304,7 +323,8 @@ class BaseCore(object): @track_statistics() def validate_structures(self, metadata, data): """Checks the data structure.""" - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): + for plugin in \ + self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): try: plugin.validate_structures(metadata, data) except Bcfg2.Server.Plugin.ValidationError: @@ -346,6 +366,8 @@ class BaseCore(object): @track_statistics() def BindStructures(self, structures, metadata, config): + """ Given a list of structures, bind all the entries in them + and add the structures to the config. """ for astruct in structures: try: self.BindStructure(astruct, metadata) @@ -372,8 +394,8 @@ class BaseCore(object): exc = sys.exc_info()[1] if 'failure' not in entry.attrib: entry.set('failure', 'bind error: %s' % format_exc()) - self.logger.error("Unexpected failure in BindStructure: %s %s" % - (entry.tag, entry.get('name')), exc_info=1) + self.logger.error("Unexpected failure in BindStructure: %s %s" + % (entry.tag, entry.get('name')), exc_info=1) def Bind(self, entry, metadata): """Bind an entry using the appropriate generator.""" @@ -393,8 +415,8 @@ class BaseCore(object): self.logger.error("Failed binding entry %s:%s with altsrc %s" % (entry.tag, entry.get('name'), entry.get('altsrc'))) - self.logger.error("Falling back to %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.error("Falling back to %s:%s" % + (entry.tag, entry.get('name'))) glist = [gen for gen in self.generators if entry.get('name') in gen.Entries.get(entry.tag, {})] @@ -557,6 +579,8 @@ class BaseCore(object): self.client_run_hook("end_statistics", meta) def resolve_client(self, address, cleanup_cache=False, metadata=True): + """ given a client address, get the client hostname and + optionally metadata """ try: client = self.metadata.resolve_client(address, cleanup_cache=cleanup_cache) @@ -581,6 +605,7 @@ class BaseCore(object): "Critical failure: %s" % operation) def _get_rmi(self): + """ Get a list of RMI calls exposed by plugins """ rmi = dict() if self.plugins: for pname, pinst in list(self.plugins.items()): @@ -588,9 +613,26 @@ class BaseCore(object): rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) return rmi + def _resolve_exposed_method(self, method_name): + """Resolve an exposed method. + + Arguments: + method_name -- name of the method to resolve + + """ + try: + func = getattr(self, method_name) + except AttributeError: + raise NoExposedMethod(method_name) + if not getattr(func, "exposed", False): + raise NoExposedMethod(method_name) + return func + # XMLRPC handlers start here + @exposed - def listMethods(self, address): + def listMethods(self, address): # pylint: disable=W0613 + """ list all exposed methods, including plugin RMI """ methods = [name for name, func in inspect.getmembers(self, callable) if getattr(func, "exposed", False)] @@ -598,13 +640,18 @@ class BaseCore(object): return methods @exposed - def methodHelp(self, address, method_name): - raise NotImplementedError + def methodHelp(self, address, method_name): # pylint: disable=W0613 + """ get help on an exposed method """ + try: + func = self._resolve_exposed_method(method_name) + except NoExposedMethod: + return "" + return func.__doc__ @exposed def DeclareVersion(self, address, version): """ declare the client version """ - client, metadata = self.resolve_client(address) + client = self.resolve_client(address, metadata=False)[0] try: self.metadata.set_version(client, version) except (Bcfg2.Server.Plugin.MetadataConsistencyError, @@ -645,26 +692,27 @@ class BaseCore(object): try: xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) - except: + except lxml.etree.XMLSyntaxError: err = sys.exc_info()[1] self.critical_error("Failed to parse probe data from client %s: %s" % (client, err)) sources = [] - [sources.append(data.get('source')) for data in xpdata - if data.get('source') not in sources] - for source in sources: - if source not in self.plugins: - self.logger.warning("Failed to locate plugin %s" % source) - continue - dl = [data for data in xpdata if data.get('source') == source] - try: - self.plugins[source].ReceiveData(metadata, dl) - except: - err = sys.exc_info()[1] - self.critical_error("Failed to process probe data from client " - "%s: %s" % - (client, err)) + for data in xpdata: + source = data.get('source') + if source not in sources: + if source not in self.plugins: + self.logger.warning("Failed to locate plugin %s" % source) + continue + datalist = [data for data in xpdata + if data.get('source') == source] + try: + self.plugins[source].ReceiveData(metadata, datalist) + except: + err = sys.exc_info()[1] + self.critical_error("Failed to process probe data from " + "client %s: %s" % + (client, err)) return True @exposed @@ -681,7 +729,7 @@ class BaseCore(object): return True @exposed - def GetConfig(self, address, checksum=False): + def GetConfig(self, address): """Build config for a client.""" client = self.resolve_client(address)[0] try: @@ -701,6 +749,7 @@ class BaseCore(object): return "<ok/>" def authenticate(self, cert, user, password, address): + """ Authenticate a client connection """ if self.ca: acert = cert else: @@ -712,7 +761,7 @@ class BaseCore(object): @exposed def GetDecisionList(self, address, mode): """Get the data of the decision list.""" - client, metadata = self.resolve_client(address) + metadata = self.resolve_client(address)[1] return self.GetDecisions(metadata, mode) @property diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 59d18fc57..bb1217f92 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -1,12 +1,15 @@ +""" check files for various required comments """ + import os import lxml.etree import Bcfg2.Server.Lint -from Bcfg2.Server import XI, XI_NAMESPACE -from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator import CfgPlaintextGenerator +from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ + import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML + class Comments(Bcfg2.Server.Lint.ServerPlugin): """ check files for various required headers """ def __init__(self, *args, **kwargs): @@ -22,10 +25,9 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"unexpanded-keywords":"warning", - "keywords-not-found":"warning", - "comments-not-found":"warning", - "broken-xinclude-chain":"warning"} + return {"unexpanded-keywords": "warning", + "keywords-not-found": "warning", + "comments-not-found": "warning"} def required_keywords(self, rtype): """ given a file type, fetch the list of required VCS keywords @@ -69,7 +71,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): xdata = lxml.etree.XML(bundle.data) rtype = "bundler" except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = lxml.etree.parse(bundle.template.filepath).getroot() + xdata = \ + lxml.etree.parse(bundle.template.filepath).getroot() rtype = "sgenshi" self.check_xml(bundle.name, xdata, rtype) @@ -153,7 +156,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): if status is None] if unexpanded: self.LintError("unexpanded-keywords", - "%s: Required keywords(s) found but not expanded: %s" % + "%s: Required keywords(s) found but not " + "expanded: %s" % (filename, ", ".join(unexpanded))) missing = [keyword for (keyword, status) in found.items() if status is False] @@ -177,21 +181,3 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.LintError("comments-not-found", "%s: Required comments(s) not found: %s" % (filename, ", ".join(missing))) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./%sinclude' % XI_NAMESPACE): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True - diff --git a/src/lib/Bcfg2/Server/Lint/Duplicates.py b/src/lib/Bcfg2/Server/Lint/Duplicates.py index 60a02ffb9..9b36f054b 100644 --- a/src/lib/Bcfg2/Server/Lint/Duplicates.py +++ b/src/lib/Bcfg2/Server/Lint/Duplicates.py @@ -1,10 +1,11 @@ -import os -import lxml.etree +""" Find duplicate clients, groups, etc. """ + import Bcfg2.Server.Lint -from Bcfg2.Server import XI, XI_NAMESPACE + class Duplicates(Bcfg2.Server.Lint.ServerPlugin): """ Find duplicate clients, groups, etc. """ + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.groups_xdata = None @@ -25,11 +26,10 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"broken-xinclude-chain":"warning", - "duplicate-client":"error", - "duplicate-group":"error", - "duplicate-package":"error", - "multiple-default-groups":"error"} + return {"duplicate-client": "error", + "duplicate-group": "error", + "duplicate-package": "error", + "multiple-default-groups": "error"} def load_xdata(self): """ attempt to load XML data for groups and clients. only @@ -71,20 +71,3 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): self.LintError("multiple-default-groups", "Multiple default groups defined: %s" % ",".join(default_groups)) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./%sinclude' % XI_NAMESPACE): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index 74142b446..18b4ae28a 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,9 +1,13 @@ +""" Check Genshi templates for syntax errors """ + import sys import genshi.template import Bcfg2.Server.Lint + class Genshi(Bcfg2.Server.Lint.ServerPlugin): """ Check Genshi templates for syntax errors """ + def Run(self): """ run plugin """ loader = genshi.template.TemplateLoader() @@ -14,9 +18,11 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"genshi-syntax-error":"error"} + return {"genshi-syntax-error": "error"} def check_files(self, entries, loader=None): + """ Check genshi templates in a list of entries for syntax + errors """ if loader is None: loader = genshi.template.TemplateLoader() diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index 5df98a30e..52e42aa7b 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -1,11 +1,14 @@ +""" ensure that all named groups are valid group names """ + import os import re import Bcfg2.Server.Lint try: from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile - has_genshi = True + HAS_GENSHI = True except ImportError: - has_genshi = False + HAS_GENSHI = False + class GroupNames(Bcfg2.Server.Lint.ServerPlugin): """ ensure that all named groups are valid group names """ @@ -28,6 +31,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): return {"invalid-group-name": "error"} def check_rules(self): + """ Check groups used in the Rules plugin for validity """ for rules in self.core.plugins['Rules'].entries.values(): if not self.HandlesFile(rules.name): continue @@ -36,20 +40,23 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): os.path.join(self.config['repo'], rules.name)) def check_bundles(self): - """ check bundles for BoundPath entries with missing attrs """ + """ Check groups used in the Bundler plugin for validity """ for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and - (not has_genshi or + (not HAS_GENSHI or not isinstance(bundle, BundleTemplateFile))): self.check_entries(bundle.xdata.xpath("//Group"), bundle.name) def check_metadata(self): + """ Check groups used or declared in the Metadata plugin for + validity """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), os.path.join(self.config['repo'], self.metadata.groups_xml.name)) def check_grouppatterns(self): + """ Check groups used in the GroupPatterns plugin for validity """ cfg = self.core.plugins['GroupPatterns'].config if not self.HandlesFile(cfg.name): return @@ -60,7 +67,9 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): (cfg.name, self.RenderXML(grp, keep_text=True))) def check_cfg(self): - for root, dirs, files in os.walk(self.core.plugins['Cfg'].data): + """ Check groups used in group-specific files in the Cfg + plugin for validity """ + for root, _, files in os.walk(self.core.plugins['Cfg'].data): for fname in files: basename = os.path.basename(root) if (re.search(r'^%s\.G\d\d_' % basename, fname) and @@ -71,6 +80,8 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): os.path.join(root, fname)) def check_entries(self, entries, fname): + """ Check a generic list of XML entries for <Group> tags with + invalid name attributes """ for grp in entries: if not self.valid.search(grp.get("name")): self.LintError("invalid-group-name", diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 5e4e21e18..e34f387ff 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -1,9 +1,12 @@ +""" ensure that all config files have an info.xml file""" + import os import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML from Bcfg2.Server.Plugins.Cfg.CfgLegacyInfo import CfgLegacyInfo + class InfoXML(Bcfg2.Server.Lint.ServerPlugin): """ ensure that all config files have an info.xml file""" def Run(self): @@ -34,13 +37,14 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"no-infoxml":"warning", - "deprecated-info-file":"warning", - "paranoid-false":"warning", - "broken-xinclude-chain":"warning", - "required-infoxml-attrs-missing":"error"} + return {"no-infoxml": "warning", + "deprecated-info-file": "warning", + "paranoid-false": "warning", + "broken-xinclude-chain": "warning", + "required-infoxml-attrs-missing": "error"} def check_infoxml(self, fname, xdata): + """ verify that info.xml contains everything it should """ for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: @@ -50,7 +54,8 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if missing: self.LintError("required-infoxml-attrs-missing", "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), fname, self.RenderXML(info))) + (",".join(missing), fname, + self.RenderXML(info))) if ((Bcfg2.Options.MDATA_PARANOID.value and info.get("paranoid") is not None and @@ -61,4 +66,3 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): self.LintError("paranoid-false", "Paranoid must be true in %s:%s" % (fname, self.RenderXML(info))) - diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 68d010316..44d02c2ff 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -1,9 +1,13 @@ +""" find Probes or Cfg files with multiple similar files that might be +merged into one """ + import os import copy from difflib import SequenceMatcher import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg import CfgGenerator + class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ find Probes or Cfg files with multiple similar files that might be merged into one """ @@ -15,11 +19,11 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"merge-cfg":"warning", - "merge-probes":"warning"} - + return {"merge-cfg": "warning", + "merge-probes": "warning"} def check_cfg(self): + """ check Cfg for similar files """ for filename, entryset in self.core.plugins['Cfg'].entries.items(): candidates = dict([(f, e) for f, e in entryset.entries.items() if isinstance(e, CfgGenerator)]) @@ -32,6 +36,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): for p in mset])) def check_probes(self): + """ check Probes for similar files """ probes = self.core.plugins['Probes'].probes.entries for mset in self.get_similar(probes): self.LintError("merge-probes", @@ -40,6 +45,9 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): ", ".join([p for p in mset])) def get_similar(self, entries): + """ Get a list of similar files from the entry dict. Return + value is a list of lists, each of which gives the filenames of + similar files """ if "threshold" in self.config: # accept threshold either as a percent (e.g., "threshold=75") or # as a ratio (e.g., "threshold=.75") @@ -61,17 +69,20 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): return rv def _find_similar(self, ftuple, others, threshold): + """ Find files similar to the one described by ftupe in the + list of other files. ftuple is a tuple of (filename, data); + others is a list of such tuples. threshold is a float between + 0 and 1 that describes how similar two files much be to rate + as 'similar' """ fname, fdata = ftuple rv = [fname] while others: cname, cdata = others.pop(0) - sm = SequenceMatcher(None, fdata.data, cdata.data) + seqmatch = SequenceMatcher(None, fdata.data, cdata.data) # perform progressively more expensive comparisons - if (sm.real_quick_ratio() > threshold and - sm.quick_ratio() > threshold and - sm.ratio() > threshold): + if (seqmatch.real_quick_ratio() > threshold and + seqmatch.quick_ratio() > threshold and + seqmatch.ratio() > threshold): rv.extend(self._find_similar((cname, cdata), copy.copy(others), threshold)) return rv - - diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index b9d5d79c4..299d6a246 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -1,3 +1,6 @@ +""" verify attributes for configuration entries that cannot be +verified with an XML schema alone""" + import os import re import lxml.etree @@ -7,41 +10,51 @@ import Bcfg2.Client.Tools.VCS from Bcfg2.Server.Plugins.Packages import Apt, Yum try: from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile - has_genshi = True + HAS_GENSHI = True except ImportError: - has_genshi = False + HAS_GENSHI = False + # format verifying functions def is_filename(val): + """ Return True if val is a string describing a valid full path + """ return val.startswith("/") and len(val) > 1 -def is_relative_filename(val): - return len(val) > 1 def is_selinux_type(val): + """ Return True if val is a string describing a valid (although + not necessarily existent) SELinux type """ return re.match(r'^[a-z_]+_t', val) + def is_selinux_user(val): + """ Return True if val is a string describing a valid (although + not necessarily existent) SELinux user """ return re.match(r'^[a-z_]+_u', val) + def is_octal_mode(val): + """ Return True if val is a string describing a valid octal + permissions mode """ return re.match(r'[0-7]{3,4}', val) + def is_username(val): + """ Return True if val is a string giving either a positive + integer uid, or a valid Unix username """ return re.match(r'^([a-z]\w{0,30}|\d+)$', val) + def is_device_mode(val): - try: - # checking upper bound seems like a good way to discover some - # obscure OS with >8-bit device numbers - return int(val) > 0 - except: - return False + """ Return True if val is a string describing a positive integer + """ + return re.match(r'^\d+$', val) class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): - """ verify attributes for configuration entries (as defined in - doc/server/configurationentries) """ + """ verify attributes for configuration entries that cannot be + verified with an XML schema alone """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = dict( @@ -56,7 +69,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): group=is_username, perms=is_octal_mode, __text__=None), hardlink=dict(name=is_filename, to=is_filename), - symlink=dict(name=is_filename, to=is_relative_filename), + symlink=dict(name=is_filename), ignore=dict(name=is_filename), nonexistent=dict(name=is_filename), permissions=dict(name=is_filename, owner=is_username, @@ -83,7 +96,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): access=dict(scope=lambda v: v in ['user', 'group'], perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', v)), - mask=dict(perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', v))), + mask=dict(perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', + v))), Package={None: dict(name=None)}, SELinux=dict( boolean=dict(name=None, @@ -163,15 +177,17 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): if 'Bundler' in self.core.plugins: for bundle in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(bundle.name) and - (not has_genshi or + (not HAS_GENSHI or not isinstance(bundle, BundleTemplateFile))): try: xdata = lxml.etree.XML(bundle.data) except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = \ - lxml.etree.parse(bundle.template.filepath).getroot() + xdata = lxml.etree.parse( + bundle.template.filepath).getroot() - for path in xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): + for path in xdata.xpath( + "//*[substring(name(), 1, 5) = 'Bound']" + ): self.check_entry(path, bundle.name) def check_entry(self, entry, filename): @@ -219,14 +235,15 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): self.RenderXML(entry))) if not attrs.issuperset(required_attrs.keys()): - self.LintError("required-attrs-missing", - "The following required attribute(s) are " - "missing for %s %s in %s: %s\n%s" % - (tag, name, filename, - ", ".join([attr - for attr in - set(required_attrs.keys()).difference(attrs)]), - self.RenderXML(entry))) + self.LintError( + "required-attrs-missing", + "The following required attribute(s) are missing for %s " + "%s in %s: %s\n%s" % + (tag, name, filename, + ", ".join([attr + for attr in + set(required_attrs.keys()).difference(attrs)]), + self.RenderXML(entry))) for attr, fmt in required_attrs.items(): if fmt and attr in attrs and not fmt(entry.attrib[attr]): @@ -235,4 +252,3 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "malformed\n%s" % (attr, tag, name, filename, self.RenderXML(entry))) - diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index e4c4bddd0..d6fd1df0c 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -1,34 +1,38 @@ +""" Ensure that the repo validates """ + import os import sys import glob import fnmatch import lxml.etree from subprocess import Popen, PIPE, STDOUT -from Bcfg2.Server import XI, XI_NAMESPACE +from Bcfg2.Server import XI_NAMESPACE import Bcfg2.Server.Lint + class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that the repo validates """ def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) - self.filesets = {"metadata:groups":"%s/metadata.xsd", - "metadata:clients":"%s/clients.xsd", - "info":"%s/info.xsd", - "%s/Bundler/*.xml":"%s/bundle.xsd", - "%s/Bundler/*.genshi":"%s/bundle.xsd", - "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", - "%s/Base/*.xml":"%s/base.xsd", - "%s/Rules/*.xml":"%s/rules.xsd", - "%s/Defaults/*.xml":"%s/defaults.xsd", - "%s/etc/report-configuration.xml":"%s/report-configuration.xsd", - "%s/Deps/*.xml":"%s/deps.xsd", - "%s/Decisions/*.xml":"%s/decisions.xsd", - "%s/Packages/sources.xml":"%s/packages.xsd", - "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", - "%s/FileProbes/config.xml":"%s/fileprobes.xsd", - } + self.filesets = \ + {"metadata:groups": "%s/metadata.xsd", + "metadata:clients": "%s/clients.xsd", + "info": "%s/info.xsd", + "%s/Bundler/*.xml": "%s/bundle.xsd", + "%s/Bundler/*.genshi": "%s/bundle.xsd", + "%s/Pkgmgr/*.xml": "%s/pkglist.xsd", + "%s/Base/*.xml": "%s/base.xsd", + "%s/Rules/*.xml": "%s/rules.xsd", + "%s/Defaults/*.xml": "%s/defaults.xsd", + "%s/etc/report-configuration.xml": "%s/report-configuration.xsd", + "%s/Deps/*.xml": "%s/deps.xsd", + "%s/Decisions/*.xml": "%s/decisions.xsd", + "%s/Packages/sources.xml": "%s/packages.xsd", + "%s/GroupPatterns/config.xml": "%s/grouppatterns.xsd", + "%s/NagiosGen/config.xml": "%s/nagiosgen.xsd", + "%s/FileProbes/config.xml": "%s/fileprobes.xsd", + } self.filelists = {} self.get_filelists() @@ -54,13 +58,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): @classmethod def Errors(cls): - return {"broken-xinclude-chain":"warning", - "schema-failed-to-parse":"warning", - "properties-schema-not-found":"warning", - "xml-failed-to-parse":"error", - "xml-failed-to-read":"error", - "xml-failed-to-verify":"error", - "input-output-error":"error"} + return {"broken-xinclude-chain": "warning", + "schema-failed-to-parse": "warning", + "properties-schema-not-found": "warning", + "xml-failed-to-parse": "error", + "xml-failed-to-read": "error", + "xml-failed-to-verify": "error", + "input-output-error": "error"} def check_properties(self): """ check Properties files against their schemas """ @@ -124,12 +128,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filelists[path] = \ [f for f in self.files if os.path.basename(f) == 'info.xml'] - else: # self.files is None + else: # self.files is None self.filelists[path] = [] - for infodir in ['Cfg', 'TGenshi', 'TCheetah']: - for root, dirs, files in os.walk('%s/%s' % - (self.config['repo'], - infodir)): + for infodir in ['Cfg', 'SSHbase', 'SSLCA', 'TGenshi', + 'TCheetah']: + for root, _, files in \ + os.walk(os.path.join(self.config['repo'], + infodir)): self.filelists[path].extend([os.path.join(root, f) for f in files if f == 'info.xml']) @@ -146,7 +151,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if (fname not in self.filelists['metadata:groups'] and fname not in self.filelists['metadata:clients']): self.LintError("broken-xinclude-chain", - "Broken XInclude chain: Could not determine file type of %s" % fname) + "Broken XInclude chain: Could not determine " + "file type of %s" % fname) def get_metadata_list(self, mtype): """ get all metadata files for the specified type (clients or @@ -163,11 +169,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: rv.extend(self.follow_xinclude(rv[0])) except lxml.etree.XMLSyntaxError: - e = sys.exc_info()[1] + err = sys.exc_info()[1] self.LintError("xml-failed-to-parse", - "%s fails to parse:\n%s" % (rv[0], e)) - - + "%s fails to parse:\n%s" % (rv[0], err)) return rv def follow_xinclude(self, xfile): @@ -193,21 +197,22 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): elif self.HandlesFile(path): rv.append(path) groupdata = lxml.etree.parse(path) - included.update(el for el in groupdata.findall('./%sinclude' % + included.update(el for el in groupdata.findall('./%sinclude' % XI_NAMESPACE)) included.discard(filename) return rv def _load_schema(self, filename): + """ load an XML schema document, returning the Schema object """ try: return lxml.etree.XMLSchema(lxml.etree.parse(filename)) except IOError: - e = sys.exc_info()[1] - self.LintError("input-output-error", str(e)) + err = sys.exc_info()[1] + self.LintError("input-output-error", str(err)) except lxml.etree.XMLSchemaParseError: - e = sys.exc_info()[1] + err = sys.exc_info()[1] self.LintError("schema-failed-to-parse", "Failed to process schema %s: %s" % - (filename, e)) + (filename, err)) return None diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index f4a9b74c2..eea205b75 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -1,51 +1,47 @@ -__all__ = ['Bundler', - 'Comments', - 'Duplicates', - 'InfoXML', - 'MergeFiles', - 'Pkgmgr', - 'RequiredAttrs', - 'Validate', - 'Genshi'] +""" Base classes for Lint plugins and error handling """ -import logging import os import sys +import logging from copy import copy import textwrap import lxml.etree -import Bcfg2.Logger import fcntl import termios import struct +from Bcfg2.Server import XI_NAMESPACE -def _ioctl_GWINSZ(fd): + +def _ioctl_GWINSZ(fd): # pylint: disable=C0103 + """ get a tuple of (height, width) giving the size of the window + from the given file descriptor """ try: - cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - except: + return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + except: # pylint: disable=W0702 return None - return cr + def get_termsize(): """ get a tuple of (width, height) giving the size of the terminal """ if not sys.stdout.isatty(): return None - cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) - if not cr: + dims = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) + if not dims: try: fd = os.open(os.ctermid(), os.O_RDONLY) - cr = _ioctl_GWINSZ(fd) + dims = _ioctl_GWINSZ(fd) os.close(fd) - except: + except: # pylint: disable=W0702 pass - if not cr: + if not dims: try: - cr = (os.environ['LINES'], os.environ['COLUMNS']) + dims = (os.environ['LINES'], os.environ['COLUMNS']) except KeyError: return None - return int(cr[1]), int(cr[0]) + return int(dims[1]), int(dims[0]) -class Plugin (object): + +class Plugin(object): """ base class for ServerlessPlugin and ServerPlugin """ def __init__(self, config, errorhandler=None, files=None): @@ -78,6 +74,7 @@ class Plugin (object): fname)) in self.files) def LintError(self, err, msg): + """ record an error in the lint process """ self.errorhandler.dispatch(err, msg) def RenderXML(self, element, keep_text=False): @@ -88,16 +85,20 @@ class Plugin (object): el = copy(element) if el.text and not keep_text: el.text = '...' - [el.remove(c) for c in el.iterchildren()] - xml = lxml.etree.tostring(el, - xml_declaration=False).decode("UTF-8").strip() + for child in el.iterchildren(): + el.remove(child) + xml = lxml.etree.tostring( + el, + xml_declaration=False).decode("UTF-8").strip() else: - xml = lxml.etree.tostring(element, - xml_declaration=False).decode("UTF-8").strip() + xml = lxml.etree.tostring( + element, + xml_declaration=False).decode("UTF-8").strip() return " line %s: %s" % (element.sourceline, xml) class ErrorHandler (object): + """ a class to handle errors for bcfg2-lint plugins """ def __init__(self, config=None): self.errors = 0 self.warnings = 0 @@ -124,6 +125,8 @@ class ErrorHandler (object): self._handlers[err] = self.debug def RegisterErrors(self, errors): + """ Register a dict of errors (name: default level) that a + plugin may raise """ for err, action in errors.items(): if err not in self._handlers: if "warn" in action: @@ -132,8 +135,9 @@ class ErrorHandler (object): self._handlers[err] = self.error else: self._handlers[err] = self.debug - + def dispatch(self, err, msg): + """ Dispatch an error to the correct handler """ if err in self._handlers: self._handlers[err](msg) self.logger.debug(" (%s)" % err) @@ -157,6 +161,8 @@ class ErrorHandler (object): self._log(msg, self.logger.debug) def _log(self, msg, logfunc, prefix=""): + """ Generic log function that logs a message with the given + function after wrapping it for the terminal width """ # a message may itself consist of multiple lines. wrap() will # elide them all into a single paragraph, which we don't want. # so we split the message into its paragraphs and wrap each @@ -186,8 +192,27 @@ class ServerlessPlugin (Plugin): class ServerPlugin (Plugin): """ base class for plugins that check things that require the running Bcfg2 server """ - def __init__(self, lintCore, config, **kwargs): + def __init__(self, core, config, **kwargs): Plugin.__init__(self, config, **kwargs) - self.core = lintCore + self.core = core self.logger = self.core.logger self.metadata = self.core.metadata + self.errorhandler.RegisterErrors({"broken-xinclude-chain": "warning"}) + + def has_all_xincludes(self, mfile): + """ return true if self.files includes all XIncludes listed in + the specified metadata type, false otherwise""" + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./%sinclude' % XI_NAMESPACE): + if not self.has_all_xincludes(el.get('href')): + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not " + "include %s" % path) + return False + + return True diff --git a/testsuite/Testsrc/testmisc.py b/testsuite/Testsrc/testmisc.py index 5f84d00bf..3ea80310e 100644 --- a/testsuite/Testsrc/testmisc.py +++ b/testsuite/Testsrc/testmisc.py @@ -38,7 +38,11 @@ class TestPylint(Bcfg2TestCase): # <directory> => <file globs within that directory>. <directory> # is relative to src/ whitelist = { - "lib/Bcfg2/Server": ["Plugin"], + "lib/Bcfg2/Server": ["Lint", + "Plugin", + "BuiltinCore.py", + "CherryPyCore.py", + "Core.py"], "lib/Bcfg2/Server/Plugins": ["PuppetENC.py", "Rules.py", "DBStats.py", @@ -60,7 +64,8 @@ class TestPylint(Bcfg2TestCase): "Svn2.py", "Bzr.py", "Cfg", - "Packages"] + "Packages"], + "lib/Bcfg2/Client/Tools": ["POSIX"], } pylint_cmd = ["pylint", "--rcfile", rcfile] diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 29e35135c..014a94de5 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -33,7 +33,7 @@ load-plugins= # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). -disable=W0142,W0511,W0603,R0201,R0901,R0903,R0904,R0921,R0922,C0302,I0011 +disable=W0142,W0511,W0603,R0201,R0901,R0902,R0903,R0904,R0921,R0922,C0302,I0011 [REPORTS] @@ -162,14 +162,14 @@ attr-rgx=(Entries|[a-z_][a-z0-9_]{2,30})$ argument-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct variable names -variable-rgx=(rv|el|fd|[a-z_][a-z0-9_]{2,30})$ +variable-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct list comprehension / # generator expression variable names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +good-names=_,rv,el,fd,ca # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata |