From b862090945322d5ba4b42e180bba92afb860df21 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Aug 2012 09:06:43 -0400 Subject: POSIX: refactored POSIX tool into multiple files to make it more manageable Added unit tests for POSIX tool and sub-tools fixed ACL handling for filesystems mounted noacl --- src/lib/Bcfg2/Client/Tools/POSIX/Device.py | 62 +++ src/lib/Bcfg2/Client/Tools/POSIX/Directory.py | 86 ++++ src/lib/Bcfg2/Client/Tools/POSIX/File.py | 219 ++++++++ src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py | 39 ++ src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py | 41 ++ src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py | 7 + src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py | 42 ++ src/lib/Bcfg2/Client/Tools/POSIX/__init__.py | 147 ++++++ src/lib/Bcfg2/Client/Tools/POSIX/base.py | 639 ++++++++++++++++++++++++ 9 files changed, 1282 insertions(+) create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/Device.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/Directory.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/File.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/__init__.py create mode 100644 src/lib/Bcfg2/Client/Tools/POSIX/base.py (limited to 'src/lib/Bcfg2/Client/Tools/POSIX') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py new file mode 100644 index 000000000..b8fb0f4d0 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py @@ -0,0 +1,62 @@ +import os +import sys +from base import POSIXTool, device_map + +class POSIXDevice(POSIXTool): + __req__ = ['name', 'dev_type', 'perms', 'owner', 'group'] + + def fully_specified(self, entry): + if entry.get('dev_type') in ['block', 'char']: + # check if major/minor are properly specified + if (entry.get('major') == None or + entry.get('minor') == None): + return False + return True + + def verify(self, entry, modlist): + """Verify device entry.""" + ondisk = self._exists(entry) + if not ondisk: + return False + + # attempt to verify device properties as specified in config + rv = True + dev_type = entry.get('dev_type') + if dev_type in ['block', 'char']: + major = int(entry.get('major')) + minor = int(entry.get('minor')) + if major != os.major(ondisk.st_rdev): + msg = ("Major number for device %s is incorrect. " + "Current major is %s but should be %s" % + (entry.get("name"), os.major(ondisk.st_rdev), major)) + self.logger.debug('POSIX: ' + msg) + entry.set('qtext', entry.get('qtext', '') + "\n" + msg) + rv = False + + if minor != os.minor(ondisk.st_rdev): + msg = ("Minor number for device %s is incorrect. " + "Current minor is %s but should be %s" % + (entry.get("name"), os.minor(ondisk.st_rdev), minor)) + self.logger.debug('POSIX: ' + msg) + entry.set('qtext', entry.get('qtext', '') + "\n" + msg) + rv = False + return POSIXTool.verify(self, entry, modlist) and rv + + def install(self, entry): + if not self._exists(entry, remove=True): + try: + dev_type = entry.get('dev_type') + mode = device_map[dev_type] | int(entry.get('perms'), 8) + if dev_type in ['block', 'char']: + major = int(entry.get('major')) + minor = int(entry.get('minor')) + device = os.makedev(major, minor) + os.mknod(entry.get('name'), mode, device) + else: + os.mknod(entry.get('name'), mode) + except (KeyError, OSError, ValueError): + err = sys.exc_info()[1] + self.logger.error('POSIX: Failed to install %s: %s' % + (entry.get('name'), err)) + return False + return POSIXTool.install(self, entry) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py new file mode 100644 index 000000000..4b0ad93ef --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py @@ -0,0 +1,86 @@ +import os +import sys +import stat +import shutil +import Bcfg2.Client.XML +from base import POSIXTool + +class POSIXDirectory(POSIXTool): + __req__ = ['name', 'perms', 'owner', 'group'] + + def verify(self, entry, modlist): + ondisk = self._exists(entry) + if not ondisk: + return False + + if not stat.S_ISDIR(ondisk[stat.ST_MODE]): + self.logger.info("POSIX: %s is not a directory" % entry.get('name')) + return False + + pruneTrue = True + if entry.get('prune', 'false').lower() == 'true': + # check for any extra entries when prune='true' attribute is set + try: + extras = [os.path.join(entry.get('name'), ent) + for ent in os.listdir(entry.get('name')) + if os.path.join(entry.get('name'), + ent) not in modlist] + if extras: + pruneTrue = False + msg = "Directory %s contains extra entries: %s" % \ + (entry.get('name'), "; ".join(extras)) + self.logger.info("POSIX: " + msg) + entry.set('qtext', entry.get('qtext', '') + '\n' + msg) + for extra in extras: + Bcfg2.Client.XML.SubElement(entry, 'Prune', path=extra) + except OSError: + pruneTrue = True + + return POSIXTool.verify(self, entry, modlist) and pruneTrue + + def install(self, entry): + """Install device entries.""" + fmode = self._exists(entry) + + if fmode and not stat.S_ISDIR(fmode[stat.ST_MODE]): + self.logger.info("POSIX: Found a non-directory entry at %s, " + "removing" % entry.get('name')) + try: + os.unlink(entry.get('name')) + fmode = False + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to unlink %s: %s" % + (entry.get('name'), err)) + return False + elif fmode: + self.logger.debug("POSIX: Found a pre-existing directory at %s" % + entry.get('name')) + + rv = True + if not fmode: + rv &= self._makedirs(entry) + + if entry.get('prune', 'false') == 'true': + ulfailed = False + for pent in entry.findall('Prune'): + pname = pent.get('path') + ulfailed = False + if os.path.isdir(pname): + rm = shutil.rmtree + else: + rm = os.unlink + try: + self.logger.debug("POSIX: Removing %s" % pname) + rm(pname) + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to unlink %s: %s" % + (pname, err)) + ulfailed = True + if ulfailed: + # even if prune failed, we still want to install the + # entry to make sure that we get permissions and + # whatnot set + rv = False + return POSIXTool.install(self, entry) and rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py new file mode 100644 index 000000000..73ed2d8bf --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -0,0 +1,219 @@ +import os +import sys +import stat +import time +import difflib +import binascii +import tempfile +from base import POSIXTool + +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str + +class POSIXFile(POSIXTool): + __req__ = ['name', 'perms', 'owner', 'group'] + + def fully_specified(self, entry): + return entry.text is not None or entry.get('empty', 'false') == 'true' + + def _is_string(self, strng, encoding): + """ Returns true if the string contains no ASCII control + characters and can be decoded from the specified encoding. """ + for char in strng: + if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: + return False + try: + strng.decode(encoding) + return True + except: + return False + + def _get_data(self, entry): + is_binary = False + if entry.get('encoding', 'ascii') == 'base64': + tempdata = binascii.a2b_base64(entry.text) + is_binary = True + elif entry.get('empty', 'false') == 'true': + tempdata = '' + else: + tempdata = entry.text + if isinstance(tempdata, unicode): + try: + tempdata = tempdata.encode(self.setup['encoding']) + except UnicodeEncodeError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Error encoding file %s: %s" % + (entry.get('name'), err)) + return (tempdata, is_binary) + + def verify(self, entry, modlist): + ondisk = self._exists(entry) + tempdata, is_binary = self._get_data(entry) + + different = False + content = None + if not ondisk: + # first, see if the target file exists at all; if not, + # they're clearly different + different = True + content = "" + elif len(tempdata) != ondisk[stat.ST_SIZE]: + # next, see if the size of the target file is different + # from the size of the desired content + different = True + else: + # finally, read in the target file and compare them + # directly. comparison could be done with a checksum, + # which might be faster for big binary files, but slower + # for everything else + try: + content = open(entry.get('name')).read() + except IOError: + self.logger.error("POSIX: Failed to read %s: %s" % + (entry.get("name"), sys.exc_info()[1])) + return False + different = content != tempdata + + if different: + self.logger.debug("POSIX: %s has incorrect contents" % + entry.get("name")) + self._get_diffs( + entry, interactive=self.setup['interactive'], + sensitive=entry.get('sensitive', 'false').lower() == 'true', + is_binary=is_binary, content=content) + return POSIXTool.verify(self, entry, modlist) and not different + + def _write_tmpfile(self, entry): + filedata, _ = self._get_data(entry) + # get a temp file to write to that is in the same directory as + # the existing file in order to preserve any permissions + # protections on that directory, and also to avoid issues with + # /tmp set nosetuid while creating files that are supposed to + # be setuid + try: + (newfd, newfile) = \ + tempfile.mkstemp(prefix=os.path.basename(entry.get("name")), + dir=os.path.dirname(entry.get("name"))) + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to create temp file in %s: %s" % + (os.path.dirname(entry.get('name')), err)) + return False + try: + os.fdopen(newfd, 'w').write(filedata) + except (OSError, IOError): + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to open temp file %s for writing " + "%s: %s" % + (newfile, entry.get("name"), err)) + return False + return newfile + + def _rename_tmpfile(self, newfile, entry): + try: + os.rename(newfile, entry.get('name')) + return True + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to rename temp file %s to %s: %s" % + (newfile, entry.get('name'), err)) + try: + os.unlink(newfile) + except: + err = sys.exc_info()[1] + self.logger.error("POSIX: Could not remove temp file %s: %s" % + (newfile, err)) + return False + + def install(self, entry): + """Install device entries.""" + if not os.path.exists(os.path.dirname(entry.get('name'))): + if not self._makedirs(entry, + path=os.path.dirname(entry.get('name'))): + return False + newfile = self._write_tmpfile(entry) + if not newfile: + return False + rv = self._set_perms(entry, path=newfile) + if not self._rename_tmpfile(newfile, entry): + return False + + return POSIXTool.install(self, entry) and rv + + def _get_diffs(entry, interactive=False, sensitive=False, is_binary=False, + content=None): + if not interactive and sensitive: + return + + prompt = [entry.get('qtext', '')] + attrs = dict() + if not is_binary and content is None: + # it's possible that we figured out the files are + # different without reading in the local file. if the + # supplied version of the file is not binary, we now have + # to read in the local file to figure out if _it_ is + # binary, and either include that fact or the diff in our + # prompts for -I and the reports + try: + content = open(entry.get('name')).read() + except IOError: + self.logger.error("POSIX: Failed to read %s: %s" % + (entry.get("name"), sys.exc_info()[1])) + return False + is_binary &= self._is_string(content, self.setup['encoding']) + if is_binary: + # don't compute diffs if the file is binary + prompt.append('Binary file, no printable diff') + attrs['current_bfile'] = binascii.b2a_base64(content) + else: + if interactive: + diff = self._diff(content, tempdata, + difflib.unified_diff, + filename=entry.get("name")) + if diff: + udiff = '\n'.join(diff) + try: + prompt.append(udiff.decode(self.setup['encoding'])) + except UnicodeEncodeError: + prompt.append("Could not encode diff") + else: + prompt.append("Diff took too long to compute, no " + "printable diff") + if not sensitive: + diff = self._diff(content, tempdata, difflib.ndiff, + filename=entry.get("name")) + if diff: + ndiff = binascii.b2a_base64("\n".join(diff)) + attrs["current_bdiff"] = ndiff + else: + attrs['current_bfile'] = binascii.b2a_base64(content) + if interactive: + entry.set("qtext", "\n".join(prompt)) + if not sensitive: + for attr, val in attrs: + entry.set(attr, val) + + def _diff(self, content1, content2, difffunc, filename=None): + rv = [] + start = time.time() + longtime = False + for diffline in difffunc(content1.split('\n'), + content2.split('\n')): + now = time.time() + rv.append(diffline) + if now - start > 5 and not longtime: + if filename: + self.logger.info("POSIX: Diff of %s taking a long time" % + filename) + else: + self.logger.info("POSIX: Diff taking a long time") + longtime = True + elif now - start > 30: + if filename: + self.logger.error("POSIX: Diff of %s took too long; giving " + "up" % filename) + else: + self.logger.error("POSIX: Diff took too long; giving up") + return False + return rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py new file mode 100644 index 000000000..569ca3445 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py @@ -0,0 +1,39 @@ +import os +import sys +from base import POSIXTool + +class POSIXHardlink(POSIXTool): + __req__ = ['name', 'to'] + + def verify(self, entry, modlist): + rv = True + + try: + if not os.path.samefile(entry.get('name'), entry.get('to')): + msg = "Hardlink %s is incorrect" % entry.get('name') + self.logger.debug("POSIX: " + msg) + entry.set('qtext', "\n".join([entry.get('qtext', ''), msg])) + rv = False + except OSError: + self.logger.debug("POSIX: %s %s does not exist" % + (entry.tag, entry.get("name"))) + entry.set('current_exists', 'false') + return False + + return POSIXTool.verify(self, entry, modlist) and rv + + def install(self, entry): + ondisk = self._exists(entry, remove=True) + if ondisk: + self.logger.info("POSIX: Hardlink %s cleanup failed" % + entry.get('name')) + try: + os.link(entry.get('to'), entry.get('name')) + rv = True + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to create hardlink %s to %s: %s" % + (entry.get('name'), entry.get('to'), err)) + rv = False + return POSIXTool.install(self, entry) and rv + diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py new file mode 100644 index 000000000..64a36cce4 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py @@ -0,0 +1,41 @@ +import os +import sys +import shutil +from base import POSIXTool + +class POSIXNonexistent(POSIXTool): + __req__ = ['name'] + + def verify(self, entry, _): + if os.path.lexists(entry.get('name')): + self.logger.debug("POSIX: %s exists but should not" % + entry.get("name")) + return False + return True + + def install(self, entry): + ename = entry.get('name') + if entry.get('recursive', '').lower() == 'true': + # ensure that configuration spec is consistent first + for struct in self.config.getchildren(): + for entry in struct.getchildren(): + if (entry.tag == 'Path' and + entry.get('type') != 'nonexistent' and + entry.get('name').startswith(ename)): + self.logger.error('POSIX: Not removing %s. One or ' + 'more files in this directory are ' + 'specified in your configuration.' % + ename) + return False + rm = shutil.rmtree + elif os.path.isdir(ename): + rm = os.rmdir + else: + rm = os.remove + try: + rm(ename) + return True + except OSError: + err = sys.exc_info()[1] + self.logger.error('POSIX: Failed to remove %s: %s' % (ename, err)) + return False diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py b/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py new file mode 100644 index 000000000..c041b9ade --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py @@ -0,0 +1,7 @@ +import os +import sys +from base import POSIXTool + +class POSIXPermissions(POSIXTool): + __req__ = ['name', 'perms', 'owner', 'group'] + diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py new file mode 100644 index 000000000..d5222513e --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py @@ -0,0 +1,42 @@ +import os +import sys +from base import POSIXTool + +class POSIXSymlink(POSIXTool): + __req__ = ['name', 'to'] + + def verify(self, entry, modlist): + rv = True + + try: + sloc = os.readlink(entry.get('name')) + if sloc != entry.get('to'): + entry.set('current_to', sloc) + msg = ("Symlink %s points to %s, should be %s" % + (entry.get('name'), sloc, entry.get('to'))) + self.logger.debug("POSIX: " + msg) + entry.set('qtext', "\n".join([entry.get('qtext', ''), msg])) + rv = False + except OSError: + self.logger.debug("POSIX: %s %s does not exist" % + (entry.tag, entry.get("name"))) + entry.set('current_exists', 'false') + return False + + return POSIXTool.verify(self, entry, modlist) and rv + + def install(self, entry): + ondisk = self._exists(entry, remove=True) + if ondisk: + self.logger.info("POSIX: Symlink %s cleanup failed" % + entry.get('name')) + try: + os.symlink(entry.get('to'), entry.get('name')) + rv = True + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to create symlink %s to %s: %s" % + (entry.get('name'), entry.get('to'), err)) + rv = False + return POSIXTool.install(self, entry) and rv + diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py new file mode 100644 index 000000000..7e649a2c1 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -0,0 +1,147 @@ +"""All POSIX Type client support for Bcfg2.""" + +import os +import re +import sys +import shutil +import pkgutil +from datetime import datetime +import Bcfg2.Client.Tools +from base import POSIXTool + +class POSIX(Bcfg2.Client.Tools.Tool): + """POSIX File support code.""" + name = 'POSIX' + + def __init__(self, logger, setup, config): + Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) + self.ppath = setup['ppath'] + self.max_copies = setup['max_copies'] + self._load_handlers() + self.logger.debug("POSIX: Handlers loaded: %s" % + (", ".join(self._handlers.keys()))) + self.__req__ = dict(Path=dict()) + for etype, hdlr in self._handlers.items(): + self.__req__['Path'][etype] = hdlr.__req__ + self.__handles__.append(('Path', etype)) + # Tool.__init__() sets up the list of handled entries, but we + # need to do it again after __handles__ has been populated. we + # can't populate __handles__ when the class is created because + # _load_handlers() _must_ be called at run-time, not at + # compile-time. + for struct in config: + self.handled = [e for e in struct if self.handlesEntry(e)] + + def _load_handlers(self): + # this must be called at run-time, not at compile-time, or we + # get wierd circular import issues. + self._handlers = dict() + if hasattr(pkgutil, 'walk_packages'): + submodules = pkgutil.walk_packages(path=__path__) + else: + # python 2.4 + import glob + submodules = [] + for path in __path__: + for submodule in glob.glob(os.path.join(path, "*.py")): + mod = os.path.splitext(os.path.basename(submodule))[0] + if mod not in ['__init__']: + submodules.append((None, mod, True)) + + for submodule in submodules: + if submodule[1] == 'base': + continue + module = getattr(__import__("%s.%s" % + (__name__, + submodule[1])).Client.Tools.POSIX, + submodule[1]) + hdlr = getattr(module, "POSIX" + submodule[1]) + if POSIXTool in hdlr.__mro__: + # figure out what entry type this handler handles + etype = hdlr.__name__[5:].lower() + self._handlers[etype] = hdlr(self.logger, + self.setup, + self.config) + + def canVerify(self, entry): + if not Bcfg2.Client.Tools.Tool.canVerify(self, entry): + return False + if not self._handlers[entry.get("type")].fully_specified(entry): + self.logger.error('POSIX: Cannot verify incomplete entry %s. ' + 'Try running bcfg2-lint.' % + entry.get('name')) + return False + return True + + def canInstall(self, entry): + """Check if entry is complete for installation.""" + if not Bcfg2.Client.Tools.Tool.canInstall(self, entry): + return False + if not self._handlers[entry.get("type")].fully_specified(entry): + self.logger.error('POSIX: Cannot install incomplete entry %s. ' + 'Try running bcfg2-lint.' % + entry.get('name')) + return False + return True + + def InstallPath(self, entry): + """Dispatch install to the proper method according to type""" + self.logger.debug("POSIX: Installing entry %s:%s:%s" % + (entry.tag, entry.get("type"), entry.get("name"))) + self._paranoid_backup(entry) + return self._handlers[entry.get("type")].install(entry) + + def VerifyPath(self, entry, modlist): + """Dispatch verify to the proper method according to type""" + self.logger.debug("POSIX: Verifying entry %s:%s:%s" % + (entry.tag, entry.get("type"), entry.get("name"))) + ret = self._handlers[entry.get("type")].verify(entry, modlist) + if self.setup['interactive'] and not ret: + entry.set('qtext', + '%s\nInstall %s %s: (y/N) ' % + (entry.get('qtext', ''), + entry.get('type'), entry.get('name'))) + return ret + + def _prune_old_backups(self, entry): + bkupnam = entry.get('name').replace('/', '_') + bkup_re = re.compile(bkupnam + \ + r'_\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}$') + # current list of backups for this file + try: + bkuplist = [f for f in os.listdir(self.ppath) if + bkup_re.match(f)] + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to create backup list in %s: %s" % + (self.ppath, err)) + return + bkuplist.sort() + while len(bkuplist) >= int(self.max_copies): + # remove the oldest backup available + oldest = bkuplist.pop(0) + self.logger.info("POSIX: Removing old backup %s" % oldest) + try: + os.remove(os.path.join(self.ppath, oldest)) + except OSError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to remove old backup %s: %s" % + (os.path.join(self.ppath, oldest), err)) + + def _paranoid_backup(self, entry): + if (entry.get("paranoid", 'false').lower() == 'true' and + self.setup.get("paranoid", False) and + entry.get('current_exists', 'true') == 'true' and + not os.path.isdir(entry.get("name"))): + self._prune_old_backups(entry) + bkupnam = "%s_%s" % (entry.get('name').replace('/', '_'), + datetime.isoformat(datetime.now())) + bfile = os.path.join(self.ppath, bkupnam) + try: + shutil.copy(entry.get('name'), bfile) + self.logger.info("POSIX: Backup of %s saved to %s" % + (entry.get('name'), bfile)) + except IOError: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to create backup file for %s: " + "%s" % (entry.get('name'), err)) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py new file mode 100644 index 000000000..1ec1d36d5 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -0,0 +1,639 @@ +import os +import sys +import pwd +import grp +import stat +import shutil +import Bcfg2.Client.Tools +import Bcfg2.Client.XML + +try: + import selinux + has_selinux = True +except ImportError: + has_selinux = False + +try: + import posix1e + has_acls = True +except ImportError: + has_acls = False + +# map between dev_type attribute and stat constants +device_map = dict(block=stat.S_IFBLK, + char=stat.S_IFCHR, + fifo=stat.S_IFIFO) + +# map between permissions characters and numeric ACL constants +acl_map = dict(r=posix1e.ACL_READ, + w=posix1e.ACL_WRITE, + x=posix1e.ACL_EXECUTE) + +class POSIXTool(Bcfg2.Client.Tools.Tool): + def fully_specified(self, entry): + # checking is done by __req__ + return True + + def verify(self, entry, modlist): + if not self._verify_metadata(entry): + return False + if entry.get('recursive', 'false').lower() == 'true': + # verify ownership information recursively + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + if not self._verify_metadata(entry, + path=os.path.join(root, p)): + return False + return True + + def install(self, entry): + plist = [entry.get('name')] + rv = True + rv &= self._set_perms(entry) + if entry.get('recursive', 'false').lower() == 'true': + # set metadata recursively + for root, dirs, files in os.walk(entry.get('name')): + for path in dirs + files: + rv &= self._set_perms(entry, path=os.path.join(root, path)) + return rv + + def _exists(self, entry, remove=False): + try: + # check for existing paths and optionally remove them + ondisk = os.lstat(entry.get('name')) + if remove: + if os.path.isdir(entry.get('name')): + rm = shutil.rmtree + else: + rm = os.unlink + try: + rm(entry.get('name')) + return False + except OSError: + err = sys.exc_info()[1] + self.logger.warning('POSIX: Failed to unlink %s: %s' % + (entry.get('name'), err)) + return ondisk # probably still exists + else: + return ondisk + except OSError: + return False + + def _set_perms(self, entry, path=None): + if path is None: + path = entry.get("name") + + rv = True + if entry.get("owner") and entry.get("group"): + try: + self.logger.debug("POSIX: Setting ownership of %s to %s:%s" % + (path, + self._norm_entry_uid(entry), + self._norm_entry_gid(entry))) + os.chown(path, self._norm_entry_uid(entry), + self._norm_entry_gid(entry)) + except KeyError: + self.logger.error('POSIX: Failed to change ownership of %s' % + path) + rv = False + os.chown(path, 0, 0) + except OSError: + self.logger.error('POSIX: Failed to change ownership of %s' % + path) + rv = False + + if entry.get("perms"): + configPerms = int(entry.get('perms'), 8) + if entry.get('dev_type'): + configPerms |= device_map[entry.get('dev_type')] + try: + self.logger.debug("POSIX: Setting permissions on %s to %s" % + (path, oct(configPerms))) + os.chmod(path, configPerms) + except (OSError, KeyError): + self.logger.error('POSIX: Failed to change permissions on %s' % + path) + rv = False + + if entry.get('mtime'): + try: + os.utime(entry.get('name'), (int(entry.get('mtime')), + int(entry.get('mtime')))) + except OSError: + self.logger.error("POSIX: Failed to set mtime of %s" % path) + rv = False + + rv &= self._set_secontext(entry, path=path) + rv &= self._set_acls(entry, path=path) + return rv + + + def _set_acls(self, entry, path=None): + """ set POSIX ACLs on the file on disk according to the config """ + if not has_acls: + if entry.findall("ACL"): + self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " + "library installed" % entry.get('name')) + return True + + if path is None: + path = entry.get("name") + + try: + acl = posix1e.ACL(file=path) + except IOError: + err = sys.exc_info()[1] + if err.errno == 95: + # fs is mounted noacl + self.logger.error("POSIX: Cannot set ACLs on filesystem " + "mounted without ACL support: %s" % path) + else: + self.logger.error("POSIX: Error getting current ACLS on %s: %s" + % (path, err)) + return False + # clear ACLs out so we start fresh -- way easier than trying + # to add/remove/modify ACLs + for aclentry in acl: + if aclentry.tag_type in [posix1e.ACL_USER, posix1e.ACL_GROUP]: + acl.delete_entry(aclentry) + if os.path.isdir(path): + defacl = posix1e.ACL(filedef=path) + if not defacl.valid(): + # when a default ACL is queried on a directory that + # has no default ACL entries at all, you get an empty + # ACL, which is not valid. in this circumstance, we + # just copy the access ACL to get a base valid ACL + # that we can add things to. + defacl = posix1e.ACL(acl=acl) + else: + for aclentry in defacl: + if aclentry.tag_type in [posix1e.ACL_USER, + posix1e.ACL_GROUP]: + defacl.delete_entry(aclentry) + else: + defacl = None + + for aclkey, perms in self._list_entry_acls(entry).items(): + atype, scope, qualifier = aclkey + if atype == "default": + if defacl is None: + self.logger.warning("POSIX: Cannot set default ACLs on " + "non-directory %s" % path) + continue + entry = posix1e.Entry(defacl) + else: + entry = posix1e.Entry(acl) + for perm in acl_map.values(): + if perm & perms: + entry.permset.add(perm) + entry.tag_type = scope + try: + if scope == posix1e.ACL_USER: + scopename = "user" + entry.qualifier = self._norm_uid(qualifier) + elif scope == posix1e.ACL_GROUP: + scopename = "group" + entry.qualifier = self._norm_gid(qualifier) + except (OSError, KeyError): + err = sys.exc_info()[1] + self.logger.error("POSIX: Could not resolve %s %s: %s" % + (scopename, qualifier, err)) + continue + acl.calc_mask() + + def _apply_acl(acl, path, atype=posix1e.ACL_TYPE_ACCESS): + if atype == posix1e.ACL_TYPE_ACCESS: + atype_str = "access" + else: + atype_str = "default" + if acl.valid(): + self.logger.debug("POSIX: Applying %s ACL to %s:" % (atype_str, + path)) + for line in str(acl).splitlines(): + self.logger.debug(" " + line) + try: + acl.applyto(path, atype) + return True + except: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to set ACLs on %s: %s" % + (path, err)) + return False + else: + self.logger.warning("POSIX: %s ACL created for %s was invalid:" + % (atype_str.title(), path)) + for line in str(acl).splitlines(): + self.logger.warning(" " + line) + return False + + rv = _apply_acl(acl, path) + if defacl: + defacl.calc_mask() + rv &= _apply_acl(defacl, path, posix1e.ACL_TYPE_DEFAULT) + return rv + + def _set_secontext(self, entry, path=None): + """ set the SELinux context of the file on disk according to the + config""" + if not has_selinux: + return True + + if path is None: + path = entry.get("name") + context = entry.get("secontext") + if context is None: + # no context listed + return True + + if context == '__default__': + try: + selinux.restorecon(path) + rv = True + except: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to restore SELinux context " + "for %s: %s" % (path, err)) + rv = False + else: + try: + rv = selinux.lsetfilecon(path, context) == 0 + except: + err = sys.exc_info()[1] + self.logger.error("POSIX: Failed to restore SELinux context " + "for %s: %s" % (path, err)) + rv = False + return rv + + def _norm_gid(self, gid): + """ This takes a group name or gid and returns the + corresponding gid. """ + try: + return int(gid) + except ValueError: + return int(grp.getgrnam(gid)[2]) + + def _norm_entry_gid(self, entry): + try: + return self._norm_gid(entry.get('group')) + except (OSError, KeyError): + err = sys.exc_info()[1] + self.logger.error('POSIX: GID normalization failed for %s on %s: %s' + % (entry.get('group'), entry.get('name'), err)) + return 0 + + def _norm_uid(self, uid): + """ This takes a username or uid and returns the + corresponding uid. """ + try: + return int(uid) + except ValueError: + return int(pwd.getpwnam(uid)[2]) + + def _norm_entry_uid(self, entry): + try: + return self._norm_uid(entry.get("owner")) + except (OSError, KeyError): + err = sys.exc_info()[1] + self.logger.error('POSIX: UID normalization failed for %s on %s: %s' + % (entry.get('owner'), entry.get('name'), err)) + return 0 + + def _norm_acl_perms(self, perms): + """ takes a representation of an ACL permset and returns a digit + representing the permissions entailed by it. representations can + either be a single octal digit, a string of up to three 'r', + 'w', 'x', or '-' characters, or a posix1e.Permset object""" + if hasattr(perms, 'test'): + # Permset object + return sum([p for p in acl_map.values() + if perms.test(p)]) + + try: + # single octal digit + rv = int(perms) + if rv > 0 and rv < 8: + return rv + else: + self.logger.error("POSIX: Permissions digit out of range in " + "ACL: %s" % perms) + return 0 + except ValueError: + # couldn't be converted to an int; process as a string + if len(perms) > 3: + self.logger.error("POSIX: Permissions string too long in ACL: " + "%s" % perms) + return 0 + rv = 0 + for char in perms: + if char == '-': + continue + elif char not in acl_map: + self.logger.warning("POSIX: Unknown permissions character " + "in ACL: %s" % char) + elif rv & acl_map[char]: + self.logger.warning("POSIX: Duplicate permissions " + "character in ACL: %s" % perms) + else: + rv |= acl_map[char] + return rv + + def _acl2string(self, aclkey, perms): + atype, scope, qualifier = aclkey + acl_str = [] + if atype == 'default': + acl_str.append(atype) + if scope == posix1e.ACL_USER: + acl_str.append("user") + elif scope == posix1e.ACL_GROUP: + acl_str.append("group") + acl_str.append(qualifier) + acl_str.append(self._acl_perm2string(perms)) + return ":".join(acl_str) + + def _acl_perm2string(self, perm): + rv = [] + for char in 'rwx': + if acl_map[char] & perm: + rv.append(char) + else: + rv.append('-') + return ''.join(rv) + + def _gather_data(self, path): + try: + ondisk = os.stat(path) + except OSError: + self.logger.debug("POSIX: %s does not exist" % path) + return (False, None, None, None, None, None) + + try: + owner = str(ondisk[stat.ST_UID]) + except OSError: + err = sys.exc_info()[1] + self.logger.debug("POSIX: Could not get current owner of %s: %s" % + (path, err)) + owner = None + except KeyError: + self.logger.error('POSIX: User resolution failed for %s' % path) + owner = None + + try: + group = str(ondisk[stat.ST_GID]) + except (OSError, KeyError): + err = sys.exc_info()[1] + self.logger.debug("POSIX: Could not get current group of %s: %s" % + (path, err)) + group = None + except KeyError: + self.logger.error('POSIX: Group resolution failed for %s' % path) + group = None + + try: + perms = oct(ondisk[stat.ST_MODE])[-4:] + except (OSError, KeyError, TypeError): + err = sys.exc_info()[1] + self.logger.debug("POSIX: Could not get current permissions of %s: " + "%s" % (path, err)) + perms = None + + if has_selinux: + try: + secontext = selinux.getfilecon(path)[1].split(":")[2] + except (OSError, KeyError): + err = sys.exc_info()[1] + self.logger.debug("POSIX: Could not get current SELinux " + "context of %s: %s" % (path, err)) + secontext = None + else: + secontext = None + + if has_acls: + acls = self._list_file_acls(path) + else: + acls = None + return (ondisk, owner, group, perms, secontext, acls) + + def _verify_metadata(self, entry, path=None): + """ generic method to verify perms, owner, group, secontext, acls, + and mtime """ + # allow setting an alternate path for recursive permissions checking + if path is None: + path = entry.get('name') + attrib = dict() + ondisk, attrib['current_owner'], attrib['current_group'], \ + attrib['current_perms'], attrib['current_secontext'], acls = \ + self._gather_data(path) + + if not ondisk: + entry.set('current_exists', 'false') + return False + + # we conditionally verify every bit of metadata only if it's + # specified on the entry. consequently, canVerify() and + # fully_specified() are preconditions of _verify_metadata(), + # since they will ensure that everything that needs to be + # specified actually is. this lets us gracefully handle + # symlink and hardlink entries, which have SELinux contexts + # but not other permissions, optional secontext and mtime + # attrs, and so on. + configOwner, configGroup, configPerms, mtime = None, None, None, -1 + if entry.get('mtime', '-1') != '-1': + mtime = str(ondisk[stat.ST_MTIME]) + if entry.get("owner"): + configOwner = str(self._norm_entry_uid(entry)) + if entry.get("group"): + configGroup = str(self._norm_entry_gid(entry)) + if entry.get("perms"): + while len(entry.get('perms', '')) < 4: + entry.set('perms', '0' + entry.get('perms', '')) + configPerms = int(entry.get('perms'), 8) + + errors = [] + if configOwner and attrib['current_owner'] != configOwner: + errors.append("Owner for path %s is incorrect. " + "Current owner is %s but should be %s" % + (path, attrib['current_owner'], entry.get('owner'))) + + if configGroup and attrib['current_group'] != configGroup: + errors.append("Group for path %s is incorrect. " + "Current group is %s but should be %s" % + (path, attrib['current_group'], entry.get('group'))) + + if (configPerms and + oct(int(attrib['current_perms'], 8)) != oct(configPerms)): + errors.append("Permissions for path %s are incorrect. " + "Current permissions are %s but should be %s" % + (path, attrib['current_perms'], entry.get('perms'))) + + if entry.get('mtime'): + attrib['current_mtime'] = mtime + if mtime != entry.get('mtime', '-1'): + errors.append("mtime for path %s is incorrect. " + "Current mtime is %s but should be %s" % + (path, mtime, entry.get('mtime'))) + + if has_selinux and entry.get("secontext"): + if entry.get("secontext") == "__default__": + configContext = selinux.matchpathcon(path, 0)[1].split(":")[2] + else: + configContext = entry.get("secontext") + if attrib['current_secontext'] != configContext: + errors.append("SELinux context for path %s is incorrect. " + "Current context is %s but should be %s" % + (path, attrib['current_secontext'], + configContext)) + + if errors: + for error in errors: + self.logger.debug("POSIX: " + error) + entry.set('qtext', "\n".join([entry.get('qtext', '')] + errors)) + if path == entry.get("name"): + for attr, val in attrib.items(): + entry.set(attr, val) + + aclVerifies = self._verify_acls(entry, path=path) + return aclVerifies and len(errors) == 0 + + def _list_entry_acls(self, entry): + wanted = dict() + for acl in entry.findall("ACL"): + if acl.get("scope") == "user": + scope = posix1e.ACL_USER + elif acl.get("scope") == "group": + scope = posix1e.ACL_GROUP + else: + self.logger.error("POSIX: Unknown ACL scope %s" % + acl.get("scope")) + continue + wanted[(acl.get("type"), scope, acl.get(acl.get("scope")))] = \ + self._norm_acl_perms(acl.get('perms')) + return wanted + + def _list_file_acls(self, path): + def _process_acl(acl, atype): + try: + if acl.tag_type == posix1e.ACL_USER: + qual = pwd.getpwuid(acl.qualifier)[0] + elif acl.tag_type == posix1e.ACL_GROUP: + qual = grp.getgrgid(acl.qualifier)[0] + else: + return + except (OSError, KeyError): + err = sys.exc_info()[1] + self.logger.error("POSIX: Lookup of %s %s failed: %s" % + (scope, acl.qualifier, err)) + qual = acl.qualifier + existing[(atype, acl.tag_type, qual)] = \ + self._norm_acl_perms(acl.permset) + + existing = dict() + try: + for acl in posix1e.ACL(file=path): + _process_acl(acl, "access") + except IOError: + err = sys.exc_info()[1] + if err.errno == 95: + # fs is mounted noacl + self.logger.debug("POSIX: Filesystem mounted without ACL " + "support: %s" % path) + else: + self.logger.error("POSIX: Error getting current ACLS on %s: %s" + % (path, err)) + return existing + + if os.path.isdir(path): + for acl in posix1e.ACL(filedef=path): + _process_acl(acl, "default") + return existing + + def _verify_acls(self, entry, path=None): + if not has_acls: + if entry.findall("ACL"): + self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " + "library installed" % entry.get('name')) + return True + + if path is None: + path = entry.get("name") + + # create lists of normalized representations of the ACLs we want + # and the ACLs we have. this will make them easier to compare + # than trying to mine that data out of the ACL objects and XML + # objects and compare it at the same time. + wanted = self._list_entry_acls(entry) + existing = self._list_file_acls(path) + + missing = [] + extra = [] + wrong = [] + for aclkey, perms in wanted.items(): + if aclkey not in existing: + missing.append(self._acl2string(aclkey, perms)) + elif existing[aclkey] != perms: + wrong.append((self._acl2string(aclkey, perms), + self._acl2string(aclkey, existing[aclkey]))) + if path == entry.get("name"): + atype, scope, qual = aclkey + aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, + perms=str(perms)) + if scope == posix1e.ACL_USER: + aclentry.set("scope", "user") + elif scope == posix1e.ACL_GROUP: + aclentry.set("scope", "group") + else: + self.logger.debug("POSIX: Unknown ACL scope %s on %s" % + (scope, path)) + continue + aclentry.set(aclentry.get("scope"), qual) + entry.append(aclentry) + + for aclkey, perms in existing.items(): + if aclkey not in wanted: + extra.append(self._acl2string(aclkey, perms)) + + msg = [] + if missing: + msg.append("%s ACLs are missing: %s" % (len(missing), + ", ".join(missing))) + if wrong: + msg.append("%s ACLs are wrong: %s" % + (len(wrong), + "; ".join(["%s should be %s" % (e, w) + for w, e in wrong]))) + if extra: + msg.append("%s extra ACLs: %s" % (len(extra), ", ".join(extra))) + + if msg: + msg.insert(0, "POSIX: ACLs for %s are incorrect." % path) + self.logger.debug(msg[0]) + for line in msg[1:]: + self.logger.debug(" " + line) + entry.set('qtext', "\n".join([entry.get("qtext", '')] + msg)) + return False + return True + + def _makedirs(self, entry, path=None): + """ os.makedirs helpfully creates all parent directories for + us, but it sets permissions according to umask, which is + probably wrong. we need to find out which directories were + created and set permissions on those + (http://trac.mcs.anl.gov/projects/bcfg2/ticket/1125) """ + created = [] + if path is None: + path = entry.get("name") + cur = path + while cur != '/': + if not os.path.exists(cur): + created.append(cur) + cur = os.path.dirname(cur) + rv = True + try: + os.makedirs(path) + except OSError: + err = sys.exc_info()[1] + self.logger.error('POSIX: Failed to create directory %s: %s' % + (path, err)) + rv = False + for cpath in created: + rv &= self._set_perms(entry, path=cpath) + return rv -- cgit v1.2.3-1-g7c22