'''This is the basic toolset class for the Bcfg2 client''' __revision__ = '$Revision$' from binascii import a2b_base64 from copy import deepcopy from grp import getgrgid, getgrnam from os import chown, chmod, lstat, mkdir, stat, unlink, rename, readlink, symlink from pwd import getpwuid, getpwnam from stat import S_ISVTX, S_ISGID, S_ISUID, S_IXUSR, S_IWUSR, S_IRUSR, S_IXGRP from stat import S_IWGRP, S_IRGRP, S_IXOTH, S_IWOTH, S_IROTH, ST_MODE, S_ISDIR from stat import S_IFREG, ST_UID, ST_GID, S_ISREG, S_IFDIR, S_ISLNK from sys import exc_info import stat as statmod from math import floor, ceil #from time import asctime, localtime from traceback import extract_tb from popen2 import Popen4 from lxml.etree import Element, SubElement, tostring def calc_perms(initial, perms): '''This compares ondisk permissions with specified ones''' pdisp = [{1:S_ISVTX, 2:S_ISGID, 4:S_ISUID}, {1:S_IXUSR, 2:S_IWUSR, 4:S_IRUSR}, {1:S_IXGRP, 2:S_IWGRP, 4:S_IRGRP}, {1:S_IXOTH, 2:S_IWOTH, 4:S_IROTH}] tempperms = initial if len(perms) == 3: perms = '0%s' % (perms) pdigits = [int(perms[digit]) for digit in range(4)] for index in range(4): for (num, perm) in pdisp[index].iteritems(): if pdigits[index] & num: tempperms |= perm return tempperms class Toolset(object): '''The toolset class contains underlying command support and all states''' __important__ = [] __name__ = 'Toolset' pkgtool = ('echo', ('%s', ['name'])) def __init__(self, cfg, setup): '''Install initial configs, and setup state structures''' object.__init__(self) self.setup = setup self.cfg = cfg self.states = {} self.structures = {} self.modified = [] self.installed = {} self.pkgwork = {'add':[], 'update':[], 'remove':[]} self.extra_services = [] (self.height, self.width) = self.get_height_width() if self.__important__: for cfile in [cfl for cfl in cfg.findall(".//ConfigFile") if cfl.get('name') in self.__important__]: self.VerifyEntry(cfile) if not self.states[cfile]: self.InstallConfigFile(cfile) def saferun(self, command): '''Run a command in a pipe dealing with stdout buffer overloads''' self.CondPrint('debug', '> %s' % command) runpipe = Popen4(command, bufsize=16384) output = runpipe.fromchild.read() if len(output) > 0: self.CondPrint('debug', '< %s' % output) cmdstat = runpipe.poll() while cmdstat == -1: moreOutput = runpipe.fromchild.read() if len(moreOutput) > 0: self.CondPrint('debug', '< %s' % moreOutput) output += moreOutput cmdstat = runpipe.poll() return (cmdstat, [line for line in output.split('\n') if line]) def CondPrint(self, state, msg): '''Conditionally print message''' if self.setup[state]: try: prefix = "%s[%s]: " % (self.__name__, state) line_len = self.width-len(prefix) for line in msg.split('\n'): inner_lines = int(floor(float(len(line)) / line_len))+1 for i in xrange(inner_lines): print "%s%s" % (prefix, line[i*line_len:(i+1)*line_len]) except IOError: pass def get_height_width(self): '''Get Terminal information''' try: import termios, struct, fcntl height, width = struct.unpack('hhhh', fcntl.ioctl(0, termios.TIOCGWINSZ, "\000"*8))[0:2] if height == 0 or width == 0: return 25, 80 return height, width except: return 25, 80 def FormattedCondPrint(self, state, items): '''Formatted conditional print''' items.sort() screenWidth = self.width - len("%s[%s]:" % (self.__name__, state)) columnWidth = 1 for item in items: if len(item) > columnWidth: columnWidth = len(item) columnWidth += 1 columns = int(floor(float(screenWidth) / columnWidth)) lines = int(ceil(float(len(items)) / columns)) for lineNumber in xrange(lines): lineItems = [] for columnNumber in xrange(columns): itemNumber = int(columnNumber*lines + lineNumber) if itemNumber < len(items): lineItems.append(items[itemNumber]) format = "%%-%ss" % columnWidth lineText = "".join([format % item for item in lineItems]) self.CondPrint(state, lineText.rstrip()) def CondDisplayList(self, state, title, items): '''Conditionally print a list of data''' self.CondPrint(state, title) self.FormattedCondPrint(state, items) self.CondPrint(state, '') def CondDisplayState(self, state, phase): '''Conditionally print tracing information''' self.CondPrint(state, 'Phase: %s' % phase) self.CondPrint(state, 'Correct entries:\t%d' % self.states.values().count(True)) self.CondPrint(state, 'Incorrect entries:\t%d' % self.states.values().count(False)) self.CondPrint(state, 'Total managed entries:\t%d' % len(self.states.values())) self.CondPrint(state, 'Unmanaged entries:\t%d' % len(self.pkgwork['remove'])) self.CondPrint(state, '') if ((self.states.values().count(False) > 0) and not self.pkgwork['remove']): self.CondPrint('verbose', 'All entries correct.') def LogFailure(self, area, entry): '''Print tracebacks in unexpected cases''' print "Failure in %s for entry: %s" % (area, tostring(entry)) (ttype, value, trace) = exc_info() for line in extract_tb(trace): print "File %s, line %i, in %s\n %s\n" % (line) print "%s: %s\n" % (ttype, value) del ttype, value, trace # These next functions form the external API def Refresh(self): '''Update based on current pkg system state''' return def Inventory(self): '''Inventory system status''' self.CondPrint('verbose', "Inventorying system...") self.Inventory_Entries() all = deepcopy(self.installed) desired = {} for entry in self.cfg.findall(".//Package"): desired[entry.attrib['name']] = entry self.pkgwork = {'update':[], 'add':[], 'remove':[]} for pkg, entry in desired.iteritems(): if self.states.get(entry, True): # package entry verifies del all[pkg] else: if all.has_key(pkg): # wrong version del all[pkg] self.pkgwork['update'].append(entry) else: # new pkg self.pkgwork['add'].append(entry) # pkgwork contains all one-way verification data now # all data remaining in all is extra packages self.pkgwork['remove'] = all.keys() def Inventory_Entries(self): '''Build up workqueue for installation''' # build initial set of states unexamined = [(child, []) for child in self.cfg.getchildren()] while unexamined: (entry, modlist) = unexamined.pop() if entry.tag not in ['Bundle', 'Independant']: self.VerifyEntry(entry, modlist) else: modlist = [cfile.get('name') for cfile in entry.getchildren() if cfile.tag == 'ConfigFile'] unexamined += [(child, modlist) for child in entry.getchildren()] self.structures[entry] = False for structure in self.cfg.getchildren(): self.CheckStructure(structure) def CheckStructure(self, structure): '''Check structures with bundle verification semantics''' if structure in self.modified: self.modified.remove(structure) if structure.tag == 'Bundle': # check for clobbered data modlist = [cfile.get('name') for cfile in structure.getchildren() if cfile.tag == 'ConfigFile'] for entry in structure.getchildren(): self.VerifyEntry(entry, modlist) try: state = [self.states[entry] for entry in structure.getchildren()] if False not in state: self.structures[structure] = True except KeyError, msg: print "State verify evidently failed for %s" % (msg) self.structures[structure] = False def GenerateStats(self, client_version): '''Generate XML summary of execution statistics''' stats = Element("Statistics") # Calculate number of total bundles and structures total = len(self.states) stats.set('total', str(total)) # Calculate number of good bundles and structures good = len([key for key, val in self.states.iteritems() if val]) stats.set('good', str(good)) stats.set('version', '2.0') stats.set('client_version', client_version) if len([key for key, val in self.structures.iteritems() if not val]) == 0: stats.set('state', 'clean') dirty = 0 else: stats.set('state', 'dirty') dirty = 1 #stats.set('time', asctime(localtime())) # List bad elements of the configuration if dirty: bad_elms = SubElement(stats, "Bad") for elm in [key for key, val in self.states.iteritems() if not val]: if elm.get('name') == None: SubElement(bad_elms, elm.tag) else: SubElement(bad_elms, elm.tag, name=elm.get('name')) if self.modified: mod = SubElement(stats, "Modified") for elm in self.modified: SubElement(mod, elm.tag, name=elm.get('name')) if self.extra_services + self.pkgwork['remove']: extra = SubElement(stats, "Extra") [SubElement(extra, "Service", name=svc) for svc in self.extra_services] [SubElement(extra, "Package", name=pkg) for pkg in self.pkgwork['remove']] return stats # the next two are dispatch functions def VerifyEntry(self, entry, modlist = []): '''Dispatch call to Verify and save state in self.states''' try: method = getattr(self, "Verify%s" % (entry.tag)) # verify state and stash value in state if entry.tag == 'Package': self.states[entry] = method(entry, modlist) else: self.states[entry] = method(entry) except: self.LogFailure("Verify", entry) def InstallEntry(self, entry): '''Dispatch call to self.Install''' try: method = getattr(self, "Install%s"%(entry.tag)) self.states[entry] = method(entry) except: self.LogFailure("Install", entry) # All remaining operations implement the mechanics of POSIX cfg elements def VerifySymLink(self, entry): '''Verify SymLink Entry''' try: sloc = readlink(entry.get('name')) if sloc == entry.get('to'): return True return False except OSError: return False def InstallSymLink(self, entry): '''Install SymLink Entry''' self.CondPrint('verbose', "Installing Symlink %s" % (entry.get('name'))) try: fmode = lstat(entry.get('name'))[ST_MODE] if S_ISREG(fmode) or S_ISLNK(fmode): self.CondPrint('debug', "Non-directory entry already exists at %s" % (entry.get('name'))) unlink(entry.get('name')) elif S_ISDIR(fmode): self.CondPrint('debug', "Directory entry already exists at %s" % (entry.get('name'))) self.saferun("mv %s/ %s.bak" % (entry.get('name'), entry.get('name'))) else: unlink(entry.get('name')) except OSError: print "Symlink %s cleanup failed" % (entry.get('name')) try: symlink(entry.get('to'), entry.get('name')) return True except OSError: return False def VerifyDirectory(self, entry): '''Verify Directory Entry''' while len(entry.get('perms', '')) < 4: entry.set('perms', '0' + entry.get('perms', '')) try: ondisk = stat(entry.get('name')) except OSError: self.CondPrint("debug", "Directory %s does not exist" % (entry.get('name'))) return False try: owner = getpwuid(ondisk[ST_UID])[0] group = getgrgid(ondisk[ST_GID])[0] except OSError: self.CondPrint('debug', 'User resolution failing') owner = 'root' group = 'root' perms = oct(stat(entry.get('name'))[ST_MODE])[-4:] if ((owner == entry.get('owner')) and (group == entry.get('group')) and (perms == entry.get('perms'))): return True else: if owner != entry.get('owner'): self.CondPrint("debug", "Directory %s ownership wrong" % (entry.get('name'))) if group != entry.get('group'): self.CondPrint("debug", "Directory %s group wrong" % (entry.get('name'))) if perms != entry.get('perms'): self.CondPrint("debug", "Directory %s permissions wrong: are %s should be %s" % (entry.get('name'), perms, entry.get('perms'))) return False def InstallDirectory(self, entry): '''Install Directory Entry''' exists = False self.CondPrint('verbose', "Installing Directory %s" % (entry.get('name'))) try: fmode = lstat(entry.get('name')) if not S_ISDIR(fmode[ST_MODE]): self.CondPrint("debug", "Found a non-directory entry at %s" % (entry.get('name'))) try: unlink(entry.get('name')) except OSError: self.CondPrint('verbose', "Failed to unlink %s" % (entry.get('name'))) return False else: exists = True except OSError: # stat failed pass if not exists: try: mkdir(entry.get('name')) except OSError: self.CondPrint('debug', 'Failed to create directory %s' % (entry.get('name'))) return False try: chown(entry.get('name'), getpwnam(entry.get('owner'))[2], getgrnam(entry.get('group'))[2]) chmod(entry.get('name'), calc_perms(S_IFDIR, entry.get('perms'))) return True except (OSError, KeyError): self.CondPrint('debug', 'Permission fixup failed for %s' % (entry.get('name'))) return False def VerifyConfigFile(self, entry): '''Install ConfigFile Entry''' filename = entry.get('name') try: ondisk = stat(filename) except OSError: self.CondPrint('debug', "File %s doesn't exist" % (filename)) return False try: data = open(filename).read() except IOError: self.CondPrint('debug', "Failed to read %s" % (filename)) return False try: owner = getpwuid(ondisk[ST_UID])[0] group = getgrgid(ondisk[ST_GID])[0] except KeyError: self.CondPrint('debug', "Owner/Group failure for %s: %s, %s" % (filename, ondisk[ST_UID], ondisk[ST_GID])) return False perms = stat(filename)[ST_MODE] if entry.get('encoding', 'ascii') == 'base64': tempdata = a2b_base64(entry.text) else: tempdata = entry.text if ((data == tempdata) and (owner == entry.get('owner')) and (group == entry.get('group')) and (perms == calc_perms(S_IFREG, entry.get('perms')))): return True else: if data != tempdata: self.CondPrint('debug', "File %s contents wrong" % (filename)) elif ((owner != entry.get('owner')) or (group != entry.get('group'))): self.CondPrint('debug', 'File %s ownership wrong' % (filename)) elif perms != calc_perms(S_IFREG, entry.get('perms')): self.CondPrint('debug', 'File %s permissions wrong' % (filename)) return False def InstallConfigFile(self, entry): '''Install ConfigFile Entry''' if entry.text == None: self.CondPrint('verbose', "Incomplete information for ConfigFile %s. Cannot install" % (entry.get('name'))) return False self.CondPrint('verbose', "Installing ConfigFile %s" % (entry.get('name'))) if self.setup['dryrun']: return False parent = "/".join(entry.get('name').split('/')[:-1]) if parent: for idx in xrange(len(parent.split('/')[:-1])): current = '/'+'/'.join(parent.split('/')[1:2+idx]) try: sloc = lstat(current) try: if not S_ISDIR(sloc[ST_MODE]): unlink(current) mkdir(current) except OSError: return False except OSError: try: mkdir(current) except OSError: return False # If we get here, then the parent directory should exist try: newfile = open("%s.new"%(entry.get('name')), 'w') if entry.get('encoding', 'ascii') == 'base64': filedata = a2b_base64(entry.text) else: filedata = entry.text newfile.write(filedata) newfile.close() try: chown(newfile.name, getpwnam(entry.get('owner'))[2], getgrnam(entry.get('group'))[2]) except KeyError: chown(newfile.name, 0, 0) chmod(newfile.name, calc_perms(S_IFREG, entry.get('perms'))) if entry.get("paranoid", False) and self.setup.get("paranoid", False): self.saferun("cp %s /var/cache/bcfg2/%s" % (entry.get('name'))) rename(newfile.name, entry.get('name')) return True except (OSError, IOError), err: if err.errno == 13: self.CondPrint('verbose', "Failed to open %s for writing" % (entry.get('name'))) else: print err return False def VerifyPackage(self, entry, modlist): '''Dummy package verification method. Cannot succeed''' return False def VerifyPermissions(self, entry): '''Verify method for abstract permission''' try: sinfo = stat(entry.get('name')) except OSError: self.CondPrint('debug', "Entry %s doesn't exist" % entry.get('name')) return False # pad out perms if needed while len(entry.get('perms', '')) < 4: entry.set('perms', '0' + entry.get('perms', '')) perms = oct(sinfo[ST_MODE])[-4:] if perms == entry.get('perms'): return True self.CondPrint('debug', "Entry %s permissions incorrect" % entry.get('name')) return False def InstallPermissions(self, entry): '''Install method for abstract permission''' try: sinfo = stat(entry.get('name')) except OSError: self.CondPrint('debug', "Entry %s doesn't exist" % entry.get('name')) return False for ftype in ['DIR', 'REG', 'CHR', 'BLK']: if getattr(statmod, "S_IS%s" % ftype)(sinfo[ST_MODE]): chmod(entry.get('name'), calc_perms(getattr(statmod, "S_IF%s" % ftype), entry.get('perms'))) return True self.CondPrint('verbose', "Entry %s has unknown file type" % entry.get('name')) return False def VerifyPostInstall(self, entry): '''Postinstall verification method''' return True def HandleBundleDeps(self): '''Handle bundles depending on what has been modified''' for entry in [child for child in self.structures if child.tag == 'Bundle']: bchildren = entry.getchildren() if [b_ent for b_ent in bchildren if b_ent in self.modified]: # This bundle has been modified self.CondPrint('verbose', "%s %s needs update" % (entry.tag, entry.get('name', '???'))) modfiles = [cfile.get('name') for cfile in bchildren if cfile.tag == 'ConfigFile'] for child in bchildren: if child.tag == 'Package': self.VerifyPackage(child, modfiles) else: self.VerifyEntry(child) if not self.states[child]: self.CondPrint('debug', "Reinstalling clobbered entry %s %s" % (child.tag, child.get('name'))) self.InstallEntry(child) self.VerifyEntry(child) self.CondPrint('debug', "Re-checked entry %s %s: %s" % (child.tag, child.get('name'), self.states[child])) for postinst in [entry for entry in bchildren if entry.tag == 'PostInstall']: self.saferun(postinst.get('name')) for svc in [svc for svc in bchildren if svc.tag == 'Service' and svc.get('status', 'off') == 'on']: if self.setup['build']: # stop services in miniroot self.saferun('/etc/init.d/%s stop' % svc.get('name')) else: self.CondPrint('debug', 'Restarting service %s' % svc.get('name')) self.saferun('/etc/init.d/%s %s' % (svc.get('name'), svc.get('reload', 'reload'))) for entry in self.structures: if [strent for strent in entry.getchildren() if not self.states.get(strent, False)]: self.CondPrint('verbose', "%s %s incomplete" % (entry.tag, entry.get('name', ""))) else: self.structures[entry] = True def HandleExtra(self): '''deal with extra configuration during installation''' return False def Install(self): '''Correct detected misconfigurations''' if self.setup['dryrun']: self.CondPrint("verbose", "Dry-run mode: no changes will be made") else: self.CondPrint("verbose", "Updating the system") self.CondPrint("verbose", "") self.HandleExtra() # use quick package ops from here on self.setup['quick'] = True self.CondDisplayList('dryrun', "Packages to update:", [pkg.get('name') for pkg in self.pkgwork['update']]) self.CondDisplayList('dryrun', "Packages to add:", [pkg.get('name') for pkg in self.pkgwork['add']]) self.CondDisplayList('dryrun', "Packages to remove:", self.pkgwork['remove']) self.CondDisplayList('dryrun', "Entries to update:", ["%s: %s" % (entry.tag, entry.get('name')) for entry in self.states if not (self.states[entry] or entry.tag == 'Package')]) self.CondDisplayList('dryrun', "Services to remove:", self.extra_services) if self.setup['dryrun']: return # build up work queue work = self.pkgwork['add'] + self.pkgwork['update'] # add non-package entries work += [ent for ent in self.states if ent.tag != 'Package' and not self.states[ent]] # Counters ## Packages left to install left = len(work) + len(self.pkgwork['remove']) ## Packages installed in previous iteration old = left + 1 ## loop iterations performed count = 1 # Installation loop while ((0 < left < old) and (count < 20)): # Print pass info self.CondPrint('verbose', "Starting pass %s" % (count)) self.CondPrint("verbose", "%s Entries left" % (len(work))) self.CondPrint('verbose', "%s new, %s update, %s remove" % (len(self.pkgwork['add']), len(self.pkgwork['update']), len(self.pkgwork['remove']))) # Update counters count = count + 1 old = left self.CondPrint("verbose", "Installing non-package entries") [self.InstallEntry(ent) for ent in work if ent.tag != 'Package'] packages = [pkg for pkg in work if pkg.tag == 'Package'] ptypes = [] for pkg in packages: if pkg.get('type') not in ptypes: ptypes.append(pkg.get('type')) if packages: for pkgtype in ptypes: # try single large install self.CondPrint("verbose", "Trying single pass package install for pkgtype %s" % pkgtype) if not self.pkgtool.has_key(pkgtype): self.CondPrint("verbose", "No support for pkgtype %s" % (pkgtype)) continue pkgtool = self.pkgtool[pkgtype] pkglist = [pkg for pkg in packages if pkg.get('type') == pkgtype] for field in pkgtool[1][1]: pkglist = [pkg for pkg in pkglist if pkg.attrib.has_key(field)] if not pkglist: self.CondPrint("debug", "No complete/installable packages of type %s" % pkgtype) continue pkgargs = " ".join([pkgtool[1][0] % tuple([pkg.get(field) for field in pkgtool[1][1]]) for pkg in pkglist]) self.CondPrint("debug", "Installing packages: :%s:" % pkgargs) self.CondPrint("debug", "Running command ::%s::" % (pkgtool[0] % pkgargs)) cmdrc = self.saferun(pkgtool[0] % pkgargs)[0] if cmdrc == 0: self.CondPrint('verbose', "Single Pass Succeded") # set all package states to true and flush workqueues pkgnames = [pkg.get('name') for pkg in pkglist] for entry in [entry for entry in self.states.keys() if entry.tag == 'Package' and entry.get('type') == pkgtype and entry.get('name') in pkgnames]: self.CondPrint('debug', 'Setting state to true for pkg %s' % (entry.get('name'))) self.states[entry] = True [self.pkgwork[listname].remove(entry) for listname in ['add', 'update'] if self.pkgwork[listname].count(entry)] self.Refresh() else: self.CondPrint("verbose", "Single Pass Failed") # do single pass installs self.Refresh() for pkg in pkglist: # handle state tracking updates if self.VerifyPackage(pkg, []): self.CondPrint("verbose", "Forcing state to true for pkg %s" % (pkg.get('name'))) self.states[pkg] = True else: self.CondPrint("verbose", "Installing pkg %s version %s" % (pkg.get('name'), pkg.get('version'))) cmdrc = self.saferun(pkgtool[0] % (pkgtool[1][0] % tuple([pkg.get(field) for field in pkgtool[1][1]]))) if cmdrc[0] == 0: self.states[pkg] = True else: self.CondPrint('verbose', "Failed to install package %s" % (pkg.get('name'))) for entry in [ent for ent in work if self.states[ent]]: work.remove(entry) self.modified.append(entry) left = len(work) + len(self.pkgwork['remove']) self.HandleBundleDeps()