From f3eb3148238ea38683c1586518bbecd108353c65 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Tue, 13 Dec 2005 21:38:02 +0000 Subject: set keyword attributes on most files added Ed's client and server cleanups Modified the debian and redhat toolsets to produce nicer output ** Broke performance reports git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1623 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Client/Debian.py | 49 +-- src/lib/Client/Gentoo.py | 2 +- src/lib/Client/Redhat.py | 11 +- src/lib/Client/Solaris.py | 1 + src/lib/Client/Toolset.py | 143 ++++++-- src/lib/Server/Component.py | 2 +- src/lib/Server/Plugins/Account.py | 4 +- src/lib/Server/Plugins/SSHbase.py | 4 +- src/lib/Server/Statistics.py | 2 +- src/lib/Server/__init__.py | 2 +- src/sbin/Bcfg2Server | 16 +- src/sbin/bcfg2 | 711 ++++++++++++++++++++++---------------- 12 files changed, 576 insertions(+), 371 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Debian.py b/src/lib/Client/Debian.py index 35b398ffd..9260a2afe 100644 --- a/src/lib/Client/Debian.py +++ b/src/lib/Client/Debian.py @@ -7,32 +7,35 @@ from re import compile as regcompile import apt_pkg -from Bcfg2.Client.Toolset import Toolset, saferun +from Bcfg2.Client.Toolset import Toolset class ToolsetImpl(Toolset): '''The Debian toolset implements package and service operations and inherits the rest from Toolset.Toolset''' + __name__ = 'Debian' __important__ = ["/etc/apt/sources.list", "/var/cache/debconf/config.dat", \ "/var/cache/debconf/templates.dat", '/etc/passwd', '/etc/group', \ '/etc/apt/apt.conf'] - pkgtool = {'deb':('DEBIAN_FRONTEND=noninteractive apt-get --reinstall -q=2 --force-yes -y install %s >/dev/null 2>&1', + pkgtool = {'deb':('DEBIAN_FRONTEND=noninteractive apt-get --reinstall -q=2 --force-yes -y install %s', ('%s=%s', ['name', 'version']))} svcre = regcompile("/etc/.*/[SK]\d\d(?P\S+)") def __init__(self, cfg, setup): Toolset.__init__(self, cfg, setup) self.cfg = cfg + self.CondPrint('debug', 'Configuring Debian toolset') environ["DEBIAN_FRONTEND"] = 'noninteractive' - system("dpkg --force-confold --configure -a > /dev/null 2>&1") + self.saferun("dpkg --force-confold --configure -a") if not self.setup['build']: - system("dpkg-reconfigure -f noninteractive debconf < /dev/null > /dev/null 2>&1") - system("apt-get clean > /dev/null 2>&1") - system("apt-get -q=2 -y update > /dev/null 2>&1") + self.saferun("/usr/sbin/dpkg-reconfigure -f noninteractive debconf < /dev/null") + self.saferun("apt-get clean") + self.saferun("apt-get -q=2 -y update") self.installed = {} self.pkgwork = {'add':[], 'update':[], 'remove':[]} for pkg in [cpkg for cpkg in self.cfg.findall(".//Package") if not cpkg.attrib.has_key('type')]: pkg.set('type', 'deb') self.Refresh() + self.CondPrint('debug', 'Done configuring Debian toolset') def Refresh(self): '''Refresh memory hashes of packages''' @@ -44,7 +47,6 @@ class ToolsetImpl(Toolset): self.installed[pkg.Name] = pkg.CurrentVer.VerStr # implement entry (Verify|Install) ops - def VerifyService(self, entry): '''Verify Service status for entry''' rawfiles = glob("/etc/rc*.d/*%s" % (entry.get('name'))) @@ -74,24 +76,27 @@ class ToolsetImpl(Toolset): if self.setup['dryrun']: print "Disabling service %s" % (entry.get('name')) else: - system("/etc/init.d/%s stop > /dev/null 2>&1" % (entry.get('name'))) - cmdrc = system("/usr/sbin/update-rc.d -f %s remove > /dev/null 2>&1" % - entry.get('name')) + self.saferun("/etc/init.d/%s stop" % (entry.get('name'))) + cmdrc = self.saferun("/usr/sbin/update-rc.d -f %s remove" % entry.get('name'))[0] else: if self.setup['dryrun']: print "Enabling service %s" % (entry.attrib['name']) else: - cmdrc = system("/usr/sbin/update-rc.d %s defaults > /dev/null 2>&1" % (entry.attrib['name'])) + cmdrc = self.saferun("/usr/sbin/update-rc.d %s defaults" % (entry.attrib['name']))[0] if cmdrc: return False return True def VerifyPackage(self, entry, modlist): - '''Verify Package for entry''' + '''Verify package for entry''' + if not entry.attrib.has_key('version'): + self.CondPrint('verbose', "Cannot verify unversioned package %s" % + (entry.attrib['name'])) + return False if self.installed.has_key(entry.attrib['name']): if self.installed[entry.attrib['name']] == entry.attrib['version']: if not self.setup['quick']: - output = saferun("debsums -s %s" % entry.get('name'))[1] + output = self.saferun("/usr/bin/debsums -s %s" % entry.get('name'))[1] if [filename for filename in output if filename not in modlist]: return False return True @@ -103,7 +108,7 @@ class ToolsetImpl(Toolset): allsrv = [] [allsrv.append(self.svcre.match(fname).group('name')) for fname in glob("/etc/rc[12345].d/S*") if self.svcre.match(fname).group('name') not in allsrv] - self.CondPrint('debug', "Found active services: %s" % allsrv) + self.CondDisplayList('debug', "Found active services:", allsrv) csrv = self.cfg.findall(".//Service") [allsrv.remove(svc.get('name')) for svc in csrv if svc.get('status') == 'on' and svc.get('name') in allsrv] @@ -111,18 +116,22 @@ class ToolsetImpl(Toolset): def HandleExtra(self): '''Deal with extra configuration detected''' + if self.setup['dryrun']: + return + if len(self.pkgwork) > 0: if self.setup['remove'] in ['all', 'packages']: - self.CondPrint('verbose', "Removing packages: %s" % self.pkgwork['remove']) - if not system("apt-get remove %s" % " ".join(self.pkgwork['remove'])): + self.CondDisplayList('verbose', "Removing packages", self.pkgwork['remove']) + if not self.saferun("apt-get remove %s" % " ".join(self.pkgwork['remove']))[0]: self.pkgwork['remove'] = [] else: - self.CondPrint('verbose', "Need to remove packages: %s" % self.pkgwork['remove']) + self.CondDisplayList('verbose', "Need to remove packages:", self.pkgwork['remove']) + if len(self.extra_services) > 0: if self.setup['remove'] in ['all', 'services']: - self.CondPrint('verbose', "Removing services: %s" % self.extra_services) + self.CondDisplayList('verbose', "Removing services:", self.extra_services) [self.extra_services.remove(serv) for serv in self.extra_services if - not system("rm -f /etc/rc*.d/S??%s" % serv)] + not self.saferun("rm -f /etc/rc*.d/S??%s" % serv)[0]] else: - self.CondPrint('verbose', "Need to remove services: %s" % self.extra_services) + self.CondDisplayList('verbose', "Need to remove services:", self.extra_services) diff --git a/src/lib/Client/Gentoo.py b/src/lib/Client/Gentoo.py index 1bd698a4a..84ca313c5 100644 --- a/src/lib/Client/Gentoo.py +++ b/src/lib/Client/Gentoo.py @@ -1,5 +1,5 @@ '''This provides bcfg2 support for Gentoo''' -__revision__ = '$Revision: $' +__revision__ = '$Revision$' from os import popen, system, stat from popen2 import Popen4 diff --git a/src/lib/Client/Redhat.py b/src/lib/Client/Redhat.py index 57f1a17ba..6c9d0d7c9 100644 --- a/src/lib/Client/Redhat.py +++ b/src/lib/Client/Redhat.py @@ -1,5 +1,5 @@ # This is the bcfg2 support for redhat -# $Id: $ +# $Id$ '''This is redhat client support''' __revision__ = '$Revision$' @@ -10,6 +10,7 @@ from Bcfg2.Client.Toolset import Toolset, saferun class ToolsetImpl(Toolset): '''This class implelements support for rpm packages and standard chkconfig services''' + __name__ = 'Redhat' pkgtool = {'rpm':("rpm --oldpackage --replacepkgs --quiet -U %s", ("%s", ["url"]))} def __init__(self, cfg, setup): @@ -111,21 +112,21 @@ class ToolsetImpl(Toolset): self.pkgwork['remove'] = [] self.Inventory() else: - self.CondPrint('verbose', "Need to remove packages: %s" % self.pkgwork['remove']) + self.CondDisplayList('verbose', "Need to remove packages", self.pkgwork['remove']) if len(self.extra_services) > 0: if self.setup['remove'] in ['all', 'services']: - self.CondPrint('verbose', "Removing services: %s" % self.extra_services) + self.CondDisplayList('verbose', 'Removing services:', self.extra_services) for service in self.extra_services: if not system("/sbin/chkconfig %s off" % service): self.extra_services.remove(service) else: - self.CondPrint('verbose', "Need to remove services: %s" % self.extra_services) + self.CondDisplayList('verbose', 'Need to remove services:', self.extra_services) def Inventory(self): '''Do standard inventory plus debian extra service check''' Toolset.Inventory(self) allsrv = [line.split()[0] for line in popen("/sbin/chkconfig --list|grep :on").readlines()] - self.CondPrint('debug', "Found active services: %s" % allsrv) + self.CondDisplayList('debug', 'Found active services:', allsrv) csrv = self.cfg.findall(".//Service") [allsrv.remove(svc.get('name')) for svc in csrv if svc.get('status') == 'on' and svc.get('name') in allsrv] diff --git a/src/lib/Client/Solaris.py b/src/lib/Client/Solaris.py index a02df4cce..8ab506aa9 100644 --- a/src/lib/Client/Solaris.py +++ b/src/lib/Client/Solaris.py @@ -31,6 +31,7 @@ class ToolsetImpl(Toolset): 'encap':("/local/sbin/epkg -l -q %s", ("%s", ["url"]))} splitter = regcompile('.*/(?P[\w-]+)\-(?P[\w\.-]+)') ptypes = {} + __name__ = 'Solaris' def __init__(self, cfg, setup): Toolset.__init__(self, cfg, setup) diff --git a/src/lib/Client/Toolset.py b/src/lib/Client/Toolset.py index 32deea99b..7c2da5f14 100644 --- a/src/lib/Client/Toolset.py +++ b/src/lib/Client/Toolset.py @@ -11,10 +11,10 @@ from stat import S_IWGRP, S_IRGRP, S_IXOTH, S_IWOTH, S_IROTH, ST_MODE, S_ISDIR from stat import S_IFREG, ST_UID, ST_GID, S_ISREG, S_IFDIR, S_ISLNK from sys import exc_info import stat as statmod +from math import floor, ceil #from time import asctime, localtime from traceback import extract_tb from popen2 import Popen4 - from lxml.etree import Element, SubElement, tostring def calc_perms(initial, perms): @@ -31,19 +31,10 @@ def calc_perms(initial, perms): tempperms |= perm return tempperms -def saferun(command): - '''Run a command in a pipe dealing with stdout buffer overloads''' - runpipe = Popen4(command, bufsize=16384) - output = runpipe.fromchild.read() - cmdstat = runpipe.poll() - while cmdstat == -1: - output += runpipe.fromchild.read() - cmdstat = runpipe.poll() - return (cmdstat, [line for line in output.split('\n') if line]) - class Toolset(object): '''The toolset class contains underlying command support and all states''' __important__ = [] + __name__ = 'Toolset' pkgtool = ('echo', ('%s', ['name'])) def __init__(self, cfg, setup): @@ -57,20 +48,97 @@ class Toolset(object): self.installed = {} self.pkgwork = {'add':[], 'update':[], 'remove':[]} self.extra_services = [] + (self.height, self.width) = self.get_height_width() if self.__important__: for cfile in [cfl for cfl in cfg.findall(".//ConfigFile") if cfl.get('name') in self.__important__]: self.VerifyEntry(cfile) if not self.states[cfile]: self.InstallConfigFile(cfile) + def saferun(self, command): + '''Run a command in a pipe dealing with stdout buffer overloads''' + self.CondPrint('debug', '> %s' % command) + + runpipe = Popen4(command, bufsize=16384) + output = runpipe.fromchild.read() + if len(output) > 0: + self.CondPrint('debug', '< %s' % output) + cmdstat = runpipe.poll() + while cmdstat == -1: + moreOutput = runpipe.fromchild.read() + if len(moreOutput) > 0: + self.CondPrint('debug', '< %s' % moreOutput) + output += moreOutput + cmdstat = runpipe.poll() + + return (cmdstat, [line for line in output.split('\n') if line]) + def CondPrint(self, state, msg): '''Conditionally print message''' if self.setup[state]: try: - print msg + prefix = "%s[%s]: " % (self.__name__, state) + line_len = self.width-len(prefix) + for line in msg.split('\n'): + inner_lines = int(floor(float(len(line)) / line_len))+1 + for i in xrange(inner_lines): + print "%s%s" % (prefix, line[i*line_len:(i+1)*line_len]) except IOError: pass + def get_height_width(self): + try: + import termios, struct, fcntl + height, width = struct.unpack('hhhh', + fcntl.ioctl(0, termios.TIOCGWINSZ, + "\000"*8))[0:2] + return height, width + except: + return 25, 80 + + def FormattedCondPrint(self, state, items): + items.sort() + screenWidth = self.width - len("%s[%s]:" % (self.__name__, state)) + columnWidth = 1 + for item in items: + if len(item) > columnWidth: + columnWidth = len(item) + columnWidth += 1 + + columns = int(floor(float(screenWidth) / columnWidth)) + lines = int(ceil(float(len(items)) / columns)) + + for lineNumber in xrange(lines): + lineItems = [] + for columnNumber in xrange(columns): + itemNumber = int(columnNumber*lines + lineNumber) + if itemNumber < len(items): + lineItems.append(items[itemNumber]) + format = "%%-%ss" % columnWidth + lineText = "".join([format % item for item in lineItems]) + self.CondPrint(state, lineText.rstrip()) + + def CondDisplayList(self, state, title, items): + self.CondPrint(state, title) + self.FormattedCondPrint(state, items) + self.CondPrint(state, '') + + def CondDisplayState(self, state, phase): + self.CondPrint(state, 'Phase: %s' % phase) + self.CondPrint(state, 'Correct entries:\t%d' + % self.states.values().count(True)) + self.CondPrint(state, 'Incorrect entries:\t%d' % + self.states.values().count(False)) + self.CondPrint(state, 'Total managed entries:\t%d' % + len(self.states.values())) + self.CondPrint(state, 'Unmanaged entries:\t%d' % + len(self.pkgwork['remove'])) + self.CondPrint(state, '') + + if ((self.states.values().count(False) > 0) and + not self.pkgwork['remove']): + self.CondPrint('All entries correct.') + def LogFailure(self, area, entry): '''Print tracebacks in unexpected cases''' print "Failure in %s for entry: %s" % (area, tostring(entry)) @@ -229,7 +297,7 @@ class Toolset(object): unlink(entry.get('name')) elif S_ISDIR(fmode): self.CondPrint('debug', "Directory entry already exists at %s" % (entry.get('name'))) - system("mv %s/ %s.bak" % (entry.get('name'), entry.get('name'))) + self.saferun("mv %s/ %s.bak" % (entry.get('name'), entry.get('name'))) else: unlink(entry.get('name')) except OSError: @@ -386,7 +454,7 @@ class Toolset(object): chown(newfile.name, 0, 0) chmod(newfile.name, calc_perms(S_IFREG, entry.get('perms'))) if entry.get("paranoid", False) and self.setup.get("paranoid", False): - system("cp %s /var/cache/bcfg2/%s" % (entry.get('name'))) + self.saferun("cp %s /var/cache/bcfg2/%s" % (entry.get('name'))) rename(newfile.name, entry.get('name')) return True except (OSError, IOError), err: @@ -413,9 +481,10 @@ class Toolset(object): perms = oct(sinfo[ST_MODE])[-4:] if perms == entry.get('perms'): return True - else: - self.CondPrint('debug', "Entry %s permissions incorrect" % entry.get('name')) + self.CondPrint('debug', "Entry %s permissions incorrect" % entry.get('name')) + return False + def InstallPermissions(self, entry): '''Install method for abstract permission''' try: @@ -455,15 +524,15 @@ class Toolset(object): self.CondPrint('debug', "Re-checked entry %s %s: %s" % (child.tag, child.get('name'), self.states[child])) for postinst in [entry for entry in bchildren if entry.tag == 'PostInstall']: - system(postinst.get('name')) + self.saferun(postinst.get('name')) for svc in [svc for svc in bchildren if svc.tag == 'Service' and svc.get('status', 'off') == 'on']: if self.setup['build']: # stop services in miniroot - system('/etc/init.d/%s stop' % svc.get('name')) + self.saferun('/etc/init.d/%s stop' % svc.get('name')) else: self.CondPrint('debug', 'Restarting service %s' % svc.get('name')) - system('/etc/init.d/%s %s' % (svc.get('name'), svc.get('reload', 'reload'))) + self.saferun('/etc/init.d/%s %s' % (svc.get('name'), svc.get('reload', 'reload'))) for entry in self.structures: if [strent for strent in entry.getchildren() if not self.states.get(strent, False)]: @@ -477,18 +546,27 @@ class Toolset(object): def Install(self): '''Correct detected misconfigurations''' - self.CondPrint("verbose", "Installing needed configuration changes") + if self.setup['dryrun']: + self.CondPrint("verbose", "Dry-run mode: no changes will be made") + else: + self.CondPrint("verbose", "Updating the system") + self.CondPrint("verbose", "") self.HandleExtra() # use quick package ops from here on self.setup['quick'] = True - self.CondPrint('dryrun', "Packages to update: %s" % - (" ".join([pkg.get('name') for pkg in self.pkgwork['update']]))) - self.CondPrint('dryrun', "Packages to add: %s" % - (" ".join([pkg.get('name') for pkg in self.pkgwork['add']]))) - self.CondPrint('dryrun', "Packages to remove %s" % (" ".join(self.pkgwork['remove']))) - for entry in [entry for entry in self.states if (not self.states[entry] - and (entry.tag != 'Package'))]: - self.CondPrint('dryrun', "Entry %s %s updated" % (entry.tag, entry.get('name'))) + + self.CondDisplayList('dryrun', "Packages to update:", + [pkg.get('name') for pkg in self.pkgwork['update']]) + self.CondDisplayList('dryrun', "Packages to add:", + [pkg.get('name') for pkg in self.pkgwork['add']]) + self.CondDisplayList('dryrun', "Packages to remove:", + self.pkgwork['remove']) + self.CondDisplayList('dryrun', "Entries to update:", + ["%s: %s" % (entry.tag, entry.get('name')) + for entry in self.states if not (self.states[entry] + or entry.tag == 'Package')]) + self.CondDisplayList('dryrun', "Services to remove:", self.extra_services) + if self.setup['dryrun']: return @@ -518,7 +596,7 @@ class Toolset(object): count = count + 1 old = left - self.CondPrint("verbose", "Installing Non Package entries") + self.CondPrint("verbose", "Installing non-package entries") [self.InstallEntry(ent) for ent in work if ent.tag != 'Package'] packages = [pkg for pkg in work if pkg.tag == 'Package'] @@ -545,7 +623,7 @@ class Toolset(object): self.CondPrint("debug", "Installing packages: :%s:" % pkgargs) self.CondPrint("debug", "Running command ::%s::" % (pkgtool[0] % pkgargs)) - cmdrc = system(pkgtool[0] % pkgargs) + (cmdrc, cmdoutput) = self.saferun(pkgtool[0] % pkgargs) if cmdrc == 0: self.CondPrint('verbose', "Single Pass Succeded") @@ -569,8 +647,9 @@ class Toolset(object): else: self.CondPrint("verbose", "Installing pkg %s version %s" % (pkg.get('name'), pkg.get('version'))) - cmdrc = system(pkgtool[0] % - (pkgtool[1][0]%tuple([pkg.get(field) for field in pkgtool[1][1]]))) + (cmdrc, cmdoutput) = self.saferun(pkgtool[0] % + (pkgtool[1][0] % + tuple([pkg.get(field) for field in pkgtool[1][1]]))) if cmdrc == 0: self.states[pkg] = True else: diff --git a/src/lib/Server/Component.py b/src/lib/Server/Component.py index d24f6576f..97444bb10 100644 --- a/src/lib/Server/Component.py +++ b/src/lib/Server/Component.py @@ -1,5 +1,5 @@ '''Cobalt component base classes''' -__revision__ = '$Revision: 1.4 $' +__revision__ = '$Revision$' from ConfigParser import ConfigParser, NoOptionError from cPickle import loads, dumps diff --git a/src/lib/Server/Plugins/Account.py b/src/lib/Server/Plugins/Account.py index 5d2853e9e..05174486d 100644 --- a/src/lib/Server/Plugins/Account.py +++ b/src/lib/Server/Plugins/Account.py @@ -1,5 +1,5 @@ '''This handles authentication setup''' -__revision__ = '$Revision: 1.28 $' +__revision__ = '$Revision$' from Bcfg2.Server.Plugin import Plugin, PluginInitError, DirectoryBacked @@ -13,7 +13,7 @@ class Account(Plugin): rootlike -> users to be granted root privs on some hosts ''' __name__ = 'Account' - __version__ = '$Id: Account.py 1.28 05/11/30 17:31:55-06:00 desai@topaz.mcs.anl.gov $' + __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cc93cc1c3..6cab373b0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -1,5 +1,5 @@ '''This module manages ssh key files for bcfg2''' -__revision__ = '$Revision: 1.56 $' +__revision__ = '$Revision$' from binascii import b2a_base64 from os import system, popen @@ -26,7 +26,7 @@ class SSHbase(Plugin): is regenerated each time a new key is generated. ''' __name__ = 'SSHbase' - __version__ = '$Id: SSHbase.py 1.56 05/09/27 16:06:14-05:00 desai@topaz.mcs.anl.gov $' + __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", diff --git a/src/lib/Server/Statistics.py b/src/lib/Server/Statistics.py index 6f7c5cd4f..15f1586ef 100644 --- a/src/lib/Server/Statistics.py +++ b/src/lib/Server/Statistics.py @@ -1,5 +1,5 @@ '''This file manages the statistics collected by the BCFG2 Server''' -__revision__ = '$Revision: $' +__revision__ = '$Revision$' from lxml.etree import XML, SubElement, Element, XMLSyntaxError from syslog import syslog, LOG_ERR diff --git a/src/lib/Server/__init__.py b/src/lib/Server/__init__.py index 6a490cae8..c998f1c12 100644 --- a/src/lib/Server/__init__.py +++ b/src/lib/Server/__init__.py @@ -1,4 +1,4 @@ -# $Id: $ +# $Id$ '''This is the set of modules for Bcfg2.Server''' __revision__ = '$Revision$' diff --git a/src/sbin/Bcfg2Server b/src/sbin/Bcfg2Server index 4b4193b76..8477fd4ca 100644 --- a/src/sbin/Bcfg2Server +++ b/src/sbin/Bcfg2Server @@ -24,9 +24,9 @@ def critical_error(operation): (ttype, value, trace) = exc_info() for line in extract_tb(trace): syslog(LOG_ERR, "File %s, line %i, in %s\n %s" % (line)) - syslog(LOG_ERR, "%s: %s" % (ttype, value)) - del trace, val, trb + syslog(LOG_ERR, "%s: %s" % (ttype, value)) warning_error("An unexpected failure occurred in %s" % (operation) ) + raise Fault, (7, "Critical unexpected failure: %s" % (operation)) def fatal_error(message): '''Signal a fatal error''' @@ -166,14 +166,18 @@ class Bcfg2(Component): try: meta = self.Core.metadata.FetchMetadata(client) + + for generator in self.Core.generators: + for probe in generator.GetProbes(meta): + resp.append(probe) + return tostring(resp) except MetadataConsistencyError: warning = 'metadata consistency error' warning_error(warning) raise Fault, (6, warning) - for generator in self.Core.generators: - for probe in generator.GetProbes(meta): - resp.append(probe) - return tostring(resp) + except: + critical_error("determining client probes") + def Bcfg2RecvProbeData(self, address, probedata): '''Receive probe data from clients''' diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 82e22603c..fcc3a757c 100644 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -3,346 +3,457 @@ '''Bcfg2 Client''' __revision__ = '$Revision$' -import ConfigParser -import getopt -import signal -import socket -import sys -import tempfile -import time -import traceback -import xmlrpclib - +from getopt import getopt, GetoptError from os import popen, chmod, unlink, _exit +from signal import signal, SIGINT +from sys import argv +from tempfile import mktemp +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from xmlrpclib import ServerProxy, Fault from lxml.etree import Element, XML, tostring, XMLSyntaxError +from time import sleep, time +from sys import exc_info +from traceback import extract_tb +import socket def cb_sigint_handler(signum, frame): '''Exit upon CTRL-C''' _exit(1) +def if_then(cond, value_if, value_else): + ''' Replacement for ternary operator ''' + if cond == True: + return value_if + else: + return value_else + class SafeProxy: '''Wrapper for proxy''' - def __init__(self, user, password, retries, serverUrl): - self.user = user - self.password = password - self.retries = retries - self.serverUrl = serverUrl - self.proxy = xmlrpclib.ServerProxy(serverUrl) + def __init__(self, setup, client): self.retryCount = 0 + self.client = client + self.setup = setup + try: + self.proxy = ServerProxy(self.setup["server"]) + except IOError, io_error: + self.client.fatal_error("Invalid server URL %s: %s" % + (self.setup["server"], io_error)) + except: + self.client.critical_error("initialising XML-RPC") - def runMethod(self, operationDescription, methodName, methodArgs): - '''Execute xmlrpc method call''' - method = getattr(self.proxy, methodName) - instanceRetries = 0 - for i in xrange(self.retries): + def run_method(self, operation_desc, method_name, method_args): + ''' Perform an XMLRPC invocation against the server''' + method = getattr(self.proxy, method_name) + instance_retries = 0 + for i in xrange(int(self.setup["retries"])): try: - verbose("Attempting %s (%d of %d)" % (operationDescription, (i+1), self.retries)) - ret = apply(method, (self.user, self.password) + methodArgs) - if(instanceRetries > 0): - warning_error("during %s:\nRequired %d attempts to contact server (%s)" % - (instanceRetries, operationDescription, self.serverUrl)) - verbose("%s completed successfully" % (operationDescription)) + self.client.cond_print("debug", "Attempting %s (%d of %d)" % + (operation_desc,(i+1), + int(self.setup["retries"]))) + ret = apply(method, (self.setup['user'], + self.setup['password']) + method_args) + if instance_retries > 0: + self.client.warning_error( + "during %s:\nRequired %d attempts to contact server (%s)" + % (operation_desc, instance_retries, + self.setup["server"])) + self.client.cond_print("debug", "%s completed successfully" % + (operation_desc)) return ret - except xmlrpclib.Fault, f: - fatal_error("%s encountered a server error:\n%s" % - (operationDescription, f)) - except socket.error, e: - instanceRetries += 1 + except Fault, fault: + self.client.fatal_error("%s encountered a server error:\n%s" % + (operation_desc, fault)) + except socket.error: + instance_retries += 1 self.retryCount += 1 - time.sleep(0.5) + sleep(1.0) except: - critical_error(operationDescription) - - fatal_error("%s failed:\nCould not connect to server (%s)" % - (operationDescription, self.serverUrl)) - -def load_toolset(toolset, config, clientsetup): - '''Import client toolset modules''' - - toolsetPackages = { - 'debian': "Bcfg2.Client.Debian", - 'rh': "Bcfg2.Client.Redhat", - 'solaris': "Bcfg2.Client.Solaris" - } - - try: - mod = __import__(toolsetPackages[toolset], globals(), locals(), ['*']) - except KeyError, k: - fatal_error("got unsupported toolset %s from server." % (toolset)) + self.client.critical_error(operation_desc) + + self.client.fatal_error("%s failed:\nCould not connect to server (%s)" % + (operation_desc, self.setup["server"])) - try: - myToolset = mod.ToolsetImpl(config, clientsetup) - - verbose("Selected %s toolset..." % (toolset)) - return myToolset; - except: - critical_error("instantiating toolset %s" % (toolset)) - -def run_probe(probe): - '''Execute probe''' - probeName = probe.attrib['name'] - ret = Element("probe-data", probeName, source=probe.attrib['source']) - try: - script = open(tempfile.mktemp(), 'w+') - try: - script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) - script.write(probe.text) - script.close() - chmod(script.name, 0755) - ret.text = popen(script.name).read() - - finally: - unlink(script.name) - except: - critical_error("executing probe %s" % (probeName)) - return ret - -def critical_error(operation): - '''Print tracebacks in unexpected cases''' - print "Traceback information (please include in any bug report):" - (ttype, value, trace) = sys.exc_info() - for line in traceback.extract_tb(trace): - print "File %s, line %i, in %s\n %s\n" % (line) - print "%s: %s\n" % (ttype, value) - - fatal_error("An unexpected failure occurred in %s" % (operation) ) - -def fatal_error(message): - '''Signal a fatal error''' - print "Fatal error: %s\n" % (message) - raise SystemExit, 1 - -def warning_error(message): - '''Warn about a problem but continue''' - print "Warning: %s\n" % (message) - -def usage_error(message, opt, vopt, descs, argDescs): - '''Die because script was called the wrong way''' - print "Usage error: %s" % (message) - print_usage(opt, vopt, descs, argDescs) - raise SystemExit, 2 - -verboseMode = False - -def verbose(message): - '''Conditionally output information in verbose mode''' - global verboseMode - - if(verboseMode == True): - print "bcfg2: %s\n" % (message) - -def print_usage(opt, vopt, descs, argDescs): - print "bcfg2 usage:" - for arg in opt.iteritems(): - print " -%s\t\t\t%s" % (arg[0], descs[arg[0]]) - for arg in vopt.iteritems(): - print " -%s %s\t%s" % (arg[0], argDescs[arg[0]], descs[arg[0]]) - -def dgetopt(arglist, opt, vopt, descs, argDescs): - '''parse options into a dictionary''' - global verboseMode - - ret = {} - for optname in opt.values() + vopt.values(): - ret[optname] = False + +class Client: + ''' The main bcfg2 client class ''' + def __init__(self, args): + self.toolset = None + self.config = None + self.options = { + 'verbose': 'v', + 'quick': 'q', + 'debug': 'd', + 'dryrun': 'n', + 'build': 'B', + 'paranoid': 'P', + 'bundle': 'b', + 'file': 'f', + 'cache': 'c', + 'profile': 'p', + 'image': 'i', + 'remove': 'r', + 'help': 'h', + 'setup': 's', + 'server': 'S', + 'user': 'u', + 'password': 'x', + 'retries': 'R' + } + self.argOptions = { + 'v': 'verbose', + 'q': 'quick', + 'd': 'debug', + 'n': 'dryrun', + 'B': 'build', + 'P': 'paranoid', + 'b': 'bundle', + 'f': 'file', + 'c': 'cache', + 'p': 'profile', + 'i': 'image', + 'r': 'remove', + 'h': 'help', + 's': 'setup', + 'S': 'server', + 'u': 'user', + 'x': 'password', + 'R': 'retries' + } + self.descriptions = { + 'verbose': "enable verbose output", + 'quick': "disable some checksum verification", + 'debug': "enable debugging output", + 'dryrun': "do not actually change the system", + 'build': "disable service control (implies -q)", + 'paranoid': "make automatic backups of config files", + 'bundle': "only configure the given bundle", + 'file': "configure from a file rather than querying the server", + 'cache': "store the configuration in a file", + 'image': "assert the given image for the host", + 'profile': "assert the given profile for the host", + 'remove': "force removal of additional configuration items", + 'help': "print this help message", + 'setup': "use given setup file (default /etc/bcfg2.conf)", + 'server': 'the server hostname to connect to', + 'user': 'the user to provide for authentication', + 'password': 'the password to use', + 'retries': 'the number of times to retry network communication' + } + self.argumentDescriptions = { + 'bundle': "", + 'file': "", + 'cache': "", + 'profile': "", + 'image': "", + 'remove': "(pkgs | svcs | all)", + 'setup': "", + 'server': ' ', + 'user': ' ', + 'password': ' ', + 'retries': '' + } + + self.setup = {} + self.get_setup(args) + + self.cond_print_setup('debug') + + def cond_print_setup(self, state): + ''' Display the clients current setup information ''' + for (key, value) in self.setup.iteritems(): + if self.setup[key]: + self.cond_print(state, "%s => %s" % (key, value)) + + + def load_toolset(self, toolset_name): + '''Import client toolset modules''' - gstr = "".join(opt.keys()) + "".join([optionkey + ':' for optionkey in vopt.keys()]) - try: - ginfo = getopt.getopt(arglist, gstr) - except getopt.GetoptError, gerr: - usage_error(gerr, opt, vopt, descs, argDescs) - - for (gopt, garg) in ginfo[0]: - option = gopt[1:] - if opt.has_key(option): - ret[opt[option]] = True + toolset_packages = { + 'debian': "Bcfg2.Client.Debian", + 'rh': "Bcfg2.Client.Redhat", + 'solaris': "Bcfg2.Client.Solaris" + } + + if toolset_packages.has_key(toolset_name): + toolset_class = toolset_packages[toolset_name] else: - ret[vopt[option]] = garg + toolset_class = toolset_name - if (ret["file"] != False) and (ret["cache"] != False): - usage_error("cannot use -f and -c together", - opt, vopt, descs, argDescs) + try: + mod = __import__(toolset_class, globals(), locals(), ['*']) + except: + self.fatal_error("got unsupported toolset %s from server." + % (toolset_name)) + + try: + self.toolset = mod.ToolsetImpl(self.config, self.setup) + + self.cond_print('debug', "Selected %s toolset..." % + (toolset_name)) + except: + self.critical_error("instantiating toolset %s" % + (toolset_name)) + + def run_probe(self, probe): + '''Execute probe''' + probe_name = probe.attrib['name'] + ret = Element("probe-data", probe_name, source=probe.attrib['source']) + try: + script = open(mktemp(), 'w+') + try: + script.write("#!%s\n" % + (probe.attrib.get('interpreter', '/bin/sh'))) + script.write(probe.text) + script.close() + chmod(script.name, 0755) + ret.text = popen(script.name).read() + finally: + unlink(script.name) + except: + self.critical_error("executing probe %s" % (probe_name)) + return ret + + def critical_error(self, operation): + '''Print tracebacks in unexpected cases''' + print "Traceback information (please include in any bug report):" + (ttype, value, trace) = exc_info() + for line in extract_tb(trace): + print "File %s, line %i, in %s\n %s\n" % (line) + print "%s: %s\n" % (ttype, value) + + self.fatal_error("An unexpected failure occurred in %s" % (operation) ) + + def fatal_error(self, message): + '''Signal a fatal error''' + print "Fatal error: %s" % (message) + raise SystemExit, 1 + + def warning_error(self, message): + '''Warn about a problem but continue''' + print "Warning: %s" % (message) + + def usage_error(self, message): + '''Die because script was called the wrong way''' + print "Usage error: %s" % (message) + self.print_usage() + raise SystemExit, 2 + + def cond_print(self, state, message): + '''Output debugging information''' + if self.setup[state]: + print "bcfg2[%s]: %s" % (state, message) + + def print_usage(self): + ''' Display usage information for bcfg2 ''' + print "bcfg2 usage:" + for arg in self.options.iteritems(): + if self.argumentDescriptions.has_key(arg[0]): + print " -%s %s\t%s" % (arg[1], + self.argumentDescriptions[arg[0]], + self.descriptions[arg[0]]) + else: + print " -%s\t\t\t%s" % (arg[1], self.descriptions[arg[0]]) + + def fill_setup_from_file(self, setup_file, ret): + ''' Read any missing configuration information from a file''' + default = { + 'server': 'http://localhost:6789/', + 'user': 'root', + 'retries': '6' + } + config_locations = { + 'server': ('components', 'bcfg2'), + 'user': ('communication', 'user'), + 'password': ('communication', 'password'), + 'retries': ('communicaton', 'retries') + } + + self.cond_print_setup('debug') + + config_parser = None + + for (key, (section, option)) in config_locations.iteritems(): + try: + if not (ret.has_key(key) and ret[key]): + if config_parser == None: + self.cond_print('debug', "no %s provided, reading setup info from %s" % + (key, setup_file)) + config_parser = ConfigParser() + config_parser.read(setup_file) + try: + ret[key] = config_parser.get(section, option) + except (NoSectionError, NoOptionError): + if default.has_key(key): + ret[key] = default[key] + else: + self.fatal_error( + "%s does not contain a value for %s (in %s)" % + (setup_file, option, section)) + except IOError, io_error: + self.fatal_error("unable to read %s: %s" % + (setup_file, io_error)) + except SystemExit: + raise + except: + self.critical_error("reading config file") - if ret["help"] == True: - print_usage(opt, vopt, descs, argDescs) - raise SystemExit, 0 + def get_setup(self, args): + '''parse options into a dictionary''' - if ret["verbose"] == True: - verboseMode = True + for option in self.options.keys(): + self.setup[option] = False - return ret + gstr = "".join([self.options[option] + + if_then(self.argumentDescriptions.has_key(option), + ':', '') + for option in self.options.keys()]) -if __name__ == '__main__': - # parse command line options - signal.signal(signal.SIGINT, cb_sigint_handler) - options = { - 'v':'verbose', - 'q':'quick', - 'd':'debug', - 'n':'dryrun', - 'B':'build', - 'P':'paranoid', - 'h':'help' - } - doptions = { - 'b':'bundle', - 'f':'file', - 'c':'cache', - 'p':'profile', - 'i':'image', - 'r':'remove' - } - descriptions = { - 'v': "enable verbose output", - 'q': "disable some checksum verification", - 'd': "enable debugging output", - 'n': "do not actually change the system", - 'B': "disable service control (implies -q)", - 'P': "make automatic backups of config files", - 'b': "only configure the given bundle", - 'f': "configure from a file rather than querying the server", - 'c': "store the configuration in a file", - 'p': "assert the given profile for the client", - 'i': "assert the given image for the client", - 'r': "force removal of additional configuration items", - 'h': "print this help message" - } - argumentDescriptions = { - 'b': "", - 'f': "", - 'c': "", - 'p': "", - 'i': "", - 'r': "(pkgs | svcs | all)" - } - setup = dgetopt(sys.argv[1:], options, doptions, - descriptions, argumentDescriptions) - timeinfo = Element("Times") - - # begin configuration - start = time.time() - - comm = None - if setup['file']: try: - verbose("reading cached configuration from %s" % (setup['file'])) - configfile = open(setup['file'], 'r') - r = configfile.read() - configfile.close() - except IOError: - fatal_error("failed to read cached configuration from: %s" % (setup['file'])) - else: - cf = ConfigParser.ConfigParser() - try: - bcfgConf = '/etc/bcfg2.conf' - verbose("reading setup info from %s" % (bcfgConf)) - cf.read(bcfgConf) - location = cf.get("components", "bcfg2") - user = 'root' - password = cf.get("communication", "password") - proxy = SafeProxy(user, password, 6, location) - except: - fatal_error("unable to read %s" % (bcfgConf)) + ginfo = getopt(args, gstr) + except GetoptError, gerr: + self.usage_error(gerr) + + for (gopt, garg) in ginfo[0]: + option = self.argOptions[gopt[1:]] + if self.argumentDescriptions.has_key(option): + self.setup[option] = garg + else: + self.setup[option] = True + + if (self.setup["file"] != False) and (self.setup["cache"] != False): + self.usage_error("cannot use -f and -c together") + + if self.setup["help"] == True: + self.print_usage() + raise SystemExit, 0 + + if self.setup["setup"]: + setup_file = self.setup["setup"] + else: + setup_file = '/etc/bcfg2.conf' + + self.fill_setup_from_file(setup_file, self.setup) - probedata = proxy.runMethod("probe download", "GetProbes", ()) + def run(self): + ''' Perform client execution phase ''' + times = {} + + # begin configuration + times['start'] = time() - timeinfo.set('probefetch', str(time.time() - start)) + if self.setup['file']: + # read config from file + try: + self.cond_print('debug', "reading cached configuration from %s" % + (self.setup['file'])) + configfile = open(self.setup['file'], 'r') + rawconfig = configfile.read() + configfile.close() + except IOError: + self.fatal_error("failed to read cached configuration from: %s" + % (self.setup['file'])) + else: + # retrieve config from server + proxy = SafeProxy(self.setup, self) - try: - probes = XML(probedata) - except XMLSyntaxError, e: - fatal_error("server returned invalid probe information") + probe_data = proxy.run_method("probe download", "GetProbes", ()) + + times['probe_download'] = time() + + try: + probes = XML(probe_data) + except XMLSyntaxError, syntax_error: + self.fatal_error( + "server returned invalid probe requests: %s" % + (syntax_error)) - # execute probes - try: - probeinfo = [run_probe(x) for x in probes.findall(".//probe")] - except: - fatal_error("bcfg encountered an unknown error running probes") + # execute probes + try: + probe_info = [self.run_probe(probe) + for probe in probes.findall(".//probe")] + except: + self.critical_error("executing probes") - # upload probe responses - proxy.runMethod("probe data upload", "RecvProbeData", (probeinfo, )) + # upload probe responses + proxy.run_method("probe data upload", "RecvProbeData", + (probe_info, )) - cstart = time.time() + times['probe_upload'] = time() + + rawconfig = proxy.run_method("configuration download", "GetConfig", + (self.setup['image'], + self.setup['profile'])) + + times['config_download'] = time() - cfginfo = proxy.runMethod("configuration download", "GetConfig", - (setup['image'], setup['profile'])) + if self.setup['cache']: + try: + open(self.setup['cache'], 'w').write(rawconfig) + except IOError: + self.warning_error("failed to write config cache file %s" % + (self.setup['cache'])) + times['caching'] = time() + + try: + self.config = XML(rawconfig) + except XMLSyntaxError, syntax_error: + self.fatal_error("the configuration could not be parsed: %s" % + (syntax_error)) - timeinfo.set('config', str(time.time() - cstart )) + times['config_parse'] = time() + + if self.config.tag == 'error': + self.fatal_error("server error: %s" % (self.config.text)) - if setup['cache']: + # Get toolset from server try: - open(setup['cache'], 'w').write(cfginfo) - except IOError: - warning_error("failed to write config cache file %s" % (setup['cache'])) + toolset_name = self.config.get('toolset') + except: + self.fatal_error("server did not specify a toolset") - pt = time.time() - try: - cfg = XML(cfginfo) - except XMLSyntaxError, e: - fatal_error("the configuration could not be parsed") + if self.setup['bundle']: + replacement_xml = Element("Configuration", version='2.0') + for child in self.config.getroot().getchildren(): + if ((child.tag == 'Bundle') and + (child.attrib['name'] == self.setup['bundle'])): + replacement_xml.append(child) + self.config = replacement_xml - timeinfo.set('parse', str(time.time() - pt)) - - if cfg.tag == 'error': - fatal_error("server error: %s" % (cfg.text)) - - # Get toolset from server - try: - cfg_toolset = cfg.get('toolset') - except: - fatal_error("server did not specify a toolset") - - if setup['bundle']: - c = Element("Configuration", version='2.0') - for child in cfg.getroot().getchildren(): - if ((child.tag == 'Bundle') and (child.attrib['name'] == setup['bundle'])): - c.append(child) - cfg = c - - # Create toolset handle - client = load_toolset(cfg_toolset, cfg, setup) - - istart = time.time() - # verify state - client.Inventory() - timeinfo.set('inventory', str(time.time() - istart)) - - correct = client.states.values().count(True) - total = len(client.states.values()) - - istart = time.time() + # Create toolset handle + self.load_toolset(toolset_name) + + times['initialization'] = time() + + # verify state + self.toolset.Inventory() + + times['inventory'] = time() - if ((correct < total) or client.pkgwork['remove']): - if client.pkgwork['remove']: - client.CondPrint('verbose', "Extra packages detected") # summarize current state - client.CondPrint('verbose', "--> %s of %s config elements correct" % (correct, total)) + self.toolset.CondDisplayState('verbose', 'initial') # install incorrect aspects of configuration - client.Install() - - client.CondPrint('verbose', "--> %s of %s config elements correct" % - (client.states.values().count(True), total)) - failed = [key for key, value in client.states.iteritems() if not value] - if failed: - client.CondPrint('verbose', "Failing Entries:") - [client.CondPrint('verbose', "%s:%s" % - (key.tag, key.get('name'))) - for key in failed if key.tag != 'Package'] - [client.CondPrint('verbose', "%s:%s-%s" % - (key.tag, key.get('name'), key.get('version', 'unset'))) - for key in failed if key.tag == 'Package'] - else: - client.CondPrint("verbose", "All entries correct") + self.toolset.Install() + + self.toolset.CondDisplayState('verbose', "final") - timeinfo.set('install', str(time.time() - istart)) - timeinfo.set('total', str(time.time() - start)) + times['install'] = time() + times['finished'] = time() - if not setup['file']: - # upload statistics - m = Element("upload-statistics") - stats = client.GenerateStats(__revision__) - stats.append(timeinfo) - m.append(stats) + if not self.setup['file']: + # upload statistics + feedback = Element("upload-statistics") + timeinfo = Element("OpStamps") + for (event, timestamp) in times.iteritems(): + timeinfo.set(event, str(timestamp)) + stats = self.toolset.GenerateStats(__revision__) + stats.append(timeinfo) + feedback.append(stats) - proxy.runMethod("uploading statistics", "RecvStats", (tostring(m),)) + proxy.run_method("uploading statistics", + "RecvStats", (tostring(feedback),)) + + +if __name__ == '__main__': + signal(SIGINT, cb_sigint_handler) + Client(argv[1:]).run() -- cgit v1.2.3-1-g7c22