diff options
author | Narayan Desai <desai@mcs.anl.gov> | 2005-12-13 21:38:02 +0000 |
---|---|---|
committer | Narayan Desai <desai@mcs.anl.gov> | 2005-12-13 21:38:02 +0000 |
commit | f3eb3148238ea38683c1586518bbecd108353c65 (patch) | |
tree | e31930d3411b23eebecec43cb4ddc358074d7059 /src/sbin | |
parent | a4b92de008aa1d56b521aebbfeaf442201df5a18 (diff) | |
download | bcfg2-f3eb3148238ea38683c1586518bbecd108353c65.tar.gz bcfg2-f3eb3148238ea38683c1586518bbecd108353c65.tar.bz2 bcfg2-f3eb3148238ea38683c1586518bbecd108353c65.zip |
set keyword attributes on most files
added Ed's client and server cleanups
Modified the debian and redhat toolsets to produce nicer output
** Broke performance reports
git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1623 ce84e21b-d406-0410-9b95-82705330c041
Diffstat (limited to 'src/sbin')
-rw-r--r-- | src/sbin/Bcfg2Server | 16 | ||||
-rw-r--r-- | src/sbin/bcfg2 | 711 |
2 files changed, 421 insertions, 306 deletions
diff --git a/src/sbin/Bcfg2Server b/src/sbin/Bcfg2Server index 4b4193b76..8477fd4ca 100644 --- a/src/sbin/Bcfg2Server +++ b/src/sbin/Bcfg2Server @@ -24,9 +24,9 @@ def critical_error(operation): (ttype, value, trace) = exc_info() for line in extract_tb(trace): syslog(LOG_ERR, "File %s, line %i, in %s\n %s" % (line)) - syslog(LOG_ERR, "%s: %s" % (ttype, value)) - del trace, val, trb + syslog(LOG_ERR, "%s: %s" % (ttype, value)) warning_error("An unexpected failure occurred in %s" % (operation) ) + raise Fault, (7, "Critical unexpected failure: %s" % (operation)) def fatal_error(message): '''Signal a fatal error''' @@ -166,14 +166,18 @@ class Bcfg2(Component): try: meta = self.Core.metadata.FetchMetadata(client) + + for generator in self.Core.generators: + for probe in generator.GetProbes(meta): + resp.append(probe) + return tostring(resp) except MetadataConsistencyError: warning = 'metadata consistency error' warning_error(warning) raise Fault, (6, warning) - for generator in self.Core.generators: - for probe in generator.GetProbes(meta): - resp.append(probe) - return tostring(resp) + except: + critical_error("determining client probes") + def Bcfg2RecvProbeData(self, address, probedata): '''Receive probe data from clients''' diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 82e22603c..fcc3a757c 100644 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -3,346 +3,457 @@ '''Bcfg2 Client''' __revision__ = '$Revision$' -import ConfigParser -import getopt -import signal -import socket -import sys -import tempfile -import time -import traceback -import xmlrpclib - +from getopt import getopt, GetoptError from os import popen, chmod, unlink, _exit +from signal import signal, SIGINT +from sys import argv +from tempfile import mktemp +from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from xmlrpclib import ServerProxy, Fault from lxml.etree import Element, XML, tostring, XMLSyntaxError +from time import sleep, time +from sys import exc_info +from traceback import extract_tb +import socket def cb_sigint_handler(signum, frame): '''Exit upon CTRL-C''' _exit(1) +def if_then(cond, value_if, value_else): + ''' Replacement for ternary operator ''' + if cond == True: + return value_if + else: + return value_else + class SafeProxy: '''Wrapper for proxy''' - def __init__(self, user, password, retries, serverUrl): - self.user = user - self.password = password - self.retries = retries - self.serverUrl = serverUrl - self.proxy = xmlrpclib.ServerProxy(serverUrl) + def __init__(self, setup, client): self.retryCount = 0 + self.client = client + self.setup = setup + try: + self.proxy = ServerProxy(self.setup["server"]) + except IOError, io_error: + self.client.fatal_error("Invalid server URL %s: %s" % + (self.setup["server"], io_error)) + except: + self.client.critical_error("initialising XML-RPC") - def runMethod(self, operationDescription, methodName, methodArgs): - '''Execute xmlrpc method call''' - method = getattr(self.proxy, methodName) - instanceRetries = 0 - for i in xrange(self.retries): + def run_method(self, operation_desc, method_name, method_args): + ''' Perform an XMLRPC invocation against the server''' + method = getattr(self.proxy, method_name) + instance_retries = 0 + for i in xrange(int(self.setup["retries"])): try: - verbose("Attempting %s (%d of %d)" % (operationDescription, (i+1), self.retries)) - ret = apply(method, (self.user, self.password) + methodArgs) - if(instanceRetries > 0): - warning_error("during %s:\nRequired %d attempts to contact server (%s)" % - (instanceRetries, operationDescription, self.serverUrl)) - verbose("%s completed successfully" % (operationDescription)) + self.client.cond_print("debug", "Attempting %s (%d of %d)" % + (operation_desc,(i+1), + int(self.setup["retries"]))) + ret = apply(method, (self.setup['user'], + self.setup['password']) + method_args) + if instance_retries > 0: + self.client.warning_error( + "during %s:\nRequired %d attempts to contact server (%s)" + % (operation_desc, instance_retries, + self.setup["server"])) + self.client.cond_print("debug", "%s completed successfully" % + (operation_desc)) return ret - except xmlrpclib.Fault, f: - fatal_error("%s encountered a server error:\n%s" % - (operationDescription, f)) - except socket.error, e: - instanceRetries += 1 + except Fault, fault: + self.client.fatal_error("%s encountered a server error:\n%s" % + (operation_desc, fault)) + except socket.error: + instance_retries += 1 self.retryCount += 1 - time.sleep(0.5) + sleep(1.0) except: - critical_error(operationDescription) - - fatal_error("%s failed:\nCould not connect to server (%s)" % - (operationDescription, self.serverUrl)) - -def load_toolset(toolset, config, clientsetup): - '''Import client toolset modules''' - - toolsetPackages = { - 'debian': "Bcfg2.Client.Debian", - 'rh': "Bcfg2.Client.Redhat", - 'solaris': "Bcfg2.Client.Solaris" - } - - try: - mod = __import__(toolsetPackages[toolset], globals(), locals(), ['*']) - except KeyError, k: - fatal_error("got unsupported toolset %s from server." % (toolset)) + self.client.critical_error(operation_desc) + + self.client.fatal_error("%s failed:\nCould not connect to server (%s)" % + (operation_desc, self.setup["server"])) - try: - myToolset = mod.ToolsetImpl(config, clientsetup) - - verbose("Selected %s toolset..." % (toolset)) - return myToolset; - except: - critical_error("instantiating toolset %s" % (toolset)) - -def run_probe(probe): - '''Execute probe''' - probeName = probe.attrib['name'] - ret = Element("probe-data", probeName, source=probe.attrib['source']) - try: - script = open(tempfile.mktemp(), 'w+') - try: - script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) - script.write(probe.text) - script.close() - chmod(script.name, 0755) - ret.text = popen(script.name).read() - - finally: - unlink(script.name) - except: - critical_error("executing probe %s" % (probeName)) - return ret - -def critical_error(operation): - '''Print tracebacks in unexpected cases''' - print "Traceback information (please include in any bug report):" - (ttype, value, trace) = sys.exc_info() - for line in traceback.extract_tb(trace): - print "File %s, line %i, in %s\n %s\n" % (line) - print "%s: %s\n" % (ttype, value) - - fatal_error("An unexpected failure occurred in %s" % (operation) ) - -def fatal_error(message): - '''Signal a fatal error''' - print "Fatal error: %s\n" % (message) - raise SystemExit, 1 - -def warning_error(message): - '''Warn about a problem but continue''' - print "Warning: %s\n" % (message) - -def usage_error(message, opt, vopt, descs, argDescs): - '''Die because script was called the wrong way''' - print "Usage error: %s" % (message) - print_usage(opt, vopt, descs, argDescs) - raise SystemExit, 2 - -verboseMode = False - -def verbose(message): - '''Conditionally output information in verbose mode''' - global verboseMode - - if(verboseMode == True): - print "bcfg2: %s\n" % (message) - -def print_usage(opt, vopt, descs, argDescs): - print "bcfg2 usage:" - for arg in opt.iteritems(): - print " -%s\t\t\t%s" % (arg[0], descs[arg[0]]) - for arg in vopt.iteritems(): - print " -%s %s\t%s" % (arg[0], argDescs[arg[0]], descs[arg[0]]) - -def dgetopt(arglist, opt, vopt, descs, argDescs): - '''parse options into a dictionary''' - global verboseMode - - ret = {} - for optname in opt.values() + vopt.values(): - ret[optname] = False + +class Client: + ''' The main bcfg2 client class ''' + def __init__(self, args): + self.toolset = None + self.config = None + self.options = { + 'verbose': 'v', + 'quick': 'q', + 'debug': 'd', + 'dryrun': 'n', + 'build': 'B', + 'paranoid': 'P', + 'bundle': 'b', + 'file': 'f', + 'cache': 'c', + 'profile': 'p', + 'image': 'i', + 'remove': 'r', + 'help': 'h', + 'setup': 's', + 'server': 'S', + 'user': 'u', + 'password': 'x', + 'retries': 'R' + } + self.argOptions = { + 'v': 'verbose', + 'q': 'quick', + 'd': 'debug', + 'n': 'dryrun', + 'B': 'build', + 'P': 'paranoid', + 'b': 'bundle', + 'f': 'file', + 'c': 'cache', + 'p': 'profile', + 'i': 'image', + 'r': 'remove', + 'h': 'help', + 's': 'setup', + 'S': 'server', + 'u': 'user', + 'x': 'password', + 'R': 'retries' + } + self.descriptions = { + 'verbose': "enable verbose output", + 'quick': "disable some checksum verification", + 'debug': "enable debugging output", + 'dryrun': "do not actually change the system", + 'build': "disable service control (implies -q)", + 'paranoid': "make automatic backups of config files", + 'bundle': "only configure the given bundle", + 'file': "configure from a file rather than querying the server", + 'cache': "store the configuration in a file", + 'image': "assert the given image for the host", + 'profile': "assert the given profile for the host", + 'remove': "force removal of additional configuration items", + 'help': "print this help message", + 'setup': "use given setup file (default /etc/bcfg2.conf)", + 'server': 'the server hostname to connect to', + 'user': 'the user to provide for authentication', + 'password': 'the password to use', + 'retries': 'the number of times to retry network communication' + } + self.argumentDescriptions = { + 'bundle': "<bundle name>", + 'file': "<cache file>", + 'cache': "<cache file>", + 'profile': "<profile name>", + 'image': "<image name>", + 'remove': "(pkgs | svcs | all)", + 'setup': "<setup file>", + 'server': '<hostname> ', + 'user': '<user name> ', + 'password': '<password> ', + 'retries': '<number of retries>' + } + + self.setup = {} + self.get_setup(args) + + self.cond_print_setup('debug') + + def cond_print_setup(self, state): + ''' Display the clients current setup information ''' + for (key, value) in self.setup.iteritems(): + if self.setup[key]: + self.cond_print(state, "%s => %s" % (key, value)) + + + def load_toolset(self, toolset_name): + '''Import client toolset modules''' - gstr = "".join(opt.keys()) + "".join([optionkey + ':' for optionkey in vopt.keys()]) - try: - ginfo = getopt.getopt(arglist, gstr) - except getopt.GetoptError, gerr: - usage_error(gerr, opt, vopt, descs, argDescs) - - for (gopt, garg) in ginfo[0]: - option = gopt[1:] - if opt.has_key(option): - ret[opt[option]] = True + toolset_packages = { + 'debian': "Bcfg2.Client.Debian", + 'rh': "Bcfg2.Client.Redhat", + 'solaris': "Bcfg2.Client.Solaris" + } + + if toolset_packages.has_key(toolset_name): + toolset_class = toolset_packages[toolset_name] else: - ret[vopt[option]] = garg + toolset_class = toolset_name - if (ret["file"] != False) and (ret["cache"] != False): - usage_error("cannot use -f and -c together", - opt, vopt, descs, argDescs) + try: + mod = __import__(toolset_class, globals(), locals(), ['*']) + except: + self.fatal_error("got unsupported toolset %s from server." + % (toolset_name)) + + try: + self.toolset = mod.ToolsetImpl(self.config, self.setup) + + self.cond_print('debug', "Selected %s toolset..." % + (toolset_name)) + except: + self.critical_error("instantiating toolset %s" % + (toolset_name)) + + def run_probe(self, probe): + '''Execute probe''' + probe_name = probe.attrib['name'] + ret = Element("probe-data", probe_name, source=probe.attrib['source']) + try: + script = open(mktemp(), 'w+') + try: + script.write("#!%s\n" % + (probe.attrib.get('interpreter', '/bin/sh'))) + script.write(probe.text) + script.close() + chmod(script.name, 0755) + ret.text = popen(script.name).read() + finally: + unlink(script.name) + except: + self.critical_error("executing probe %s" % (probe_name)) + return ret + + def critical_error(self, operation): + '''Print tracebacks in unexpected cases''' + print "Traceback information (please include in any bug report):" + (ttype, value, trace) = exc_info() + for line in extract_tb(trace): + print "File %s, line %i, in %s\n %s\n" % (line) + print "%s: %s\n" % (ttype, value) + + self.fatal_error("An unexpected failure occurred in %s" % (operation) ) + + def fatal_error(self, message): + '''Signal a fatal error''' + print "Fatal error: %s" % (message) + raise SystemExit, 1 + + def warning_error(self, message): + '''Warn about a problem but continue''' + print "Warning: %s" % (message) + + def usage_error(self, message): + '''Die because script was called the wrong way''' + print "Usage error: %s" % (message) + self.print_usage() + raise SystemExit, 2 + + def cond_print(self, state, message): + '''Output debugging information''' + if self.setup[state]: + print "bcfg2[%s]: %s" % (state, message) + + def print_usage(self): + ''' Display usage information for bcfg2 ''' + print "bcfg2 usage:" + for arg in self.options.iteritems(): + if self.argumentDescriptions.has_key(arg[0]): + print " -%s %s\t%s" % (arg[1], + self.argumentDescriptions[arg[0]], + self.descriptions[arg[0]]) + else: + print " -%s\t\t\t%s" % (arg[1], self.descriptions[arg[0]]) + + def fill_setup_from_file(self, setup_file, ret): + ''' Read any missing configuration information from a file''' + default = { + 'server': 'http://localhost:6789/', + 'user': 'root', + 'retries': '6' + } + config_locations = { + 'server': ('components', 'bcfg2'), + 'user': ('communication', 'user'), + 'password': ('communication', 'password'), + 'retries': ('communicaton', 'retries') + } + + self.cond_print_setup('debug') + + config_parser = None + + for (key, (section, option)) in config_locations.iteritems(): + try: + if not (ret.has_key(key) and ret[key]): + if config_parser == None: + self.cond_print('debug', "no %s provided, reading setup info from %s" % + (key, setup_file)) + config_parser = ConfigParser() + config_parser.read(setup_file) + try: + ret[key] = config_parser.get(section, option) + except (NoSectionError, NoOptionError): + if default.has_key(key): + ret[key] = default[key] + else: + self.fatal_error( + "%s does not contain a value for %s (in %s)" % + (setup_file, option, section)) + except IOError, io_error: + self.fatal_error("unable to read %s: %s" % + (setup_file, io_error)) + except SystemExit: + raise + except: + self.critical_error("reading config file") - if ret["help"] == True: - print_usage(opt, vopt, descs, argDescs) - raise SystemExit, 0 + def get_setup(self, args): + '''parse options into a dictionary''' - if ret["verbose"] == True: - verboseMode = True + for option in self.options.keys(): + self.setup[option] = False - return ret + gstr = "".join([self.options[option] + + if_then(self.argumentDescriptions.has_key(option), + ':', '') + for option in self.options.keys()]) -if __name__ == '__main__': - # parse command line options - signal.signal(signal.SIGINT, cb_sigint_handler) - options = { - 'v':'verbose', - 'q':'quick', - 'd':'debug', - 'n':'dryrun', - 'B':'build', - 'P':'paranoid', - 'h':'help' - } - doptions = { - 'b':'bundle', - 'f':'file', - 'c':'cache', - 'p':'profile', - 'i':'image', - 'r':'remove' - } - descriptions = { - 'v': "enable verbose output", - 'q': "disable some checksum verification", - 'd': "enable debugging output", - 'n': "do not actually change the system", - 'B': "disable service control (implies -q)", - 'P': "make automatic backups of config files", - 'b': "only configure the given bundle", - 'f': "configure from a file rather than querying the server", - 'c': "store the configuration in a file", - 'p': "assert the given profile for the client", - 'i': "assert the given image for the client", - 'r': "force removal of additional configuration items", - 'h': "print this help message" - } - argumentDescriptions = { - 'b': "<bundle name>", - 'f': "<cache file>", - 'c': "<cache file>", - 'p': "<profile name>", - 'i': "<image name>", - 'r': "(pkgs | svcs | all)" - } - setup = dgetopt(sys.argv[1:], options, doptions, - descriptions, argumentDescriptions) - timeinfo = Element("Times") - - # begin configuration - start = time.time() - - comm = None - if setup['file']: try: - verbose("reading cached configuration from %s" % (setup['file'])) - configfile = open(setup['file'], 'r') - r = configfile.read() - configfile.close() - except IOError: - fatal_error("failed to read cached configuration from: %s" % (setup['file'])) - else: - cf = ConfigParser.ConfigParser() - try: - bcfgConf = '/etc/bcfg2.conf' - verbose("reading setup info from %s" % (bcfgConf)) - cf.read(bcfgConf) - location = cf.get("components", "bcfg2") - user = 'root' - password = cf.get("communication", "password") - proxy = SafeProxy(user, password, 6, location) - except: - fatal_error("unable to read %s" % (bcfgConf)) + ginfo = getopt(args, gstr) + except GetoptError, gerr: + self.usage_error(gerr) + + for (gopt, garg) in ginfo[0]: + option = self.argOptions[gopt[1:]] + if self.argumentDescriptions.has_key(option): + self.setup[option] = garg + else: + self.setup[option] = True + + if (self.setup["file"] != False) and (self.setup["cache"] != False): + self.usage_error("cannot use -f and -c together") + + if self.setup["help"] == True: + self.print_usage() + raise SystemExit, 0 + + if self.setup["setup"]: + setup_file = self.setup["setup"] + else: + setup_file = '/etc/bcfg2.conf' + + self.fill_setup_from_file(setup_file, self.setup) - probedata = proxy.runMethod("probe download", "GetProbes", ()) + def run(self): + ''' Perform client execution phase ''' + times = {} + + # begin configuration + times['start'] = time() - timeinfo.set('probefetch', str(time.time() - start)) + if self.setup['file']: + # read config from file + try: + self.cond_print('debug', "reading cached configuration from %s" % + (self.setup['file'])) + configfile = open(self.setup['file'], 'r') + rawconfig = configfile.read() + configfile.close() + except IOError: + self.fatal_error("failed to read cached configuration from: %s" + % (self.setup['file'])) + else: + # retrieve config from server + proxy = SafeProxy(self.setup, self) - try: - probes = XML(probedata) - except XMLSyntaxError, e: - fatal_error("server returned invalid probe information") + probe_data = proxy.run_method("probe download", "GetProbes", ()) + + times['probe_download'] = time() + + try: + probes = XML(probe_data) + except XMLSyntaxError, syntax_error: + self.fatal_error( + "server returned invalid probe requests: %s" % + (syntax_error)) - # execute probes - try: - probeinfo = [run_probe(x) for x in probes.findall(".//probe")] - except: - fatal_error("bcfg encountered an unknown error running probes") + # execute probes + try: + probe_info = [self.run_probe(probe) + for probe in probes.findall(".//probe")] + except: + self.critical_error("executing probes") - # upload probe responses - proxy.runMethod("probe data upload", "RecvProbeData", (probeinfo, )) + # upload probe responses + proxy.run_method("probe data upload", "RecvProbeData", + (probe_info, )) - cstart = time.time() + times['probe_upload'] = time() + + rawconfig = proxy.run_method("configuration download", "GetConfig", + (self.setup['image'], + self.setup['profile'])) + + times['config_download'] = time() - cfginfo = proxy.runMethod("configuration download", "GetConfig", - (setup['image'], setup['profile'])) + if self.setup['cache']: + try: + open(self.setup['cache'], 'w').write(rawconfig) + except IOError: + self.warning_error("failed to write config cache file %s" % + (self.setup['cache'])) + times['caching'] = time() + + try: + self.config = XML(rawconfig) + except XMLSyntaxError, syntax_error: + self.fatal_error("the configuration could not be parsed: %s" % + (syntax_error)) - timeinfo.set('config', str(time.time() - cstart )) + times['config_parse'] = time() + + if self.config.tag == 'error': + self.fatal_error("server error: %s" % (self.config.text)) - if setup['cache']: + # Get toolset from server try: - open(setup['cache'], 'w').write(cfginfo) - except IOError: - warning_error("failed to write config cache file %s" % (setup['cache'])) + toolset_name = self.config.get('toolset') + except: + self.fatal_error("server did not specify a toolset") - pt = time.time() - try: - cfg = XML(cfginfo) - except XMLSyntaxError, e: - fatal_error("the configuration could not be parsed") + if self.setup['bundle']: + replacement_xml = Element("Configuration", version='2.0') + for child in self.config.getroot().getchildren(): + if ((child.tag == 'Bundle') and + (child.attrib['name'] == self.setup['bundle'])): + replacement_xml.append(child) + self.config = replacement_xml - timeinfo.set('parse', str(time.time() - pt)) - - if cfg.tag == 'error': - fatal_error("server error: %s" % (cfg.text)) - - # Get toolset from server - try: - cfg_toolset = cfg.get('toolset') - except: - fatal_error("server did not specify a toolset") - - if setup['bundle']: - c = Element("Configuration", version='2.0') - for child in cfg.getroot().getchildren(): - if ((child.tag == 'Bundle') and (child.attrib['name'] == setup['bundle'])): - c.append(child) - cfg = c - - # Create toolset handle - client = load_toolset(cfg_toolset, cfg, setup) - - istart = time.time() - # verify state - client.Inventory() - timeinfo.set('inventory', str(time.time() - istart)) - - correct = client.states.values().count(True) - total = len(client.states.values()) - - istart = time.time() + # Create toolset handle + self.load_toolset(toolset_name) + + times['initialization'] = time() + + # verify state + self.toolset.Inventory() + + times['inventory'] = time() - if ((correct < total) or client.pkgwork['remove']): - if client.pkgwork['remove']: - client.CondPrint('verbose', "Extra packages detected") # summarize current state - client.CondPrint('verbose', "--> %s of %s config elements correct" % (correct, total)) + self.toolset.CondDisplayState('verbose', 'initial') # install incorrect aspects of configuration - client.Install() - - client.CondPrint('verbose', "--> %s of %s config elements correct" % - (client.states.values().count(True), total)) - failed = [key for key, value in client.states.iteritems() if not value] - if failed: - client.CondPrint('verbose', "Failing Entries:") - [client.CondPrint('verbose', "%s:%s" % - (key.tag, key.get('name'))) - for key in failed if key.tag != 'Package'] - [client.CondPrint('verbose', "%s:%s-%s" % - (key.tag, key.get('name'), key.get('version', 'unset'))) - for key in failed if key.tag == 'Package'] - else: - client.CondPrint("verbose", "All entries correct") + self.toolset.Install() + + self.toolset.CondDisplayState('verbose', "final") - timeinfo.set('install', str(time.time() - istart)) - timeinfo.set('total', str(time.time() - start)) + times['install'] = time() + times['finished'] = time() - if not setup['file']: - # upload statistics - m = Element("upload-statistics") - stats = client.GenerateStats(__revision__) - stats.append(timeinfo) - m.append(stats) + if not self.setup['file']: + # upload statistics + feedback = Element("upload-statistics") + timeinfo = Element("OpStamps") + for (event, timestamp) in times.iteritems(): + timeinfo.set(event, str(timestamp)) + stats = self.toolset.GenerateStats(__revision__) + stats.append(timeinfo) + feedback.append(stats) - proxy.runMethod("uploading statistics", "RecvStats", (tostring(m),)) + proxy.run_method("uploading statistics", + "RecvStats", (tostring(feedback),)) + + +if __name__ == '__main__': + signal(SIGINT, cb_sigint_handler) + Client(argv[1:]).run() |