From d2be8c33d02eedc6787c6106e9526f916a2234b6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 27 Jun 2013 10:35:47 -0400 Subject: Options: migrated client to new parser --- src/lib/Bcfg2/Client/Client.py | 337 --------------- src/lib/Bcfg2/Client/Frame.py | 558 ------------------------- src/lib/Bcfg2/Client/Proxy.py | 105 +++-- src/lib/Bcfg2/Client/__init__.py | 868 ++++++++++++++++++++++++++++++++++++++- src/sbin/bcfg2 | 26 +- 5 files changed, 935 insertions(+), 959 deletions(-) delete mode 100644 src/lib/Bcfg2/Client/Client.py delete mode 100644 src/lib/Bcfg2/Client/Frame.py diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py deleted file mode 100644 index 994ce7c84..000000000 --- a/src/lib/Bcfg2/Client/Client.py +++ /dev/null @@ -1,337 +0,0 @@ -""" The main Bcfg2 client class """ - -import os -import sys -import stat -import time -import fcntl -import socket -import logging -import tempfile -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Client.XML -import Bcfg2.Client.Proxy -import Bcfg2.Client.Frame -import Bcfg2.Client.Tools -from Bcfg2.Utils import locked, Executor -from Bcfg2.Compat import xmlrpclib -from Bcfg2.version import __version__ - - -class Client(object): - """ The main Bcfg2 client class """ - - def __init__(self): - self.toolset = None - self.tools = None - self.config = None - self._proxy = None - self.setup = Bcfg2.Options.get_option_parser() - - if self.setup['debug']: - level = logging.DEBUG - elif self.setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2', - to_syslog=self.setup['syslog'], - level=level, - to_file=self.setup['logging']) - self.logger = logging.getLogger('bcfg2') - self.logger.debug(self.setup) - - self.cmd = Executor(self.setup['command_timeout']) - - if self.setup['bundle_quick']: - if not self.setup['bundle'] and not self.setup['skipbundle']: - self.logger.error("-Q option requires -b or -B") - raise SystemExit(1) - elif self.setup['remove']: - self.logger.error("-Q option incompatible with -r") - raise SystemExit(1) - if 'drivers' in self.setup and self.setup['drivers'] == 'help': - self.logger.info("The following drivers are available:") - self.logger.info(Bcfg2.Client.Tools.__all__) - raise SystemExit(0) - if self.setup['remove'] and 'services' in self.setup['remove'].lower(): - self.logger.error("Service removal is nonsensical; " - "removed services will only be disabled") - if (self.setup['remove'] and - self.setup['remove'].lower() not in ['all', 'services', 'packages', - 'users']): - self.logger.error("Got unknown argument %s for -r" % - self.setup['remove']) - if self.setup["file"] and self.setup["cache"]: - print("cannot use -f and -c together") - raise SystemExit(1) - if not self.setup['server'].startswith('https://'): - self.setup['server'] = 'https://' + self.setup['server'] - - def _probe_failure(self, probename, msg): - """ handle failure of a probe in the way the user wants us to - (exit or continue) """ - message = "Failed to execute probe %s: %s" % (probename, msg) - if self.setup['probe_exit']: - self.fatal_error(message) - else: - self.logger.error(message) - - def run_probe(self, probe): - """Execute probe.""" - name = probe.get('name') - self.logger.info("Running probe %s" % name) - ret = Bcfg2.Client.XML.Element("probe-data", - name=name, - source=probe.get('source')) - try: - scripthandle, scriptname = tempfile.mkstemp() - script = os.fdopen(scripthandle, 'w') - try: - script.write("#!%s\n" % - (probe.attrib.get('interpreter', '/bin/sh'))) - if sys.hexversion >= 0x03000000: - script.write(probe.text) - else: - script.write(probe.text.encode('utf-8')) - script.close() - os.chmod(scriptname, - stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | - stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | - stat.S_IWUSR) # 0755 - rv = self.cmd.run(scriptname, timeout=self.setup['timeout']) - if rv.stderr: - self.logger.warning("Probe %s has error output: %s" % - (name, rv.stderr)) - if not rv.success: - self._probe_failure(name, "Return value %s" % rv) - self.logger.info("Probe %s has result:" % name) - self.logger.info(rv.stdout) - if sys.hexversion >= 0x03000000: - ret.text = rv.stdout - else: - ret.text = rv.stdout.decode('utf-8') - finally: - os.unlink(scriptname) - except SystemExit: - raise - except: - self._probe_failure(name, sys.exc_info()[1]) - return ret - - def fatal_error(self, message): - """Signal a fatal error.""" - self.logger.error("Fatal error: %s" % (message)) - raise SystemExit(1) - - @property - def proxy(self): - """ get an XML-RPC proxy to the server """ - if self._proxy is None: - self._proxy = Bcfg2.Client.Proxy.ComponentProxy( - self.setup['server'], - self.setup['user'], - self.setup['password'], - key=self.setup['key'], - cert=self.setup['certificate'], - ca=self.setup['ca'], - allowedServerCNs=self.setup['serverCN'], - timeout=self.setup['timeout'], - retries=int(self.setup['retries']), - delay=int(self.setup['retry_delay'])) - return self._proxy - - def run_probes(self, times=None): - """ run probes and upload probe data """ - if times is None: - times = dict() - - try: - probes = Bcfg2.Client.XML.XML(str(self.proxy.GetProbes())) - except (Bcfg2.Client.Proxy.ProxyError, - Bcfg2.Client.Proxy.CertificateError, - socket.gaierror, - socket.error): - err = sys.exc_info()[1] - self.fatal_error("Failed to download probes from bcfg2: %s" % err) - except Bcfg2.Client.XML.ParseError: - err = sys.exc_info()[1] - self.fatal_error("Server returned invalid probe requests: %s" % - err) - - times['probe_download'] = time.time() - - # execute probes - probedata = Bcfg2.Client.XML.Element("ProbeData") - for probe in probes.findall(".//probe"): - probedata.append(self.run_probe(probe)) - - if len(probes.findall(".//probe")) > 0: - try: - # upload probe responses - self.proxy.RecvProbeData( - Bcfg2.Client.XML.tostring( - probedata, - xml_declaration=False).decode('utf-8')) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to upload probe data: %s" % err) - - times['probe_upload'] = time.time() - - def get_config(self, times=None): - """ load the configuration, either from the cached - configuration file (-f), or from the server """ - if times is None: - times = dict() - - if self.setup['file']: - # read config from file - try: - self.logger.debug("Reading cached configuration from %s" % - self.setup['file']) - return open(self.setup['file'], 'r').read() - except IOError: - self.fatal_error("Failed to read cached configuration from: %s" - % (self.setup['file'])) - else: - # retrieve config from server - if self.setup['profile']: - try: - self.proxy.AssertProfile(self.setup['profile']) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to set client profile: %s" % err) - - try: - self.proxy.DeclareVersion(__version__) - except xmlrpclib.Fault: - err = sys.exc_info()[1] - if (err.faultCode == xmlrpclib.METHOD_NOT_FOUND or - (err.faultCode == 7 and - err.faultString.startswith("Unknown method"))): - self.logger.debug("Server does not support declaring " - "client version") - else: - self.logger.error("Failed to declare version: %s" % err) - except (Bcfg2.Client.Proxy.ProxyError, - Bcfg2.Client.Proxy.CertificateError, - socket.gaierror, - socket.error): - err = sys.exc_info()[1] - self.logger.error("Failed to declare version: %s" % err) - - self.run_probes(times=times) - - if self.setup['decision'] in ['whitelist', 'blacklist']: - try: - self.setup['decision_list'] = \ - self.proxy.GetDecisionList(self.setup['decision']) - self.logger.info("Got decision list from server:") - self.logger.info(self.setup['decision_list']) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to get decision list: %s" % err) - - try: - rawconfig = self.proxy.GetConfig().encode('utf-8') - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.fatal_error("Failed to download configuration from " - "Bcfg2: %s" % err) - - times['config_download'] = time.time() - return rawconfig - - def run(self): - """Perform client execution phase.""" - times = {} - - # begin configuration - times['start'] = time.time() - - self.logger.info("Starting Bcfg2 client run at %s" % times['start']) - - rawconfig = self.get_config(times=times).decode('utf-8') - - if self.setup['cache']: - try: - open(self.setup['cache'], 'w').write(rawconfig) - os.chmod(self.setup['cache'], 33152) - except IOError: - self.logger.warning("Failed to write config cache file %s" % - (self.setup['cache'])) - times['caching'] = time.time() - - try: - self.config = Bcfg2.Client.XML.XML(rawconfig) - except Bcfg2.Client.XML.ParseError: - syntax_error = sys.exc_info()[1] - self.fatal_error("The configuration could not be parsed: %s" % - syntax_error) - - times['config_parse'] = time.time() - - if self.config.tag == 'error': - self.fatal_error("Server error: %s" % (self.config.text)) - return(1) - - if self.setup['bundle_quick']: - newconfig = Bcfg2.Client.XML.XML('') - for bundle in self.config.getchildren(): - if (bundle.tag == 'Bundle' and - ((self.setup['bundle'] and - bundle.get('name') in self.setup['bundle']) or - (self.setup['skipbundle'] and - bundle.get('name') not in self.setup['skipbundle']))): - newconfig.append(bundle) - self.config = newconfig - - self.tools = Bcfg2.Client.Frame.Frame(self.config, times) - - if not self.setup['omit_lock_check']: - #check lock here - try: - lockfile = open(self.setup['lockfile'], 'w') - if locked(lockfile.fileno()): - self.fatal_error("Another instance of Bcfg2 is running. " - "If you want to bypass the check, run " - "with the %s option" % - Bcfg2.Options.OMIT_LOCK_CHECK.cmd) - except SystemExit: - raise - except: - lockfile = None - self.logger.error("Failed to open lockfile %s: %s" % - (self.setup['lockfile'], sys.exc_info()[1])) - - # execute the configuration - self.tools.Execute() - - if not self.setup['omit_lock_check']: - # unlock here - if lockfile: - try: - fcntl.lockf(lockfile.fileno(), fcntl.LOCK_UN) - os.remove(self.setup['lockfile']) - except OSError: - self.logger.error("Failed to unlock lockfile %s" % - lockfile.name) - - if not self.setup['file'] and not self.setup['bundle_quick']: - # upload statistics - feedback = self.tools.GenerateStats() - - try: - self.proxy.RecvStats( - Bcfg2.Client.XML.tostring( - feedback, - xml_declaration=False).decode('utf-8')) - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - self.logger.error("Failed to upload configuration statistics: " - "%s" % err) - raise SystemExit(2) - - self.logger.info("Finished Bcfg2 client run at %s" % time.time()) diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py deleted file mode 100644 index a668a0870..000000000 --- a/src/lib/Bcfg2/Client/Frame.py +++ /dev/null @@ -1,558 +0,0 @@ -""" Frame is the Client Framework that verifies and installs entries, -and generates statistics. """ - -import time -import fnmatch -import logging -import Bcfg2.Client.Tools -from Bcfg2.Client import prompt -from Bcfg2.Options import get_option_parser -from Bcfg2.Compat import any, all, cmp # pylint: disable=W0622 - - -def cmpent(ent1, ent2): - """Sort entries.""" - if ent1.tag != ent2.tag: - return cmp(ent1.tag, ent2.tag) - else: - return cmp(ent1.get('name'), ent2.get('name')) - - -def matches_entry(entryspec, entry): - """ Determine if the Decisions-style entry specification matches - the entry. Both are tuples of (tag, name). The entryspec can - handle the wildcard * in either position. """ - if entryspec == entry: - return True - return all(fnmatch.fnmatch(entry[i], entryspec[i]) for i in [0, 1]) - - -def matches_white_list(entry, whitelist): - """ Return True if (, ) is in the given - whitelist. """ - return any(matches_entry(we, (entry.tag, entry.get('name'))) - for we in whitelist) - - -def passes_black_list(entry, blacklist): - """ Return True if (, ) is not in the given - blacklist. """ - return not any(matches_entry(be, (entry.tag, entry.get('name'))) - for be in blacklist) - - -# pylint: disable=W0702 -# in frame we frequently want to catch all exceptions, regardless of -# type, so disable the pylint rule that catches that. - - -class Frame(object): - """Frame is the container for all Tool objects and state information.""" - - def __init__(self, config, times): - self.setup = get_option_parser() - self.config = config - self.times = times - self.dryrun = self.setup['dryrun'] - self.times['initialization'] = time.time() - self.tools = [] - - #: A dict of the state of each entry. Keys are the entries. - #: Values are boolean: True means that the entry is good, - #: False means that the entry is bad. - self.states = {} - self.whitelist = [] - self.blacklist = [] - self.removal = [] - self.logger = logging.getLogger(__name__) - drivers = self.setup['drivers'] - for driver in drivers[:]: - if (driver not in Bcfg2.Client.Tools.__all__ and - isinstance(driver, str)): - self.logger.error("Tool driver %s is not available" % driver) - drivers.remove(driver) - - tclass = {} - for tool in drivers: - if not isinstance(tool, str): - tclass[time.time()] = tool - tool_class = "Bcfg2.Client.Tools.%s" % tool - try: - tclass[tool] = getattr(__import__(tool_class, globals(), - locals(), ['*']), - tool) - except ImportError: - continue - except: - self.logger.error("Tool %s unexpectedly failed to load" % tool, - exc_info=1) - - for tool in list(tclass.values()): - try: - self.tools.append(tool(config)) - except Bcfg2.Client.Tools.ToolInstantiationError: - continue - except: - self.logger.error("Failed to instantiate tool %s" % tool, - exc_info=1) - - for tool in self.tools[:]: - for conflict in getattr(tool, 'conflicts', []): - for item in self.tools: - if item.name == conflict: - self.tools.remove(item) - - self.logger.info("Loaded tool drivers:") - self.logger.info([tool.name for tool in self.tools]) - - deprecated = [tool.name for tool in self.tools if tool.deprecated] - if deprecated: - self.logger.warning("Loaded deprecated tool drivers:") - self.logger.warning(deprecated) - experimental = [tool.name for tool in self.tools if tool.experimental] - if experimental: - self.logger.info("Loaded experimental tool drivers:") - self.logger.info(experimental) - - # find entries not handled by any tools - self.unhandled = [entry for struct in config - for entry in struct - if entry not in self.handled] - - if self.unhandled: - self.logger.error("The following entries are not handled by any " - "tool:") - for entry in self.unhandled: - self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'), - entry.get('name'))) - - self.find_dups(config) - - pkgs = [(entry.get('name'), entry.get('origin')) - for struct in config - for entry in struct - if entry.tag == 'Package'] - if pkgs: - self.logger.debug("The following packages are specified in bcfg2:") - self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None]) - self.logger.debug("The following packages are prereqs added by " - "Packages:") - self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) - - def find_dups(self, config): - """ Find duplicate entries and warn about them """ - entries = dict() - for struct in config: - for entry in struct: - for tool in self.tools: - if tool.handlesEntry(entry): - pkey = tool.primarykey(entry) - if pkey in entries: - entries[pkey] += 1 - else: - entries[pkey] = 1 - multi = [e for e, c in entries.items() if c > 1] - if multi: - self.logger.debug("The following entries are included multiple " - "times:") - for entry in multi: - self.logger.debug(entry) - - def promptFilter(self, msg, entries): - """Filter a supplied list based on user input.""" - ret = [] - entries.sort(key=lambda e: e.tag + ":" + e.get('name')) - for entry in entries[:]: - if entry in self.unhandled: - # don't prompt for entries that can't be installed - continue - if 'qtext' in entry.attrib: - iprompt = entry.get('qtext') - else: - iprompt = msg % (entry.tag, entry.get('name')) - if prompt(iprompt): - ret.append(entry) - return ret - - def __getattr__(self, name): - if name in ['extra', 'handled', 'modified', '__important__']: - ret = [] - for tool in self.tools: - ret += getattr(tool, name) - return ret - elif name in self.__dict__: - return self.__dict__[name] - raise AttributeError(name) - - def InstallImportant(self): - """Install important entries - - We also process the decision mode stuff here because we want to prevent - non-whitelisted/blacklisted 'important' entries from being installed - prior to determining the decision mode on the client. - """ - # Need to process decision stuff early so that dryrun mode - # works with it - self.whitelist = [entry for entry in self.states - if not self.states[entry]] - if not self.setup['file']: - if self.setup['decision'] == 'whitelist': - dwl = self.setup['decision_list'] - w_to_rem = [e for e in self.whitelist - if not matches_white_list(e, dwl)] - if w_to_rem: - self.logger.info("In whitelist mode: " - "suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in w_to_rem]) - self.whitelist = [x for x in self.whitelist - if x not in w_to_rem] - elif self.setup['decision'] == 'blacklist': - b_to_rem = \ - [e for e in self.whitelist - if not passes_black_list(e, self.setup['decision_list'])] - if b_to_rem: - self.logger.info("In blacklist mode: " - "suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in b_to_rem]) - self.whitelist = [x for x in self.whitelist - if x not in b_to_rem] - - # take care of important entries first - if not self.dryrun: - for parent in self.config.findall(".//Path/.."): - if ((parent.tag == "Bundle" and - ((self.setup['bundle'] and - parent.get("name") not in self.setup['bundle']) or - (self.setup['skipbundle'] and - parent.get("name") in self.setup['skipbundle']))) or - (parent.tag == "Independent" and - (self.setup['bundle'] or self.setup['skipindep']))): - continue - for cfile in parent.findall("./Path"): - if (cfile.get('name') not in self.__important__ or - cfile.get('type') != 'file' or - cfile not in self.whitelist): - continue - tools = [t for t in self.tools - if t.handlesEntry(cfile) and t.canVerify(cfile)] - if not tools: - continue - if (self.setup['interactive'] and not - self.promptFilter("Install %s: %s? (y/N):", [cfile])): - self.whitelist.remove(cfile) - continue - try: - self.states[cfile] = tools[0].InstallPath(cfile) - if self.states[cfile]: - tools[0].modified.append(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) - cfile.set('qtext', '') - if tools[0].VerifyPath(cfile, []): - self.whitelist.remove(cfile) - - def Inventory(self): - """ - Verify all entries, - find extra entries, - and build up workqueues - - """ - # initialize all states - for struct in self.config.getchildren(): - for entry in struct.getchildren(): - self.states[entry] = False - for tool in self.tools: - try: - self.states.update(tool.Inventory()) - except: - self.logger.error("%s.Inventory() call failed:" % tool.name, - exc_info=1) - - def Decide(self): # pylint: disable=R0912 - """Set self.whitelist based on user interaction.""" - iprompt = "Install %s: %s? (y/N): " - rprompt = "Remove %s: %s? (y/N): " - if self.setup['remove']: - if self.setup['remove'] == 'all': - self.removal = self.extra - elif self.setup['remove'].lower() == 'services': - self.removal = [entry for entry in self.extra - if entry.tag == 'Service'] - elif self.setup['remove'].lower() == 'packages': - self.removal = [entry for entry in self.extra - if entry.tag == 'Package'] - elif self.setup['remove'].lower() == 'users': - self.removal = [entry for entry in self.extra - if entry.tag in ['POSIXUser', 'POSIXGroup']] - - candidates = [entry for entry in self.states - if not self.states[entry]] - - if self.dryrun: - if self.whitelist: - self.logger.info("In dryrun mode: " - "suppressing entry installation for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) - for entry in self.whitelist]) - self.whitelist = [] - if self.removal: - self.logger.info("In dryrun mode: " - "suppressing entry removal for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) - for entry in self.removal]) - self.removal = [] - - # Here is where most of the work goes - # first perform bundle filtering - all_bundle_names = [b.get('name') - for b in self.config.findall('./Bundle')] - bundles = self.config.getchildren() - if self.setup['bundle']: - # warn if non-existent bundle given - for bundle in self.setup['bundle']: - if bundle not in all_bundle_names: - self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = [b for b in bundles - if b.get('name') in self.setup['bundle']] - elif self.setup['indep']: - bundles = [b for b in bundles if b.tag != 'Bundle'] - if self.setup['skipbundle']: - # warn if non-existent bundle given - if not self.setup['bundle_quick']: - for bundle in self.setup['skipbundle']: - if bundle not in all_bundle_names: - self.logger.info("Warning: Bundle %s not found" % - bundle) - bundles = [b for b in bundles - if b.get('name') not in self.setup['skipbundle']] - if self.setup['skipindep']: - bundles = [b for b in bundles if b.tag == 'Bundle'] - - self.whitelist = [e for e in self.whitelist - if any(e in b for b in bundles)] - - # first process prereq actions - for bundle in bundles[:]: - if bundle.tag == 'Bundle': - bmodified = any(item in self.whitelist for item in bundle) - else: - bmodified = False - actions = [a for a in bundle.findall('./Action') - if (a.get('timing') in ['pre', 'both'] and - (bmodified or a.get('when') == 'always'))] - # now we process all "always actions" - if self.setup['interactive']: - self.promptFilter(iprompt, actions) - self.DispatchInstallCalls(actions) - - if bundle.tag != 'Bundle': - continue - - # need to test to fail entries in whitelist - if not all(self.states[a] for a in actions): - # then display bundles forced off with entries - self.logger.info("%s %s failed prerequisite action" % - (bundle.tag, bundle.get('name'))) - bundles.remove(bundle) - b_to_remv = [ent for ent in self.whitelist if ent in bundle] - if b_to_remv: - self.logger.info("Not installing entries from %s %s" % - (bundle.tag, bundle.get('name'))) - self.logger.info(["%s:%s" % (e.tag, e.get('name')) - for e in b_to_remv]) - for ent in b_to_remv: - self.whitelist.remove(ent) - - self.logger.debug("Installing entries in the following bundle(s):") - self.logger.debug(" %s" % ", ".join(b.get("name") for b in bundles - if b.get("name"))) - - if self.setup['interactive']: - self.whitelist = self.promptFilter(iprompt, self.whitelist) - self.removal = self.promptFilter(rprompt, self.removal) - - for entry in candidates: - if entry not in self.whitelist: - self.blacklist.append(entry) - - def DispatchInstallCalls(self, entries): - """Dispatch install calls to underlying tools.""" - for tool in self.tools: - handled = [entry for entry in entries if tool.canInstall(entry)] - if not handled: - continue - try: - self.states.update(tool.Install(handled)) - except: - self.logger.error("%s.Install() call failed:" % tool.name, - exc_info=1) - - def Install(self): - """Install all entries.""" - self.DispatchInstallCalls(self.whitelist) - mods = self.modified - mbundles = [struct for struct in self.config.findall('Bundle') - if any(True for mod in mods if mod in struct)] - - if self.modified: - # Handle Bundle interdeps - if mbundles: - self.logger.info("The Following Bundles have been modified:") - self.logger.info([mbun.get('name') for mbun in mbundles]) - tbm = [(t, b) for t in self.tools for b in mbundles] - for tool, bundle in tbm: - try: - self.states.update(tool.Inventory(structures=[bundle])) - except: - self.logger.error("%s.Inventory() call failed:" % - tool.name, - exc_info=1) - clobbered = [entry for bundle in mbundles for entry in bundle - if (not self.states[entry] and - entry not in self.blacklist)] - if clobbered: - self.logger.debug("Found clobbered entries:") - self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) - for entry in clobbered]) - if not self.setup['interactive']: - self.DispatchInstallCalls(clobbered) - - for bundle in self.config.findall('.//Bundle'): - if (self.setup['bundle'] and - bundle.get('name') not in self.setup['bundle']): - # prune out unspecified bundles when running with -b - continue - if bundle in mbundles: - self.logger.debug("Bundle %s was modified" % - bundle.get('name')) - func = "BundleUpdated" - else: - self.logger.debug("Bundle %s was not modified" % - bundle.get('name')) - func = "BundleNotUpdated" - for tool in self.tools: - try: - self.states.update(getattr(tool, func)(bundle)) - except: - self.logger.error("%s.%s(%s:%s) call failed:" % - (tool.name, func, bundle.tag, - bundle.get("name")), exc_info=1) - - for indep in self.config.findall('.//Independent'): - for tool in self.tools: - try: - self.states.update(tool.BundleNotUpdated(indep)) - except: - self.logger.error("%s.BundleNotUpdated(%s:%s) call failed:" - % (tool.name, indep.tag, - indep.get("name")), exc_info=1) - - def Remove(self): - """Remove extra entries.""" - for tool in self.tools: - extras = [entry for entry in self.removal - if tool.handlesEntry(entry)] - if extras: - try: - tool.Remove(extras) - except: - self.logger.error("%s.Remove() failed" % tool.name, - exc_info=1) - - def CondDisplayState(self, phase): - """Conditionally print tracing information.""" - self.logger.info('Phase: %s' % phase) - self.logger.info('Correct entries: %d' % - list(self.states.values()).count(True)) - self.logger.info('Incorrect entries: %d' % - list(self.states.values()).count(False)) - if phase == 'final' and list(self.states.values()).count(False): - for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + - e.get('name')): - if not self.states[entry]: - etype = entry.get('type') - if etype: - self.logger.info("%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) - else: - self.logger.info("%s:%s" % (entry.tag, - entry.get('name'))) - self.logger.info('Total managed entries: %d' % - len(list(self.states.values()))) - self.logger.info('Unmanaged entries: %d' % len(self.extra)) - if phase == 'final' and self.setup['extra']: - for entry in sorted(self.extra, key=lambda e: e.tag + ":" + - e.get('name')): - etype = entry.get('type') - if etype: - self.logger.info("%s:%s:%s" % (entry.tag, etype, - entry.get('name'))) - else: - self.logger.info("%s:%s" % (entry.tag, - entry.get('name'))) - - if ((list(self.states.values()).count(False) == 0) and not self.extra): - self.logger.info('All entries correct.') - - def ReInventory(self): - """Recheck everything.""" - if not self.dryrun and self.setup['kevlar']: - self.logger.info("Rechecking system inventory") - self.Inventory() - - def Execute(self): - """Run all methods.""" - self.Inventory() - self.times['inventory'] = time.time() - self.CondDisplayState('initial') - self.InstallImportant() - self.Decide() - self.Install() - self.times['install'] = time.time() - self.Remove() - self.times['remove'] = time.time() - if self.modified: - self.ReInventory() - self.times['reinventory'] = time.time() - self.times['finished'] = time.time() - self.CondDisplayState('final') - - def GenerateStats(self): - """Generate XML summary of execution statistics.""" - feedback = Bcfg2.Client.XML.Element("upload-statistics") - stats = Bcfg2.Client.XML.SubElement( - feedback, - 'Statistics', - total=str(len(self.states)), - version='2.0', - revision=self.config.get('revision', '-1')) - good_entries = [key for key, val in list(self.states.items()) if val] - good = len(good_entries) - stats.set('good', str(good)) - if any(not val for val in list(self.states.values())): - stats.set('state', 'dirty') - else: - stats.set('state', 'clean') - - # List bad elements of the configuration - for (data, ename) in [(self.modified, 'Modified'), - (self.extra, "Extra"), - (good_entries, "Good"), - ([entry for entry in self.states - if not self.states[entry]], "Bad")]: - container = Bcfg2.Client.XML.SubElement(stats, ename) - for item in data: - item.set('qtext', '') - container.append(item) - item.text = None - - timeinfo = Bcfg2.Client.XML.Element("OpStamps") - feedback.append(stats) - for (event, timestamp) in list(self.times.items()): - timeinfo.set(event, str(timestamp)) - stats.append(timeinfo) - return feedback diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index fbf114de6..98d081b10 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -1,6 +1,10 @@ import re +import sys +import time import socket import logging +import Bcfg2.Options +from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus # The ssl module is provided by either Python 2.6 or a separate ssl # package that works on older versions of Python (see @@ -16,11 +20,6 @@ except ImportError: SSL_LIB = 'm2crypto' SSL_ERROR = SSL.SSLError -import sys -import time - -# Compatibility imports -from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) @@ -64,6 +63,7 @@ class CertificateError(Exception): _orig_Method = xmlrpclib._Method + class RetryMethod(xmlrpclib._Method): """Method with error handling and retries built in.""" log = logging.getLogger('xmlrpc') @@ -104,7 +104,6 @@ class RetryMethod(xmlrpclib._Method): err = sys.exc_info()[1] msg = err except: - raise etype, err = sys.exc_info()[:2] msg = "Unknown failure: %s (%s)" % (err, etype.__name__) if msg: @@ -218,12 +217,15 @@ class SSLHTTPConnection(httplib.HTTPConnection): other_side_required = ssl.CERT_REQUIRED else: other_side_required = ssl.CERT_NONE - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") + self.logger.warning("No ca is specified. Cannot authenticate the " + "server with SSL.") if self.cert and not self.key: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot " + "authenticate this client with SSL.") self.cert = None if self.key and not self.cert: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") + self.logger.warning("SSL key specfied, but no cert. Cannot " + "authenticate this client with SSL.") self.key = None rawsock.settimeout(self.timeout) @@ -234,7 +236,8 @@ class SSLHTTPConnection(httplib.HTTPConnection): self.sock.connect((self.host, self.port)) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: - scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] + scn = [x[0][1] for x in peer_cert['subject'] + if x[0][0] == 'commonName'][0] if scn not in self.scns: raise CertificateError(scn) self.sock.closeSocket = True @@ -253,20 +256,24 @@ class SSLHTTPConnection(httplib.HTTPConnection): if self.ca: # Use the certificate authority to validate the cert # presented by the server - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, + depth=9) if ctx.load_verify_locations(self.ca) != 1: raise Exception('No CA certs') else: - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") + self.logger.warning("No ca is specified. Cannot authenticate the " + "server with SSL.") if self.cert and self.key: # A cert/key is defined, use them to support client # authentication to the server ctx.load_cert(self.cert, self.key) elif self.cert: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot " + "authenticate this client with SSL.") elif self.key: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") + self.logger.warning("SSL key specfied, but no cert. Cannot " + "authenticate this client with SSL.") self.sock = SSL.Connection(ctx) if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', self.host): @@ -343,26 +350,50 @@ class XMLRPCTransport(xmlrpclib.Transport): # pylint: enable=E1101 -def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, - allowedServerCNs=None, timeout=90, retries=3, delay=1): - - """Constructs proxies to components. - - Arguments: - component_name -- name of the component to connect to - - Additional arguments are passed to the ServerProxy constructor. - - """ - xmlrpclib._Method.max_retries = retries - xmlrpclib._Method.retry_delay = delay - - if user and password: - method, path = urlparse(url)[:2] - newurl = "%s://%s:%s@%s" % (method, quote_plus(user, ''), - quote_plus(password, ''), path) - else: - newurl = url - ssl_trans = XMLRPCTransport(key, cert, ca, - allowedServerCNs, timeout=float(timeout)) - return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) +class ComponentProxy(xmlrpclib.ServerProxy): + """Constructs proxies to components. """ + + options = [ + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, + Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.password, + Bcfg2.Options.Option( + "-u", "--user", default="root", cf=('communication', 'user'), + help='The user to provide for authentication'), + Bcfg2.Options.Option( + "-R", "--retries", type=int, default=3, + cf=('communication', 'retries'), + help='The number of times to retry network communication'), + Bcfg2.Options.Option( + "-y", "--retry-delay", type=int, default=1, + cf=('communication', 'retry_delay'), + help='The time in seconds to wait between retries'), + Bcfg2.Options.Option( + '--ssl-cns', cf=('communication', 'serverCommonNames'), + type=Bcfg2.Options.Types.colon_list, + help='List of server commonNames'), + Bcfg2.Options.Option( + "-t", "--timeout", type=float, default=90.0, + cf=('communication', 'timeout'), + help='Set the client XML-RPC timeout')] + + def __init__(self): + RetryMethod.max_retries = Bcfg2.Options.setup.retries + RetryMethod.retry_delay = Bcfg2.Options.setup.retry_delay + + if Bcfg2.Options.setup.user and Bcfg2.Options.setup.password: + method, path = urlparse(Bcfg2.Options.setup.server)[:2] + url = "%s://%s:%s@%s" % ( + method, + quote_plus(Bcfg2.Options.setup.user, ''), + quote_plus(Bcfg2.Options.setup.password, ''), + path) + else: + url = Bcfg2.Options.setup.server + ssl_trans = XMLRPCTransport(Bcfg2.Options.setup.key, + Bcfg2.Options.setup.cert, + Bcfg2.Options.setup.ca, + Bcfg2.Options.setup.ssl_cns, + Bcfg2.Options.setup.timeout) + xmlrpclib.ServerProxy.__init__(self, url, + allow_none=True, transport=ssl_trans) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 25603186e..dd32fc45c 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -2,8 +2,55 @@ import os import sys -import select -from Bcfg2.Compat import input # pylint: disable=W0622 +import stat +import time +import fcntl +import socket +import fnmatch +import logging +import argparse +import tempfile +import Bcfg2.Logger +import Bcfg2.Options +import XML +import Proxy +import Tools +from Bcfg2.Utils import locked, Executor, safe_input +from Bcfg2.version import __version__ +# pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, walk_packages, any, all, cmp +# pylint: enable=W0622 + + +def cmpent(ent1, ent2): + """Sort entries.""" + if ent1.tag != ent2.tag: + return cmp(ent1.tag, ent2.tag) + else: + return cmp(ent1.get('name'), ent2.get('name')) + + +def matches_entry(entryspec, entry): + """ Determine if the Decisions-style entry specification matches + the entry. Both are tuples of (tag, name). The entryspec can + handle the wildcard * in either position. """ + if entryspec == entry: + return True + return all(fnmatch.fnmatch(entry[i], entryspec[i]) for i in [0, 1]) + + +def matches_white_list(entry, whitelist): + """ Return True if (, ) is in the given + whitelist. """ + return any(matches_entry(we, (entry.tag, entry.get('name'))) + for we in whitelist) + + +def passes_black_list(entry, blacklist): + """ Return True if (, ) is not in the given + blacklist. """ + return not any(matches_entry(be, (entry.tag, entry.get('name'))) + for be in blacklist) def prompt(msg): @@ -16,11 +63,822 @@ def prompt(msg): contain "[y/N]" if desired, etc. :type msg: string :returns: bool - True if yes, False if no """ - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) try: - ans = input(msg) + ans = safe_input(msg) return ans in ['y', 'Y'] except EOFError: # handle ^C on rhel-based platforms raise SystemExit(1) + except: + print("Error while reading input: %s" % sys.exc_info()[1]) + return False + + +class ClientDriverAction(Bcfg2.Options.ComponentAction): + bases = ['Bcfg2.Client.Tools'] + fail_silently = True + + +class Client(object): + """ The main Bcfg2 client class """ + + options = Proxy.ComponentProxy.options + [ + Bcfg2.Options.Common.syslog, + Bcfg2.Options.Common.location, + Bcfg2.Options.Common.interactive, + Bcfg2.Options.BooleanOption( + "-q", "--quick", help="Disable some checksum verification"), + Bcfg2.Options.Option( + cf=('client', 'probe_timeout'), + type=Bcfg2.Options.Types.timeout, + help="Timeout when running client probes"), + Bcfg2.Options.Option( + "-b", "--only-bundles", default=[], + type=Bcfg2.Options.Types.colon_list, + help='Only configure the given bundle(s)'), + Bcfg2.Options.Option( + "-B", "--except-bundles", default=[], + type=Bcfg2.Options.Types.colon_list, + help='Configure everything except the given bundle(s)'), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "-Q", "--bundle-quick", + help='Only verify the given bundle(s)'), + Bcfg2.Options.Option( + '-r', '--remove', + choices=['all', 'services', 'packages', 'users'], + help='Force removal of additional configuration items')), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.PathOption( + '-f', '--file', type=argparse.FileType('r'), + help='Configure from a file rather than querying the server'), + Bcfg2.Options.PathOption( + '-c', '--cache', type=argparse.FileType('w'), + help='Store the configuration in a file')), + Bcfg2.Options.BooleanOption( + '--exit-on-probe-failure', default=True, + cf=('client', 'exit_on_probe_failure'), + help="The client should exit if a probe fails"), + Bcfg2.Options.Option( + '-p', '--profile', cf=('client', 'profile'), + help='Assert the given profile for the host'), + Bcfg2.Options.Option( + '-l', '--decision', cf=('client', 'decision'), + choices=['whitelist', 'blacklist', 'none'], + help='Run client in server decision list mode'), + Bcfg2.Options.BooleanOption( + "-O", "--no-lock", help='Omit lock check'), + Bcfg2.Options.PathOption( + cf=('components', 'lockfile'), default='/var/lock/bcfg2.run', + help='Client lock file'), + Bcfg2.Options.BooleanOption( + "-n", "--dry-run", help='Do not actually change the system'), + Bcfg2.Options.Option( + "-D", "--drivers", cf=('client', 'drivers'), + type=Bcfg2.Options.Types.comma_list, + default=[m[1] for m in walk_packages(path=Tools.__path__)], + action=ClientDriverAction, help='Client drivers'), + Bcfg2.Options.BooleanOption( + "-e", "--show-extra", help='Enable extra entry output'), + Bcfg2.Options.BooleanOption( + "-k", "--kevlar", help='Run in bulletproof mode')] + + def __init__(self): + self.config = None + self._proxy = None + self.logger = logging.getLogger('bcfg2') + self.cmd = Executor(Bcfg2.Options.setup.probe_timeout) + self.tools = [] + self.times = dict() + self.times['initialization'] = time.time() + + if Bcfg2.Options.setup.bundle_quick: + if (not Bcfg2.Options.setup.only_bundles and + not Bcfg2.Options.setup.except_bundles): + self.logger.error("-Q option requires -b or -B") + raise SystemExit(1) + if Bcfg2.Options.setup.remove == 'services': + self.logger.error("Service removal is nonsensical; " + "removed services will only be disabled") + if not Bcfg2.Options.setup.server.startswith('https://'): + Bcfg2.Options.setup.server = \ + 'https://' + Bcfg2.Options.setup.server + + #: A dict of the state of each entry. Keys are the entries. + #: Values are boolean: True means that the entry is good, + #: False means that the entry is bad. + self.states = {} + self.whitelist = [] + self.blacklist = [] + self.removal = [] + self.logger = logging.getLogger(__name__) + + def _probe_failure(self, probename, msg): + """ handle failure of a probe in the way the user wants us to + (exit or continue) """ + message = "Failed to execute probe %s: %s" % (probename, msg) + if Bcfg2.Options.setup.exit_on_probe_failure: + self.fatal_error(message) + else: + self.logger.error(message) + + def run_probe(self, probe): + """Execute probe.""" + name = probe.get('name') + self.logger.info("Running probe %s" % name) + ret = XML.Element("probe-data", name=name, source=probe.get('source')) + try: + scripthandle, scriptname = tempfile.mkstemp() + script = os.fdopen(scripthandle, 'w') + try: + script.write("#!%s\n" % + (probe.attrib.get('interpreter', '/bin/sh'))) + if sys.hexversion >= 0x03000000: + script.write(probe.text) + else: + script.write(probe.text.encode('utf-8')) + script.close() + os.chmod(scriptname, + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | + stat.S_IWUSR) # 0755 + rv = self.cmd.run(scriptname) + if rv.stderr: + self.logger.warning("Probe %s has error output: %s" % + (name, rv.stderr)) + if not rv.success: + self._probe_failure(name, "Return value %s" % rv.retval) + self.logger.info("Probe %s has result:" % name) + self.logger.info(rv.stdout) + if sys.hexversion >= 0x03000000: + ret.text = rv.stdout + else: + ret.text = rv.stdout.decode('utf-8') + finally: + os.unlink(scriptname) + except SystemExit: + raise + except: + self._probe_failure(name, sys.exc_info()[1]) + return ret + + def fatal_error(self, message): + """Signal a fatal error.""" + self.logger.error("Fatal error: %s" % (message)) + raise SystemExit(1) + + @property + def proxy(self): + """ get an XML-RPC proxy to the server """ + if self._proxy is None: + self._proxy = Proxy.ComponentProxy() + return self._proxy + + def run_probes(self): + """ run probes and upload probe data """ + try: + probes = XML.XML(str(self.proxy.GetProbes())) + except (Proxy.ProxyError, + Proxy.CertificateError, + socket.gaierror, + socket.error): + err = sys.exc_info()[1] + self.fatal_error("Failed to download probes from bcfg2: %s" % err) + except XML.ParseError: + err = sys.exc_info()[1] + self.fatal_error("Server returned invalid probe requests: %s" % + err) + + self.times['probe_download'] = time.time() + + # execute probes + probedata = XML.Element("ProbeData") + for probe in probes.findall(".//probe"): + probedata.append(self.run_probe(probe)) + + if len(probes.findall(".//probe")) > 0: + try: + # upload probe responses + self.proxy.RecvProbeData( + XML.tostring(probedata, + xml_declaration=False).decode('utf-8')) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to upload probe data: %s" % err) + + self.times['probe_upload'] = time.time() + + def get_config(self): + """ load the configuration, either from the cached + configuration file (-f), or from the server """ + if Bcfg2.Options.setup.file: + # read config from file + try: + self.logger.debug("Reading cached configuration from %s" % + Bcfg2.Options.setup.file.name) + return Bcfg2.Options.setup.file.read() + except IOError: + self.fatal_error("Failed to read cached configuration from: %s" + % Bcfg2.Options.setup.file.name) + else: + # retrieve config from server + if Bcfg2.Options.setup.profile: + try: + self.proxy.AssertProfile(Bcfg2.Options.setup.profile) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to set client profile: %s" % err) + + try: + self.proxy.DeclareVersion(__version__) + except xmlrpclib.Fault: + err = sys.exc_info()[1] + if (err.faultCode == xmlrpclib.METHOD_NOT_FOUND or + (err.faultCode == 7 and + err.faultString.startswith("Unknown method"))): + self.logger.debug("Server does not support declaring " + "client version") + else: + self.logger.error("Failed to declare version: %s" % err) + except (Proxy.ProxyError, + Proxy.CertificateError, + socket.gaierror, + socket.error): + err = sys.exc_info()[1] + self.logger.error("Failed to declare version: %s" % err) + + self.run_probes() + + if Bcfg2.Options.setup.decision in ['whitelist', 'blacklist']: + try: + # TODO: read decision list from --decision-list + Bcfg2.Options.setup.decision_list = \ + self.proxy.GetDecisionList( + Bcfg2.Options.setup.decision) + self.logger.info("Got decision list from server:") + self.logger.info(Bcfg2.Options.setup.decision_list) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to get decision list: %s" % err) + + try: + rawconfig = self.proxy.GetConfig().encode('utf-8') + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.fatal_error("Failed to download configuration from " + "Bcfg2: %s" % err) + + self.times['config_download'] = time.time() + + if Bcfg2.Options.setup.cache: + try: + Bcfg2.Options.setup.cache.write(rawconfig) + os.chmod(Bcfg2.Options.setup.cache, 384) # 0600 + except IOError: + self.logger.warning("Failed to write config cache file %s" % + (Bcfg2.Options.setup.cache)) + self.times['caching'] = time.time() + + return rawconfig + + def parse_config(self, rawconfig): + try: + self.config = XML.XML(rawconfig) + except XML.ParseError: + syntax_error = sys.exc_info()[1] + self.fatal_error("The configuration could not be parsed: %s" % + syntax_error) + + self.load_tools() + + # find entries not handled by any tools + self.unhandled = [entry for struct in self.config + for entry in struct + if entry not in self.handled] + + if self.unhandled: + self.logger.error("The following entries are not handled by any " + "tool:") + for entry in self.unhandled: + self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'), + entry.get('name'))) + + # find duplicates + self.find_dups(self.config) + + pkgs = [(entry.get('name'), entry.get('origin')) + for struct in self.config + for entry in struct + if entry.tag == 'Package'] + if pkgs: + self.logger.debug("The following packages are specified in bcfg2:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None]) + self.logger.debug("The following packages are prereqs added by " + "Packages:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) + + self.times['config_parse'] = time.time() + + def run(self): + """Perform client execution phase.""" + # begin configuration + self.times['start'] = time.time() + + self.logger.info("Starting Bcfg2 client run at %s" % + self.times['start']) + + self.parse_config(self.get_config().decode('utf-8')) + + if self.config.tag == 'error': + self.fatal_error("Server error: %s" % (self.config.text)) + + if Bcfg2.Options.setup.bundle_quick: + newconfig = XML.XML('') + for bundle in self.config.getchildren(): + name = bundle.get("name") + if (name and (name in Bcfg2.Options.setup.only_bundles or + name not in Bcfg2.Options.setup.except_bundles)): + newconfig.append(bundle) + self.config = newconfig + + if not Bcfg2.Options.setup.no_lock: + #check lock here + try: + lockfile = open(Bcfg2.Options.setup.lockfile, 'w') + if locked(lockfile.fileno()): + self.fatal_error("Another instance of Bcfg2 is running. " + "If you want to bypass the check, run " + "with the -O/--no-lock option") + except SystemExit: + raise + except: + lockfile = None + self.logger.error("Failed to open lockfile %s: %s" % + (Bcfg2.Options.setup.lockfile, + sys.exc_info()[1])) + + # execute the configuration + self.Execute() + + if not Bcfg2.Options.setup.no_lock: + # unlock here + if lockfile: + try: + fcntl.lockf(lockfile.fileno(), fcntl.LOCK_UN) + os.remove(Bcfg2.Options.setup.lockfile) + except OSError: + self.logger.error("Failed to unlock lockfile %s" % + lockfile.name) + + if (not Bcfg2.Options.setup.file and + not Bcfg2.Options.setup.bundle_quick): + # upload statistics + feedback = self.GenerateStats() + + try: + self.proxy.RecvStats( + XML.tostring(feedback, + xml_declaration=False).decode('utf-8')) + except Proxy.ProxyError: + err = sys.exc_info()[1] + self.logger.error("Failed to upload configuration statistics: " + "%s" % err) + raise SystemExit(2) + + self.logger.info("Finished Bcfg2 client run at %s" % time.time()) + + def load_tools(self): + for tool in Bcfg2.Options.setup.drivers: + try: + self.tools.append(tool(self.config)) + except Tools.ToolInstantiationError: + continue + except: + self.logger.error("Failed to instantiate tool %s" % tool, + exc_info=1) + + for tool in self.tools[:]: + for conflict in getattr(tool, 'conflicts', []): + for item in self.tools: + if item.name == conflict: + self.tools.remove(item) + + self.logger.info("Loaded tool drivers:") + self.logger.info([tool.name for tool in self.tools]) + + deprecated = [tool.name for tool in self.tools if tool.deprecated] + if deprecated: + self.logger.warning("Loaded deprecated tool drivers:") + self.logger.warning(deprecated) + experimental = [tool.name for tool in self.tools if tool.experimental] + if experimental: + self.logger.warning("Loaded experimental tool drivers:") + self.logger.warning(experimental) + + def find_dups(self, config): + """ Find duplicate entries and warn about them """ + entries = dict() + for struct in config: + for entry in struct: + for tool in self.tools: + if tool.handlesEntry(entry): + pkey = tool.primarykey(entry) + if pkey in entries: + entries[pkey] += 1 + else: + entries[pkey] = 1 + multi = [e for e, c in entries.items() if c > 1] + if multi: + self.logger.debug("The following entries are included multiple " + "times:") + for entry in multi: + self.logger.debug(entry) + + def promptFilter(self, msg, entries): + """Filter a supplied list based on user input.""" + ret = [] + entries.sort(key=lambda e: e.tag + ":" + e.get('name')) + for entry in entries[:]: + if entry in self.unhandled: + # don't prompt for entries that can't be installed + continue + if 'qtext' in entry.attrib: + iprompt = entry.get('qtext') + else: + iprompt = msg % (entry.tag, entry.get('name')) + if prompt(iprompt): + ret.append(entry) + return ret + + def __getattr__(self, name): + if name in ['extra', 'handled', 'modified', '__important__']: + ret = [] + for tool in self.tools: + ret += getattr(tool, name) + return ret + elif name in self.__dict__: + return self.__dict__[name] + raise AttributeError(name) + + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode + # works with it + self.whitelist = [entry for entry in self.states + if not self.states[entry]] + if not Bcfg2.Options.setup.file: + if Bcfg2.Options.setup.decision == 'whitelist': + dwl = Bcfg2.Options.setup.decision_list + w_to_rem = [e for e in self.whitelist + if not matches_white_list(e, dwl)] + if w_to_rem: + self.logger.info("In whitelist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in w_to_rem]) + self.whitelist = [x for x in self.whitelist + if x not in w_to_rem] + elif Bcfg2.Options.setup.decision == 'blacklist': + b_to_rem = \ + [e for e in self.whitelist + if not passes_black_list(e, Bcfg2.Options.setup.decision_list)] + if b_to_rem: + self.logger.info("In blacklist mode: " + "suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist + if x not in b_to_rem] + + # take care of important entries first + if not Bcfg2.Options.setup.dry_run: + for parent in self.config.findall(".//Path/.."): + name = parent.get("name") + if (name and (name in Bcfg2.Options.setup.only_bundles or + name not in Bcfg2.Options.setup.except_bundles)): + continue + for cfile in parent.findall("./Path"): + if (cfile.get('name') not in self.__important__ or + cfile.get('type') != 'file' or + cfile not in self.whitelist): + continue + tools = [t for t in self.tools + if t.handlesEntry(cfile) and t.canVerify(cfile)] + if not tools: + continue + if (Bcfg2.Options.setup.interactive and not + self.promptFilter("Install %s: %s? (y/N):", [cfile])): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tools[0].InstallPath(cfile) + if self.states[cfile]: + tools[0].modified.append(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tools[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) + + def Inventory(self): + """ + Verify all entries, + find extra entries, + and build up workqueues + + """ + # initialize all states + for struct in self.config.getchildren(): + for entry in struct.getchildren(): + self.states[entry] = False + for tool in self.tools: + try: + self.states.update(tool.Inventory()) + except: + self.logger.error("%s.Inventory() call failed:" % tool.name, + exc_info=1) + + def Decide(self): # pylint: disable=R0912 + """Set self.whitelist based on user interaction.""" + iprompt = "Install %s: %s? (y/N): " + rprompt = "Remove %s: %s? (y/N): " + if Bcfg2.Options.setup.remove: + if Bcfg2.Options.setup.remove == 'all': + self.removal = self.extra + elif Bcfg2.Options.setup.remove.lower() == 'services': + self.removal = [entry for entry in self.extra + if entry.tag == 'Service'] + elif Bcfg2.Options.setup.remove.lower() == 'packages': + self.removal = [entry for entry in self.extra + if entry.tag == 'Package'] + elif Bcfg2.Options.setup.remove.lower() == 'users': + self.removal = [entry for entry in self.extra + if entry.tag in ['POSIXUser', 'POSIXGroup']] + + candidates = [entry for entry in self.states + if not self.states[entry]] + + if Bcfg2.Options.setup.dry_run: + if self.whitelist: + self.logger.info("In dryrun mode: " + "suppressing entry installation for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.whitelist]) + self.whitelist = [] + if self.removal: + self.logger.info("In dryrun mode: " + "suppressing entry removal for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.removal]) + self.removal = [] + + # Here is where most of the work goes + # first perform bundle filtering + all_bundle_names = [b.get('name') + for b in self.config.findall('./Bundle')] + bundles = self.config.getchildren() + if Bcfg2.Options.setup.only_bundles: + # warn if non-existent bundle given + for bundle in Bcfg2.Options.setup.only_bundles: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % bundle) + bundles = [b for b in bundles + if b.get('name') in Bcfg2.Options.setup.only_bundles] + if Bcfg2.Options.setup.except_bundles: + # warn if non-existent bundle given + if not Bcfg2.Options.setup.bundle_quick: + for bundle in Bcfg2.Options.setup.except_bundles: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % + bundle) + bundles = [ + b for b in bundles + if b.get('name') not in Bcfg2.Options.setup.except_bundles] + self.whitelist = [e for e in self.whitelist + if any(e in b for b in bundles)] + + # first process prereq actions + for bundle in bundles[:]: + if bundle.tag == 'Bundle': + bmodified = any(item in self.whitelist for item in bundle) + else: + bmodified = False + actions = [a for a in bundle.findall('./Action') + if (a.get('timing') in ['pre', 'both'] and + (bmodified or a.get('when') == 'always'))] + # now we process all "always actions" + if Bcfg2.Options.setup.interactive: + self.promptFilter(iprompt, actions) + self.DispatchInstallCalls(actions) + + if bundle.tag != 'Bundle': + continue + + # need to test to fail entries in whitelist + if not all(self.states[a] for a in actions): + # then display bundles forced off with entries + self.logger.info("%s %s failed prerequisite action" % + (bundle.tag, bundle.get('name'))) + bundles.remove(bundle) + b_to_remv = [ent for ent in self.whitelist if ent in bundle] + if b_to_remv: + self.logger.info("Not installing entries from %s %s" % + (bundle.tag, bundle.get('name'))) + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_remv]) + for ent in b_to_remv: + self.whitelist.remove(ent) + + self.logger.debug("Installing entries in the following bundle(s):") + self.logger.debug(" %s" % ", ".join(b.get("name") for b in bundles + if b.get("name"))) + + if Bcfg2.Options.setup.interactive: + self.whitelist = self.promptFilter(iprompt, self.whitelist) + self.removal = self.promptFilter(rprompt, self.removal) + + for entry in candidates: + if entry not in self.whitelist: + self.blacklist.append(entry) + + def DispatchInstallCalls(self, entries): + """Dispatch install calls to underlying tools.""" + for tool in self.tools: + handled = [entry for entry in entries if tool.canInstall(entry)] + if not handled: + continue + try: + self.states.update(tool.Install(handled)) + except: + self.logger.error("%s.Install() call failed:" % tool.name, + exc_info=1) + + def Install(self): + """Install all entries.""" + self.DispatchInstallCalls(self.whitelist) + mods = self.modified + mbundles = [struct for struct in self.config.findall('Bundle') + if any(True for mod in mods if mod in struct)] + + if self.modified: + # Handle Bundle interdeps + if mbundles: + self.logger.info("The Following Bundles have been modified:") + self.logger.info([mbun.get('name') for mbun in mbundles]) + tbm = [(t, b) for t in self.tools for b in mbundles] + for tool, bundle in tbm: + try: + self.states.update(tool.Inventory(structures=[bundle])) + except: + self.logger.error("%s.Inventory() call failed:" % + tool.name, + exc_info=1) + clobbered = [entry for bundle in mbundles for entry in bundle + if (not self.states[entry] and + entry not in self.blacklist)] + if clobbered: + self.logger.debug("Found clobbered entries:") + self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) + for entry in clobbered]) + if not Bcfg2.Options.setup.interactive: + self.DispatchInstallCalls(clobbered) + + for bundle in self.config.findall('.//Bundle'): + if (Bcfg2.Options.setup.only_bundles and + bundle.get('name') not in Bcfg2.Options.setup.only_bundles): + # prune out unspecified bundles when running with -b + continue + if bundle in mbundles: + self.logger.debug("Bundle %s was modified" % + bundle.get('name')) + func = "BundleUpdated" + else: + self.logger.debug("Bundle %s was not modified" % + bundle.get('name')) + func = "BundleNotUpdated" + for tool in self.tools: + try: + self.states.update(getattr(tool, func)(bundle)) + except: + self.logger.error("%s.%s(%s:%s) call failed:" % + (tool.name, func, bundle.tag, + bundle.get("name")), exc_info=1) + + for indep in self.config.findall('.//Independent'): + for tool in self.tools: + try: + self.states.update(tool.BundleNotUpdated(indep)) + except: + self.logger.error("%s.BundleNotUpdated(%s:%s) call failed:" + % (tool.name, indep.tag, + indep.get("name")), exc_info=1) + + def Remove(self): + """Remove extra entries.""" + for tool in self.tools: + extras = [entry for entry in self.removal + if tool.handlesEntry(entry)] + if extras: + try: + tool.Remove(extras) + except: + self.logger.error("%s.Remove() failed" % tool.name, + exc_info=1) + + def CondDisplayState(self, phase): + """Conditionally print tracing information.""" + self.logger.info('Phase: %s' % phase) + self.logger.info('Correct entries: %d' % + list(self.states.values()).count(True)) + self.logger.info('Incorrect entries: %d' % + list(self.states.values()).count(False)) + if phase == 'final' and list(self.states.values()).count(False): + for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + + e.get('name')): + if not self.states[entry]: + etype = entry.get('type') + if etype: + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) + else: + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) + self.logger.info('Total managed entries: %d' % + len(list(self.states.values()))) + self.logger.info('Unmanaged entries: %d' % len(self.extra)) + if phase == 'final' and Bcfg2.Options.setup.show_extra: + for entry in sorted(self.extra, + key=lambda e: e.tag + ":" + e.get('name')): + etype = entry.get('type') + if etype: + self.logger.info("%s:%s:%s" % (entry.tag, etype, + entry.get('name'))) + else: + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) + + if ((list(self.states.values()).count(False) == 0) and not self.extra): + self.logger.info('All entries correct.') + + def ReInventory(self): + """Recheck everything.""" + if not Bcfg2.Options.setup.dry_run and Bcfg2.Options.setup.kevlar: + self.logger.info("Rechecking system inventory") + self.Inventory() + + def Execute(self): + """Run all methods.""" + self.Inventory() + self.times['inventory'] = time.time() + self.CondDisplayState('initial') + self.InstallImportant() + self.Decide() + self.Install() + self.times['install'] = time.time() + self.Remove() + self.times['remove'] = time.time() + if self.modified: + self.ReInventory() + self.times['reinventory'] = time.time() + self.times['finished'] = time.time() + self.CondDisplayState('final') + + def GenerateStats(self): + """Generate XML summary of execution statistics.""" + feedback = XML.Element("upload-statistics") + stats = XML.SubElement(feedback, + 'Statistics', total=str(len(self.states)), + version='2.0', + revision=self.config.get('revision', '-1')) + good_entries = [key for key, val in list(self.states.items()) if val] + good = len(good_entries) + stats.set('good', str(good)) + if any(not val for val in list(self.states.values())): + stats.set('state', 'dirty') + else: + stats.set('state', 'clean') + + # List bad elements of the configuration + for (data, ename) in [(self.modified, 'Modified'), + (self.extra, "Extra"), + (good_entries, "Good"), + ([entry for entry in self.states + if not self.states[entry]], "Bad")]: + container = XML.SubElement(stats, ename) + for item in data: + item.set('qtext', '') + container.append(item) + item.text = None + + timeinfo = XML.Element("OpStamps") + feedback.append(stats) + for (event, timestamp) in list(self.times.items()): + timeinfo.set(event, str(timestamp)) + stats.append(timeinfo) + return feedback diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 62f749b80..eca7c3395 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -2,27 +2,9 @@ """Bcfg2 Client""" import sys -import signal -from Bcfg2.Client.Client import Client -from Bcfg2.Options import load_option_parser, CLIENT_COMMON_OPTIONS - - -def cb_sigint_handler(signum, frame): - """ Exit upon CTRL-C. """ - raise SystemExit(1) - - -def main(): - setup = load_option_parser(CLIENT_COMMON_OPTIONS) - setup.parse(sys.argv[1:]) - - if setup['args']: - print("Bcfg2 takes no arguments, only options") - print(setup.buildHelpMessage()) - raise SystemExit(1) - - signal.signal(signal.SIGINT, cb_sigint_handler) - return Client().run() +from Bcfg2.Options import get_parser +from Bcfg2.Client import Client if __name__ == '__main__': - sys.exit(main()) + get_parser("Bcfg2 client", components=[Client]).parse() + sys.exit(Client().run()) -- cgit v1.2.3-1-g7c22