diff options
Diffstat (limited to 'src/lib')
79 files changed, 2184 insertions, 506 deletions
diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py index a464d6a40..679b4c52b 100644 --- a/src/lib/Bcfg2/Client/Proxy.py +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -12,13 +12,9 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus # M2Crypto instead. try: import ssl - SSL_LIB = 'py26_ssl' SSL_ERROR = ssl.SSLError except ImportError: - from M2Crypto import SSL - import M2Crypto.SSL.Checker - SSL_LIB = 'm2crypto' - SSL_ERROR = SSL.SSLError + raise Exception("No SSL module support") version = sys.version_info[:2] @@ -123,7 +119,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): """ def __init__(self, host, port=None, strict=None, timeout=90, key=None, - cert=None, ca=None, scns=None, protocol='xmlrpc/ssl'): + cert=None, ca=None, scns=None, protocol='xmlrpc/tlsv1'): """Initializes the `httplib.HTTPConnection` object and stores security parameters @@ -148,15 +144,15 @@ class SSLHTTPConnection(httplib.HTTPConnection): specify the same file as `cert` if using a file that contains both. See http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. + for details. Required if using client certificate + authentication. cert : string, optional The file system path to the local endpoint's SSL certificate. May specify the same file as `cert` if using a file that contains both. See http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. + for details. Required if using client certificate + authentication. ca : string, optional The file system path to a set of concatenated certificate authority certs, which are used to validate certificates @@ -187,15 +183,6 @@ class SSLHTTPConnection(httplib.HTTPConnection): self.timeout = timeout def connect(self): - """Initiates a connection using previously set attributes.""" - if SSL_LIB == 'py26_ssl': - self._connect_py26ssl() - elif SSL_LIB == 'm2crypto': - self._connect_m2crypto() - else: - raise Exception("No SSL module support") - - def _connect_py26ssl(self): """Initiates a connection using the ssl module.""" # check for IPv6 hostip = socket.getaddrinfo(self.host, @@ -242,60 +229,11 @@ class SSLHTTPConnection(httplib.HTTPConnection): raise CertificateError(scn) self.sock.closeSocket = True - def _connect_m2crypto(self): - """Initiates a connection using the M2Crypto module.""" - - if self.protocol == 'xmlrpc/ssl': - ctx = SSL.Context('sslv23') - elif self.protocol == 'xmlrpc/tlsv1': - ctx = SSL.Context('tlsv1') - else: - self.logger.error("Unknown protocol %s" % (self.protocol)) - raise Exception("unknown protocol %s" % self.protocol) - - if self.ca: - # Use the certificate authority to validate the cert - # presented by the server - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, - depth=9) - if ctx.load_verify_locations(self.ca) != 1: - raise Exception('No CA certs') - else: - self.logger.warning("No ca is specified. Cannot authenticate the " - "server with SSL.") - - if self.cert and self.key: - # A cert/key is defined, use them to support client - # authentication to the server - ctx.load_cert(self.cert, self.key) - elif self.cert: - self.logger.warning("SSL cert specfied, but no key. Cannot " - "authenticate this client with SSL.") - elif self.key: - self.logger.warning("SSL key specfied, but no cert. Cannot " - "authenticate this client with SSL.") - - self.sock = SSL.Connection(ctx) - if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', self.host): - # host is ip address - try: - hostname = socket.gethostbyaddr(self.host)[0] - except: - # fall back to ip address - hostname = self.host - else: - hostname = self.host - try: - self.sock.connect((hostname, self.port)) - # automatically checks cert matches host - except M2Crypto.SSL.Checker.WrongHost: - wr = sys.exc_info()[1] - raise CertificateError(wr) - class XMLRPCTransport(xmlrpclib.Transport): def __init__(self, key=None, cert=None, ca=None, - scns=None, use_datetime=0, timeout=90): + scns=None, use_datetime=0, timeout=90, + protocol='xmlrpc/tlsv1'): if hasattr(xmlrpclib.Transport, '__init__'): xmlrpclib.Transport.__init__(self, use_datetime) self.key = key @@ -303,6 +241,7 @@ class XMLRPCTransport(xmlrpclib.Transport): self.ca = ca self.scns = scns self.timeout = timeout + self.protocol = protocol def make_connection(self, host): host, self._extra_headers = self.get_host_info(host)[0:2] @@ -311,7 +250,8 @@ class XMLRPCTransport(xmlrpclib.Transport): cert=self.cert, ca=self.ca, scns=self.scns, - timeout=self.timeout) + timeout=self.timeout, + protocol=self.protocol) def request(self, host, handler, request_body, verbose=0): """Send request to server and return response.""" @@ -354,9 +294,15 @@ class ComponentProxy(xmlrpclib.ServerProxy): """Constructs proxies to components. """ options = [ - Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, - Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_ca, Bcfg2.Options.Common.password, Bcfg2.Options.Common.client_timeout, + Bcfg2.Options.Common.protocol, + Bcfg2.Options.PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key'), + Bcfg2.Options.PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate'), Bcfg2.Options.Option( "-u", "--user", default="root", cf=('communication', 'user'), help='The user to provide for authentication'), @@ -386,10 +332,12 @@ class ComponentProxy(xmlrpclib.ServerProxy): path) else: url = Bcfg2.Options.setup.server - ssl_trans = XMLRPCTransport(Bcfg2.Options.setup.key, - Bcfg2.Options.setup.cert, - Bcfg2.Options.setup.ca, - Bcfg2.Options.setup.ssl_cns, - Bcfg2.Options.setup.client_timeout) + ssl_trans = XMLRPCTransport( + key=Bcfg2.Options.setup.key, + cert=Bcfg2.Options.setup.cert, + ca=Bcfg2.Options.setup.ca, + scns=Bcfg2.Options.setup.ssl_cns, + timeout=Bcfg2.Options.setup.client_timeout, + protocol=Bcfg2.Options.setup.protocol) xmlrpclib.ServerProxy.__init__(self, url, allow_none=True, transport=ssl_trans) diff --git a/src/lib/Bcfg2/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index 457197c28..7313f6fcc 100644 --- a/src/lib/Bcfg2/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -25,7 +25,7 @@ class APK(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): """Verify Package status for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False @@ -33,7 +33,7 @@ class APK(Bcfg2.Client.Tools.PkgTool): if entry.attrib['name'] in self.installed: if entry.attrib['version'] in \ ['auto', self.installed[entry.attrib['name']]]: - #FIXME: Does APK have any sort of verification mechanism? + # FIXME: Does APK have any sort of verification mechanism? return True else: self.logger.info(" pkg %s at version %s, not %s" % diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 5549b1717..dedc50d89 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -36,7 +36,7 @@ class Action(Bcfg2.Client.Tools.Tool): shell = True shell_string = '(in shell) ' - if not Bcfg2.Options.setup.dryrun: + if not Bcfg2.Options.setup.dry_run: if Bcfg2.Options.setup.interactive: prompt = ('Run Action %s%s, %s: (y/N): ' % (shell_string, entry.get('name'), diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py index 2ab64f86d..24bc4cf36 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py @@ -1,27 +1,143 @@ """FreeBSD Init Support for Bcfg2.""" -__revision__ = '$Rev$' - -# TODO -# - hardcoded path to ports rc.d -# - doesn't know about /etc/rc.d/ import os +import re +import Bcfg2.Options import Bcfg2.Client.Tools class FreeBSDInit(Bcfg2.Client.Tools.SvcTool): """FreeBSD service support for Bcfg2.""" name = 'FreeBSDInit' + __execs__ = ['/usr/sbin/service', '/usr/sbin/sysrc'] __handles__ = [('Service', 'freebsd')] __req__ = {'Service': ['name', 'status']} + rcvar_re = re.compile(r'^(?P<var>[a-z_]+_enable)="[A-Z]+"$') - def __init__(self, config): - Bcfg2.Client.Tools.SvcTool.__init__(self, config) - if os.uname()[0] != 'FreeBSD': - raise Bcfg2.Client.Tools.ToolInstantiationError + def get_svc_command(self, service, action): + return '/usr/sbin/service %s %s' % (service.get('name'), action) - def VerifyService(self, entry, _): + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" + cmd = self.get_svc_command(entry, 'enabled') + current_bootstatus = bool(self.cmd.run(cmd)) + + if bootstatus == 'off': + if current_bootstatus: + entry.set('current_bootstatus', 'on') + return False + return True + elif not current_bootstatus: + entry.set('current_bootstatus', 'off') + return False return True - def get_svc_command(self, service, action): - return "/usr/local/etc/rc.d/%s %s" % (service.get('name'), action) + def check_service(self, entry): + # use 'onestatus' to enable status reporting for disabled services + cmd = self.get_svc_command(entry, 'onestatus') + return bool(self.cmd.run(cmd)) + + def stop_service(self, service): + # use 'onestop' to enable stopping of disabled services + self.logger.debug('Stopping service %s' % service.get('name')) + return self.cmd.run(self.get_svc_command(service, 'onestop')) + + + def VerifyService(self, entry, _): + """Verify Service status for entry.""" + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: + return True + current_bootstatus = self.verify_bootstatus(entry, bootstatus) + + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: + entry.set('current_status', 'on') + else: + entry.set('current_status', 'off') + + return current_bootstatus and current_svcstatus + + def InstallService(self, entry): + """Install Service entry.""" + self.logger.info("Installing Service %s" % (entry.get('name'))) + bootstatus = self.get_bootstatus(entry) + + # check if service exists + all_services_cmd = '/usr/sbin/service -l' + all_services = self.cmd.run(all_services_cmd).stdout.splitlines() + if entry.get('name') not in all_services: + self.logger.debug("Service %s does not exist" % entry.get('name')) + return False + + # get rcvar for service + vars = set() + rcvar_cmd = self.get_svc_command(entry, 'rcvar') + for line in self.cmd.run(rcvar_cmd).stdout.splitlines(): + match = self.rcvar_re.match(line) + if match: + vars.add(match.group('var')) + + if bootstatus is not None: + bootcmdrv = True + sysrcstatus = None + if bootstatus == 'on': + sysrcstatus = 'YES' + elif bootstatus == 'off': + sysrcstatus = 'NO' + if sysrcstatus is not None: + for var in vars: + if not self.cmd.run('/usr/sbin/sysrc %s="%s"' % (var, sysrcstatus)): + bootcmdrv = False + break + + if Bcfg2.Options.setup.service_mode == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv + buildmode = Bcfg2.Options.setup.service_mode == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv + else: + # when bootstatus is 'None', status == 'ignore' + return True + + def FindExtra(self): + """Find Extra FreeBSD Service entries.""" + specified = [entry.get('name') for entry in self.getSupportedEntries()] + extra = set() + for path in self.cmd.run("/usr/sbin/service -e").stdout.splitlines(): + name = os.path.basename(path) + if name not in specified: + extra.add(name) + return [Bcfg2.Client.XML.Element('Service', name=name, type='freebsd') + for name in list(extra)] + + def Remove(self, _): + """Remove extra service entries.""" + # Extra service removal is nonsensical + # Extra services need to be reflected in the config + return diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py index 31925fa3c..22cf802cf 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py @@ -29,7 +29,7 @@ class FreeBSDPackage(Bcfg2.Client.Tools.PkgTool): self.installed[name] = version def VerifyPackage(self, entry, _): - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False diff --git a/src/lib/Bcfg2/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py index c998ff083..0f82b1bc1 100644 --- a/src/lib/Bcfg2/Client/Tools/IPS.py +++ b/src/lib/Bcfg2/Client/Tools/IPS.py @@ -37,7 +37,7 @@ class IPS(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): """Verify package for entry.""" pname = entry.get('name') - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (pname)) return False if pname not in self.installed: diff --git a/src/lib/Bcfg2/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index 265171a5a..1e9847c42 100644 --- a/src/lib/Bcfg2/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -31,16 +31,16 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): """Verify Package status for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False if entry.attrib['name'] in self.installed: if (self.installed[entry.attrib['name']] == entry.attrib['version'] - or entry.attrib['version'] == 'any'): - #FIXME: We should be able to check this once - # http://trac.macports.org/ticket/15709 is implemented + or entry.attrib['version'] == 'any'): + # FIXME: We should be able to check this once + # http://trac.macports.org/ticket/15709 is implemented return True else: self.logger.info(" %s: Wrong version installed. " diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py new file mode 100644 index 000000000..fc4e16904 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -0,0 +1,296 @@ +""" Augeas driver """ + +import sys +import Bcfg2.Client.XML +from augeas import Augeas +from Bcfg2.Client.Tools.POSIX.base import POSIXTool +from Bcfg2.Client.Tools.POSIX.File import POSIXFile + + +class AugeasCommand(object): + """ Base class for all Augeas command objects """ + + def __init__(self, command, augeas_obj, logger): + self._augeas = augeas_obj + self.command = command + self.entry = self.command.getparent() + self.logger = logger + + def get_path(self, attr="path"): + """ Get a fully qualified path from the name of the parent entry and + the path given in this command tag. + + @param attr: The attribute to get the relative path from + @type attr: string + @returns: string - the fully qualified Augeas path + + """ + return "/files/%s/%s" % (self.entry.get("name").strip("/"), + self.command.get(attr).lstrip("/")) + + def _exists(self, path): + """ Return True if a path exists in Augeas, False otherwise. + + Note that a False return can mean many things: A file that + doesn't exist, a node within the file that doesn't exist, no + lens to parse the file, etc. """ + return len(self._augeas.match(path)) > 1 + + def _verify_exists(self, path=None): + """ Verify that the given path exists, with friendly debug + logging. + + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path exists + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying that '%s' exists" % path) + return self._exists(path) + + def _verify_not_exists(self, path=None): + """ Verify that the given path does not exist, with friendly + debug logging. + + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path does not exist. + (I.e., True if it does not exist, False if it does + exist.) + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying that '%s' does not exist" % path) + return not self._exists(path) + + def _verify_set(self, expected, path=None): + """ Verify that the given path is set to the given value, with + friendly debug logging. + + @param expected: The expected value of the node. + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path matches the expected value. + + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying '%s' == '%s'" % (path, expected)) + actual = self._augeas.get(path) + if actual == expected: + return True + else: + self.logger.debug("Augeas: '%s' failed verification: '%s' != '%s'" + % (path, actual, expected)) + return False + + def __str__(self): + return Bcfg2.Client.XML.tostring(self.command) + + def verify(self): + """ Verify that the command has been applied. """ + raise NotImplementedError + + def install(self): + """ Run the command. """ + raise NotImplementedError + + +class Remove(AugeasCommand): + """ Augeas ``rm`` command """ + def verify(self): + return self._verify_not_exists() + + def install(self): + self.logger.debug("Augeas: Removing %s" % self.get_path()) + return self._augeas.remove(self.get_path()) + + +class Move(AugeasCommand): + """ Augeas ``move`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.source = self.get_path("source") + self.dest = self.get_path("destination") + + def verify(self): + return (self._verify_not_exists(self.source), + self._verify_exists(self.dest)) + + def install(self): + self.logger.debug("Augeas: Moving %s to %s" % (self.source, self.dest)) + return self._augeas.move(self.source, self.dest) + + +class Set(AugeasCommand): + """ Augeas ``set`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.value = self.command.get("value") + + def verify(self): + return self._verify_set(self.value) + + def install(self): + self.logger.debug("Augeas: Setting %s to %s" % (self.get_path(), + self.value)) + return self._augeas.set(self.get_path(), self.value) + + +class Clear(Set): + """ Augeas ``clear`` command """ + def __init__(self, command, augeas_obj, logger): + Set.__init__(self, command, augeas_obj, logger) + self.value = None + + +class SetMulti(AugeasCommand): + """ Augeas ``setm`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.sub = self.command.get("sub") + self.value = self.command.get("value") + self.base = self.get_path("base") + + def verify(self): + return all(self._verify_set(self.value, + path="%s/%s" % (path, self.sub)) + for path in self._augeas.match(self.base)) + + def install(self): + return self._augeas.setm(self.base, self.sub, self.value) + + +class Insert(AugeasCommand): + """ Augeas ``ins`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.label = self.command.get("label") + self.where = self.command.get("where", "before") + self.before = self.where == "before" + + def verify(self): + return self._verify_exists("%s/../%s" % (self.get_path(), self.label)) + + def install(self): + self.logger.debug("Augeas: Inserting new %s %s %s" % + (self.label, self.where, self.get_path())) + return self._augeas.insert(self.get_path(), self.label, self.before) + + +class POSIXAugeas(POSIXTool): + """ Handle <Path type='augeas'...> entries. See + :ref:`client-tools-augeas`. """ + __req__ = ['name', 'mode', 'owner', 'group'] + + def __init__(self, config): + POSIXTool.__init__(self, config) + self._augeas = dict() + # file tool for setting initial values of files that don't + # exist + self.filetool = POSIXFile(config) + + def get_augeas(self, entry): + """ Get an augeas object for the given entry. """ + if entry.get("name") not in self._augeas: + aug = Augeas() + if entry.get("lens"): + self.logger.debug("Augeas: Adding %s to include path for %s" % + (entry.get("name"), entry.get("lens"))) + incl = "/augeas/load/%s/incl" % entry.get("lens") + ilen = len(aug.match(incl)) + if ilen == 0: + self.logger.error("Augeas: Lens %s does not exist" % + entry.get("lens")) + else: + aug.set("%s[%s]" % (incl, ilen + 1), entry.get("name")) + aug.load() + self._augeas[entry.get("name")] = aug + return self._augeas[entry.get("name")] + + def fully_specified(self, entry): + return len(entry.getchildren()) != 0 + + def get_commands(self, entry): + """ Get a list of commands to verify or install. + + @param entry: The entry to get commands from. + @type entry: lxml.etree._Element + @param unverified: Only get commands that failed verification. + @type unverified: bool + @returns: list of + :class:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand` + objects representing the commands. + """ + rv = [] + for cmd in entry.iterchildren(): + if cmd.tag == "Initial": + continue + if cmd.tag in globals(): + rv.append(globals()[cmd.tag](cmd, self.get_augeas(entry), + self.logger)) + else: + err = "Augeas: Unknown command %s in %s" % (cmd.tag, + entry.get("name")) + self.logger.error(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) + return rv + + def verify(self, entry, modlist): + rv = True + for cmd in self.get_commands(entry): + try: + if not cmd.verify(): + err = "Augeas: Command has not been applied to %s: %s" % \ + (entry.get("name"), cmd) + self.logger.debug(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), + err])) + rv = False + cmd.command.set("verified", "false") + else: + cmd.command.set("verified", "true") + except: # pylint: disable=W0702 + err = "Augeas: Unexpected error verifying %s: %s: %s" % \ + (entry.get("name"), cmd, sys.exc_info()[1]) + self.logger.error(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) + rv = False + cmd.command.set("verified", "false") + return POSIXTool.verify(self, entry, modlist) and rv + + def install(self, entry): + rv = True + if entry.get("current_exists", "true") == "false": + initial = entry.find("Initial") + if initial is not None: + self.logger.debug("Augeas: Setting initial data for %s" % + entry.get("name")) + file_entry = Bcfg2.Client.XML.Element("Path", + **dict(entry.attrib)) + file_entry.text = initial.text + self.filetool.install(file_entry) + # re-parse the file + self.get_augeas(entry).load() + for cmd in self.get_commands(entry): + try: + cmd.install() + except: # pylint: disable=W0702 + self.logger.error( + "Failure running Augeas command on %s: %s: %s" % + (entry.get("name"), cmd, sys.exc_info()[1])) + rv = False + try: + self.get_augeas(entry).save() + except: # pylint: disable=W0702 + self.logger.error("Failure saving Augeas changes to %s: %s" % + (entry.get("name"), sys.exc_info()[1])) + rv = False + return POSIXTool.install(self, entry) and rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index d7a70e202..0452ea258 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -3,7 +3,6 @@ import os import sys import stat -import time import difflib import tempfile import Bcfg2.Options @@ -189,12 +188,11 @@ class POSIXFile(POSIXTool): prompt.append('Binary file, no printable diff') attrs['current_bfile'] = b64encode(content) else: + diff = self._diff(content, self._get_data(entry)[0], + filename=entry.get("name")) if interactive: - diff = self._diff(content, self._get_data(entry)[0], - difflib.unified_diff, - filename=entry.get("name")) if diff: - udiff = '\n'.join(l.rstrip('\n') for l in diff) + udiff = '\n'.join(diff) if hasattr(udiff, "decode"): udiff = udiff.decode(Bcfg2.Options.setup.encoding) try: @@ -209,8 +207,6 @@ class POSIXFile(POSIXTool): prompt.append("Diff took too long to compute, no " "printable diff") if not sensitive: - diff = self._diff(content, self._get_data(entry)[0], - difflib.ndiff, filename=entry.get("name")) if diff: attrs["current_bdiff"] = b64encode("\n".join(diff)) else: @@ -221,28 +217,12 @@ class POSIXFile(POSIXTool): for attr, val in attrs.items(): entry.set(attr, val) - def _diff(self, content1, content2, difffunc, filename=None): - """ Return a diff of the two strings, as produced by difffunc. - warns after 5 seconds and times out after 30 seconds. """ - rv = [] - start = time.time() - longtime = False - for diffline in difffunc(content1.split('\n'), - content2.split('\n')): - now = time.time() - rv.append(diffline) - if now - start > 5 and not longtime: - if filename: - self.logger.info("POSIX: Diff of %s taking a long time" % - filename) - else: - self.logger.info("POSIX: Diff taking a long time") - longtime = True - elif now - start > 30: - if filename: - self.logger.error("POSIX: Diff of %s took too long; " - "giving up" % filename) - else: - self.logger.error("POSIX: Diff took too long; giving up") - return False - return rv + def _diff(self, content1, content2, filename=None): + """ Return a unified diff of the two strings """ + + fromfile = "%s (on disk)" % filename if filename else "" + tofile = "%s (from bcfg2)" % filename if filename else "" + return difflib.unified_diff(content1.split('\n'), + content2.split('\n'), + fromfile=fromfile, + tofile=tofile) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 13b45a759..c27c7559d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -58,8 +58,11 @@ class POSIX(Bcfg2.Client.Tools.Tool): mname = submodule[1].rsplit('.', 1)[-1] if mname == 'base': continue - module = getattr(__import__(submodule[1]).Client.Tools.POSIX, - mname) + try: + module = getattr(__import__(submodule[1]).Client.Tools.POSIX, + mname) + except ImportError: + continue hdlr = getattr(module, "POSIX" + mname) if POSIXTool in hdlr.__mro__: # figure out what entry type this handler handles diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 712620206..8895eaae1 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -217,18 +217,13 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): acl.delete_entry(aclentry) if os.path.isdir(path): defacl = posix1e.ACL(filedef=path) - if not defacl.valid(): - # when a default ACL is queried on a directory that - # has no default ACL entries at all, you get an empty - # ACL, which is not valid. in this circumstance, we - # just copy the access ACL to get a base valid ACL - # that we can add things to. - defacl = posix1e.ACL(acl=acl) - else: - for aclentry in defacl: - if aclentry.tag_type in [posix1e.ACL_USER, - posix1e.ACL_GROUP]: - defacl.delete_entry(aclentry) + for aclentry in defacl: + if aclentry.tag_type in [posix1e.ACL_USER, + posix1e.ACL_USER_OBJ, + posix1e.ACL_GROUP, + posix1e.ACL_GROUP_OBJ, + posix1e.ACL_OTHER]: + defacl.delete_entry(aclentry) else: defacl = None @@ -254,10 +249,16 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: if scope == posix1e.ACL_USER: scopename = "user" - aclentry.qualifier = self._norm_uid(qualifier) + if qualifier: + aclentry.qualifier = self._norm_uid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_USER_OBJ elif scope == posix1e.ACL_GROUP: scopename = "group" - aclentry.qualifier = self._norm_gid(qualifier) + if qualifier: + aclentry.qualifier = self._norm_gid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_GROUP_OBJ except (OSError, KeyError): err = sys.exc_info()[1] self.logger.error("POSIX: Could not resolve %s %s: %s" % @@ -358,7 +359,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: # single octal digit rv = int(perms) - if rv > 0 and rv < 8: + if rv >= 0 and rv < 8: return rv else: self.logger.error("POSIX: Permissions digit out of range in " @@ -388,13 +389,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get a string representation of the given ACL. aclkey must be a tuple of (<acl type>, <acl scope>, <qualifier>) """ atype, scope, qualifier = aclkey + if not qualifier: + qualifier = '' acl_str = [] if atype == 'default': acl_str.append(atype) - if scope == posix1e.ACL_USER: + if scope == posix1e.ACL_USER or scope == posix1e.ACL_USER_OBJ: acl_str.append("user") - elif scope == posix1e.ACL_GROUP: + elif scope == posix1e.ACL_GROUP or scope == posix1e.ACL_GROUP_OBJ: acl_str.append("group") + elif scope == posix1e.ACL_OTHER: + acl_str.append("other") acl_str.append(qualifier) acl_str.append(self._acl_perm2string(perms)) return ":".join(acl_str) @@ -414,7 +419,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get data on the existing state of <path> -- e.g., whether or not it exists, owner, group, permissions, etc. """ try: - ondisk = os.stat(path) + ondisk = os.lstat(path) except OSError: self.logger.debug("POSIX: %s does not exist" % path) return (False, None, None, None, None, None) @@ -451,7 +456,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if HAS_SELINUX: try: - secontext = selinux.getfilecon(path)[1].split(":")[2] + secontext = selinux.lgetfilecon(path)[1].split(":")[2] except (OSError, KeyError): err = sys.exc_info()[1] self.logger.debug("POSIX: Could not get current SELinux " @@ -460,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if HAS_ACLS: + if HAS_ACLS and not stat.S_ISLNK(ondisk[stat.ST_MODE]): acls = self._list_file_acls(path) else: acls = None @@ -562,9 +567,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): wanted = dict() for acl in entry.findall("ACL"): if acl.get("scope") == "user": - scope = posix1e.ACL_USER + if acl.get("user"): + scope = posix1e.ACL_USER + else: + scope = posix1e.ACL_USER_OBJ elif acl.get("scope") == "group": - scope = posix1e.ACL_GROUP + if acl.get("group"): + scope = posix1e.ACL_GROUP + else: + scope = posix1e.ACL_GROUP_OBJ + elif acl.get("scope") == "other": + scope = posix1e.ACL_OTHER else: self.logger.error("POSIX: Unknown ACL scope %s" % acl.get("scope")) @@ -573,7 +586,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): self.logger.error("POSIX: No permissions set for ACL: %s" % Bcfg2.Client.XML.tostring(acl)) continue - wanted[(acl.get("type"), scope, acl.get(acl.get("scope")))] = \ + qual = acl.get(acl.get("scope")) + if not qual: + qual = '' + wanted[(acl.get("type"), scope, qual)] = \ self._norm_acl_perms(acl.get('perms')) return wanted @@ -587,11 +603,12 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Given an ACL object, process it appropriately and add it to the return value """ try: + qual = '' if acl.tag_type == posix1e.ACL_USER: qual = pwd.getpwuid(acl.qualifier)[0] elif acl.tag_type == posix1e.ACL_GROUP: qual = grp.getgrgid(acl.qualifier)[0] - else: + elif atype == "access" or acl.tag_type == posix1e.ACL_MASK: return except (OSError, KeyError): err = sys.exc_info()[1] @@ -621,9 +638,38 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): _process_acl(acl, "default") return existing - def _verify_acls(self, entry, path=None): + def _verify_acls(self, entry, path=None): # pylint: disable=R0912 """ verify POSIX ACLs on the given entry. return True if all ACLS are correct, false otherwise """ + def _verify_acl(aclkey, perms): + """ Given ACL data, process it appropriately and add it to + missing or wrong lists if appropriate """ + if aclkey not in existing: + missing.append(self._acl2string(aclkey, perms)) + elif existing[aclkey] != perms: + wrong.append((self._acl2string(aclkey, perms), + self._acl2string(aclkey, existing[aclkey]))) + if path == entry.get("name"): + atype, scope, qual = aclkey + aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, + perms=str(perms)) + if (scope == posix1e.ACL_USER or + scope == posix1e.ACL_USER_OBJ): + aclentry.set("scope", "user") + elif (scope == posix1e.ACL_GROUP or + scope == posix1e.ACL_GROUP_OBJ): + aclentry.set("scope", "group") + elif scope == posix1e.ACL_OTHER: + aclentry.set("scope", "other") + else: + self.logger.debug("POSIX: Unknown ACL scope %s on %s" % + (scope, path)) + return + + if scope != posix1e.ACL_OTHER: + aclentry.set(aclentry.get("scope"), qual) + entry.append(aclentry) + if not HAS_ACLS: if entry.findall("ACL"): self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " @@ -644,25 +690,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): extra = [] wrong = [] for aclkey, perms in wanted.items(): - if aclkey not in existing: - missing.append(self._acl2string(aclkey, perms)) - elif existing[aclkey] != perms: - wrong.append((self._acl2string(aclkey, perms), - self._acl2string(aclkey, existing[aclkey]))) - if path == entry.get("name"): - atype, scope, qual = aclkey - aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, - perms=str(perms)) - if scope == posix1e.ACL_USER: - aclentry.set("scope", "user") - elif scope == posix1e.ACL_GROUP: - aclentry.set("scope", "group") - else: - self.logger.debug("POSIX: Unknown ACL scope %s on %s" % - (scope, path)) - continue - aclentry.set(aclentry.get("scope"), qual) - entry.append(aclentry) + _verify_acl(aclkey, perms) for aclkey, perms in existing.items(): if aclkey not in wanted: diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 58a3bbdfc..a7fcb6709 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -79,7 +79,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): defined, and the uid/gid is in that whitelist; or b) no whitelist is defined, and the uid/gid is not in the blacklist. """ - if self._whitelist[tag] is None: + if not self._whitelist[tag]: return eid not in self._blacklist[tag] else: return eid in self._whitelist[tag] @@ -160,7 +160,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group")] + if entry.get("name") in g[3] and g[0] != entry.get("group") + and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index 2ab9b7403..b82b905e7 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -19,7 +19,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): for pkg in self.cmd.run("/usr/bin/pacman -Q").stdout.splitlines(): pkgname = pkg.split(' ')[0].strip() version = pkg.split(' ')[1].strip() - #self.logger.info(" pkgname: %s, version: %s" % (pkgname, version)) self.installed[pkgname] = version def VerifyPackage(self, entry, _): @@ -28,7 +27,7 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'), entry.get('version'))) - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % entry.attrib['name']) return False @@ -38,8 +37,8 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): return True elif self.installed[entry.attrib['name']] == \ entry.attrib['version']: - #FIXME: need to figure out if pacman - # allows you to verify packages + # FIXME: need to figure out if pacman + # allows you to verify packages return True else: entry.set('current_version', self.installed[entry.get('name')]) diff --git a/src/lib/Bcfg2/Client/Tools/Pkgng.py b/src/lib/Bcfg2/Client/Tools/Pkgng.py new file mode 100644 index 000000000..cd70d662d --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/Pkgng.py @@ -0,0 +1,226 @@ +"""This is the Bcfg2 support for pkg.""" + +import os +import Bcfg2.Options +import Bcfg2.Client.Tools + + +class Pkgng(Bcfg2.Client.Tools.Tool): + """Support for pkgng packages on FreeBSD.""" + + options = Bcfg2.Client.Tools.Tool.options + [ + Bcfg2.Options.PathOption( + cf=('Pkgng', 'path'), + default='/usr/sbin/pkg', dest='pkg_path', + help='Pkgng tool path')] + + name = 'Pkgng' + __execs__ = [] + __handles__ = [('Package', 'pkgng'), ('Path', 'ignore')] + __req__ = {'Package': ['name', 'version'], 'Path': ['type']} + + def __init__(self, config): + Bcfg2.Client.Tools.Tool.__init__(self, config) + + self.pkg = Bcfg2.Options.setup.pkg_path + self.__execs__ = [self.pkg] + + self.pkgcmd = self.pkg + ' install -fy' + if not Bcfg2.Options.setup.debug: + self.pkgcmd += ' -q' + self.pkgcmd += ' %s' + + self.ignores = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and + entry.get('type') == 'ignore'] + + self.__important__ = self.__important__ + \ + [entry.get('name') for struct in config + for entry in struct + if (entry.tag == 'Path' and + entry.get('name').startswith('/etc/pkg/'))] + self.nonexistent = [entry.get('name') for struct in config + for entry in struct if entry.tag == 'Path' + and entry.get('type') == 'nonexistent'] + self.actions = {} + self.pkg_cache = {} + + try: + self._load_pkg_cache() + except OSError: + raise Bcfg2.Client.Tools.ToolInstantiationError + + def _load_pkg_cache(self): + """Cache the version of all currently installed packages.""" + self.pkg_cache = {} + output = self.cmd.run([self.pkg, 'query', '-a', '%n %v']).stdout + for line in output.splitlines(): + parts = line.split(' ') + name = ' '.join(parts[:-1]) + self.pkg_cache[name] = parts[-1] + + def FindExtra(self): + """Find extra packages.""" + packages = [entry.get('name') for entry in self.getSupportedEntries()] + extras = [(name, value) for (name, value) in self.pkg_cache.items() + if name not in packages] + return [Bcfg2.Client.XML.Element('Package', name=name, + type='pkgng', version=version) + for (name, version) in extras] + + def VerifyChecksums(self, entry, modlist): + """Verify the checksum of the files, owned by a package.""" + output = self.cmd.run([self.pkg, 'check', '-s', + entry.get('name')]).stdout.splitlines() + files = [] + for item in output: + if "checksum mismatch" in item: + files.append(item.split()[-1]) + elif "No such file or directory" in item: + continue + else: + self.logger.error("Got Unsupported pattern %s " + "from pkg check" % item) + + files = list(set(files) - set(self.ignores)) + # We check if there is file in the checksum to do + if files: + # if files are found there we try to be sure our modlist is sane + # with erroneous symlinks + modlist = [os.path.realpath(filename) for filename in modlist] + bad = [filename for filename in files if filename not in modlist] + if bad: + self.logger.debug("It is suggested that you either manage " + "these files, revert the changes, or ignore " + "false failures:") + self.logger.info("Package %s failed validation. Bad files " + "are:" % entry.get('name')) + self.logger.info(bad) + entry.set('qtext', + "Reinstall Package %s-%s to fix failing files? " + "(y/N) " % (entry.get('name'), entry.get('version'))) + return False + return True + + def _get_candidate_versions(self, name): + """ + Get versions of the specified package name available for + installation from the configured remote repositories. + """ + output = self.cmd.run([self.pkg, 'search', '-Qversion', '-q', + '-Sname', '-e', name]).stdout.splitlines() + versions = [] + for line in output: + versions.append(line) + + if len(versions) == 0: + return None + + return sorted(versions) + + def VerifyPackage(self, entry, modlist, checksums=True): + """Verify package for entry.""" + if 'version' not in entry.attrib: + self.logger.info("Cannot verify unversioned package %s" % + (entry.attrib['name'])) + return False + + pkgname = entry.get('name') + if pkgname not in self.pkg_cache: + self.logger.info("Package %s not installed" % (entry.get('name'))) + entry.set('current_exists', 'false') + return False + + installed_version = self.pkg_cache[pkgname] + candidate_versions = self._get_candidate_versions(pkgname) + if candidate_versions is not None: + candidate_version = candidate_versions[0] + else: + self.logger.error("Package %s is installed but no candidate" + "version was found." % (entry.get('name'))) + return False + + if entry.get('version').startswith('auto'): + desired_version = candidate_version + entry.set('version', "auto: %s" % desired_version) + elif entry.get('version').startswith('any'): + desired_version = installed_version + entry.set('version', "any: %s" % desired_version) + else: + desired_version = entry.get('version') + + if desired_version != installed_version: + entry.set('current_version', installed_version) + entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % + (entry.get('name'), entry.get('current_version'), + desired_version)) + return False + else: + # version matches + if (not Bcfg2.Options.setup.quick and + entry.get('verify', 'true') == 'true' + and checksums): + pkgsums = self.VerifyChecksums(entry, modlist) + return pkgsums + return True + + def Remove(self, packages): + """Deal with extra configuration detected.""" + pkgnames = " ".join([pkg.get('name') for pkg in packages]) + if len(packages) > 0: + self.logger.info('Removing packages:') + self.logger.info(pkgnames) + self.cmd.run([self.pkg, 'delete', '-y', pkgnames]) + self._load_pkg_cache() + self.modified += packages + self.extra = self.FindExtra() + + def Install(self, packages): + ipkgs = [] + bad_pkgs = [] + for pkg in packages: + versions = self._get_candidate_versions(pkg.get('name')) + if versions is None: + self.logger.error("pkg has no information about package %s" % + (pkg.get('name'))) + continue + + if pkg.get('version').startswith('auto') or \ + pkg.get('version').startswith('any'): + ipkgs.append("%s-%s" % (pkg.get('name'), versions[0])) + continue + + if pkg.get('version') in versions: + ipkgs.append("%s-%s" % (pkg.get('name'), pkg.get('version'))) + continue + else: + self.logger.error("Package %s: desired version %s not in %s" % + (pkg.get('name'), pkg.get('version'), + versions)) + bad_pkgs.append(pkg.get('name')) + + if bad_pkgs: + self.logger.error("Cannot find correct versions of packages:") + self.logger.error(bad_pkgs) + if not ipkgs: + return + if not self.cmd.run(self.pkgcmd % (" ".join(ipkgs))): + self.logger.error("pkg command failed") + self._load_pkg_cache() + self.extra = self.FindExtra() + mark = [] + states = dict() + for package in packages: + states[package] = self.VerifyPackage(package, [], checksums=False) + if states[package]: + self.modified.append(package) + if package.get('origin') == 'Packages': + mark.append(package.get('name')) + if mark: + self.cmd.run([self.pkg, 'set', '-A1', '-y'] + mark) + return states + + def VerifyPath(self, _entry, _): + """Do nothing here since we only verify Path type=ignore.""" + return True diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index a61ede820..5c092f46b 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -50,7 +50,7 @@ class Portage(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, modlist): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.get('name'))) return False @@ -68,11 +68,11 @@ class Portage(Bcfg2.Client.Tools.PkgTool): if ('verify' not in entry.attrib or entry.get('verify').lower() == 'true'): - # Check the package if: - # - Not running in quick mode - # - No verify option is specified in the literal configuration - # OR - # - Verify option is specified and is true + # Check the package if: + # - Not running in quick mode + # - No verify option is specified in the literal configuration + # OR + # - Verify option is specified and is true self.logger.debug('Running equery check on %s' % entry.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/SMF.py b/src/lib/Bcfg2/Client/Tools/SMF.py index 8b23a4a37..1a580d8a5 100644 --- a/src/lib/Bcfg2/Client/Tools/SMF.py +++ b/src/lib/Bcfg2/Client/Tools/SMF.py @@ -25,7 +25,7 @@ class SMF(Bcfg2.Client.Tools.SvcTool): def GetFMRI(self, entry): """Perform FMRI resolution for service.""" - if not 'FMRI' in entry.attrib: + if 'FMRI' not in entry.attrib: rv = self.cmd.run(["/usr/bin/svcs", "-H", "-o", "FMRI", entry.get('name')]) if rv.success: diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 20a172d3d..027d91c71 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,8 +13,6 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} - conflicts = ['Chkconfig'] - def get_svc_command(self, service, action): return "/bin/systemctl %s %s.service" % (action, service.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 4e8ac76a4..449503b55 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -165,12 +165,13 @@ class VCS(Bcfg2.Client.Tools.Tool): def Verifysvn(self, entry, _): """Verify svn repositories""" + # pylint: disable=E1101 headrev = pysvn.Revision(pysvn.opt_revision_kind.head) + # pylint: enable=E1101 client = pysvn.Client() try: cur_rev = str(client.info(entry.get('name')).revision.number) - server = client.info2(entry.get('sourceurl'), - headrev, + server = client.info2(entry.get('sourceurl'), headrev, recurse=False) if server: server_rev = str(server[0][1].rev.number) diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 0b38044d4..86048cb0b 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -632,34 +632,38 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = True stat['version_fail'] = True # Just chose the first pkg for the error message + current_pkg = all_pkg_objs[0] if virt_pkg: provides = \ - [p for p in all_pkg_objs[0].provides + [p for p in current_pkg.provides if p[0] == entry.get("name")][0] - entry.set('current_version', "%s:%s-%s" % provides[2]) + current_evr = provides[2] self.logger.info( " %s: Wrong version installed. " "Want %s, but %s provides %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]), + nevra2string(current_pkg), yum.misc.prco_tuple_to_string(provides))) else: - entry.set('current_version', "%s:%s-%s.%s" % - (all_pkg_objs[0].epoch, - all_pkg_objs[0].version, - all_pkg_objs[0].release, - all_pkg_objs[0].arch)) + current_evr = (current_pkg.epoch, + current_pkg.version, + current_pkg.release) self.logger.info(" %s: Wrong version installed. " "Want %s, but have %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]))) - entry.set('version', "%s:%s-%s.%s" % - (nevra.get('epoch', 'any'), - nevra.get('version', 'any'), - nevra.get('release', 'any'), - nevra.get('arch', 'any'))) + nevra2string(current_pkg))) + wanted_evr = (nevra.get('epoch', 'any'), + nevra.get('version', 'any'), + nevra.get('release', 'any')) + entry.set('current_version', "%s:%s-%s" % current_evr) + entry.set('version', "%s:%s-%s" % wanted_evr) + if yum.compareEVR(current_evr, wanted_evr) == 1: + entry.set("package_fail_action", "downgrade") + else: + entry.set("package_fail_action", "update") + qtext_versions.append("U(%s)" % str(all_pkg_objs[0])) continue @@ -910,7 +914,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): cleanup() - def Install(self, packages): # pylint: disable=R0912,R0914 + def Install(self, packages): # pylint: disable=R0912,R0914,R0915 """ Try and fix everything that Yum.VerifyPackages() found wrong for each Package Entry. This can result in individual RPMs being installed (for the first time), deleted, downgraded @@ -932,6 +936,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): install_pkgs = [] gpg_keys = [] upgrade_pkgs = [] + downgrade_pkgs = [] reinstall_pkgs = [] def queue_pkg(pkg, inst, queue): @@ -971,11 +976,14 @@ class YUM(Bcfg2.Client.Tools.PkgTool): continue status = self.instance_status[inst] if (not status.get('installed', False) and - Bcfg2.Options.setup.yum_install_missing): + Bcfg2.Options.setup.yum_install_missing): queue_pkg(pkg, inst, install_pkgs) elif (status.get('version_fail', False) and Bcfg2.Options.setup.yum_fix_version): - queue_pkg(pkg, inst, upgrade_pkgs) + if pkg.get("package_fail_action") == "downgrade": + queue_pkg(pkg, inst, downgrade_pkgs) + else: + queue_pkg(pkg, inst, upgrade_pkgs) elif (status.get('verify_fail', False) and Bcfg2.Options.setup.yum_reinstall_broken): queue_pkg(pkg, inst, reinstall_pkgs) @@ -1039,6 +1047,19 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.error("Error upgrading package %s: %s" % (pkg_arg, yume)) + if len(downgrade_pkgs) > 0: + self.logger.info("Attempting to downgrade packages") + + for inst in downgrade_pkgs: + pkg_arg = self.instance_status[inst].get('pkg').get('name') + self.logger.debug("Downgrading %s" % pkg_arg) + try: + self.yumbase.downgrade(**build_yname(pkg_arg, inst)) + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] + self.logger.error("Error downgrading package %s: %s" % + (pkg_arg, yume)) + if len(reinstall_pkgs) > 0: self.logger.info("Attempting to reinstall packages") for inst in reinstall_pkgs: diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 91d4ac5c6..4ba06abae 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -5,9 +5,29 @@ # pylint: disable=E0611,W0611,W0613,C0103 try: - from lxml.etree import Element, SubElement, XML, tostring + from lxml.etree import Element, SubElement, tostring, XMLParser from lxml.etree import XMLSyntaxError as ParseError + from lxml.etree import XML as _XML + from Bcfg2.Compat import wraps driver = 'lxml' + + # libxml2 2.9.0+ doesn't parse 10M+ documents by default: + # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html + try: + _parser = XMLParser(huge_tree=True) + except TypeError: + _parser = XMLParser() + + @wraps(_XML) + def XML(val, **kwargs): + """ unicode strings w/encoding declaration are not supported in + recent lxml.etree, so we try to read XML, and if it fails we try + encoding the string. """ + kwargs.setdefault('parser', _parser) + try: + return _XML(val, **kwargs) + except ValueError: + return _XML(val.encode(), **kwargs) except ImportError: # lxml not available from xml.parsers.expat import ExpatError as ParseError diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 2461c1316..073aa7694 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -69,8 +69,8 @@ def prompt(msg): except UnicodeEncodeError: ans = input(msg.encode('utf-8')) return ans in ['y', 'Y'] - except EOFError: - # handle ^C on rhel-based platforms + except (EOFError, KeyboardInterrupt): + # handle ^C raise SystemExit(1) except: print("Error while reading input: %s" % sys.exc_info()[1]) @@ -113,10 +113,10 @@ class Client(object): help='Force removal of additional configuration items')), Bcfg2.Options.ExclusiveOptionGroup( Bcfg2.Options.PathOption( - '-f', '--file', type=argparse.FileType('r'), + '-f', '--file', type=argparse.FileType('rb'), help='Configure from a file rather than querying the server'), Bcfg2.Options.PathOption( - '-c', '--cache', type=argparse.FileType('w'), + '-c', '--cache', type=argparse.FileType('wb'), help='Store the configuration in a file')), Bcfg2.Options.BooleanOption( '--exit-on-probe-failure', default=True, @@ -144,7 +144,10 @@ class Client(object): Bcfg2.Options.BooleanOption( "-e", "--show-extra", help='Enable extra entry output'), Bcfg2.Options.BooleanOption( - "-k", "--kevlar", help='Run in bulletproof mode')] + "-k", "--kevlar", help='Run in bulletproof mode'), + Bcfg2.Options.BooleanOption( + "-i", "--only-important", + help='Only configure the important entries')] def __init__(self): self.config = None @@ -403,7 +406,7 @@ class Client(object): self.config = newconfig if not Bcfg2.Options.setup.no_lock: - #check lock here + # check lock here try: lockfile = open(Bcfg2.Options.setup.lockfile, 'w') if locked(lockfile.fileno()): @@ -559,11 +562,13 @@ class Client(object): if x not in b_to_rem] # take care of important entries first - if not Bcfg2.Options.setup.dry_run: + if (not Bcfg2.Options.setup.dry_run or + Bcfg2.Options.setup.only_important): + important_installs = set() for parent in self.config.findall(".//Path/.."): name = parent.get("name") - if (name and (name in Bcfg2.Options.setup.only_bundles or - name not in Bcfg2.Options.setup.except_bundles)): + if not name or (name in Bcfg2.Options.setup.except_bundles and + name not in Bcfg2.Options.setup.only_bundles): continue for cfile in parent.findall("./Path"): if (cfile.get('name') not in self.__important__ or @@ -574,6 +579,9 @@ class Client(object): if t.handlesEntry(cfile) and t.canVerify(cfile)] if not tools: continue + if Bcfg2.Options.setup.dry_run: + important_installs.add(cfile) + continue if (Bcfg2.Options.setup.interactive and not self.promptFilter("Install %s: %s? (y/N):", [cfile])): @@ -589,6 +597,11 @@ class Client(object): cfile.set('qtext', '') if tools[0].VerifyPath(cfile, []): self.whitelist.remove(cfile) + if Bcfg2.Options.setup.dry_run and len(important_installs) > 0: + self.logger.info("In dryrun mode: " + "suppressing entry installation for:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in important_installs]) def Inventory(self): """ @@ -845,11 +858,13 @@ class Client(object): self.times['inventory'] = time.time() self.CondDisplayState('initial') self.InstallImportant() - self.Decide() - self.Install() - self.times['install'] = time.time() - self.Remove() - self.times['remove'] = time.time() + if not Bcfg2.Options.setup.only_important: + self.Decide() + self.Install() + self.times['install'] = time.time() + self.Remove() + self.times['remove'] = time.time() + if self.modified: self.ReInventory() self.times['reinventory'] = time.time() diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 24835a3e8..12dba7fba 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -26,8 +26,8 @@ settings = dict( # pylint: disable=C0103 DEBUG=False, ALLOWED_HOSTS=['*'], MEDIA_URL='/site_media/', - MANAGERS=(('Root', 'root')), - ADMINS=(('Root', 'root')), + MANAGERS=(('Root', 'root'),), + ADMINS=(('Root', 'root'),), # Language code for this installation. All choices can be found # here: # http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes @@ -63,7 +63,8 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.debug', 'django.core.context_processors.i18n', 'django.core.context_processors.media', - 'django.core.context_processors.request')) + 'django.core.context_processors.request'), + DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter']) if HAS_SOUTH: settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') @@ -95,6 +96,18 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) + if hasattr(opts, "reporting_db_engine") and \ + opts.reporting_db_engine is not None: + settings['DATABASES']['Reporting'] = dict( + ENGINE="django.db.backends.%s" % opts.reporting_db_engine, + NAME=opts.reporting_db_name, + USER=opts.reporting_db_user, + PASSWORD=opts.reporting_db_password, + HOST=opts.reporting_db_host, + PORT=opts.reporting_db_port, + OPTIONS=opts.reporting_db_opts, + SCHEMA=opts.reporting_db_schema) + settings['TIME_ZONE'] = opts.timezone settings['TEMPLATE_DEBUG'] = settings['DEBUG'] = \ @@ -112,6 +125,9 @@ def finalize_django_config(opts=None, silent=False): logger = logging.getLogger() logger.debug("Finalizing Django settings: %s" % settings) + module = sys.modules[__name__] + for name, value in settings.items(): + setattr(module, name, value) try: django.conf.settings.configure(**settings) except RuntimeError: @@ -120,6 +136,66 @@ def finalize_django_config(opts=None, silent=False): sys.exc_info()[1]) +def sync_databases(**kwargs): + """ Synchronize all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Syncing database %s" % (database)) + django.core.management.call_command("syncdb", database=database, + **kwargs) + + +def migrate_databases(**kwargs): + """ Do South migrations on all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Migrating database %s" % (database)) + django.core.management.call_command("migrate", database=database, + **kwargs) + + +def get_db_label(application): + """ Get the name of the database for a given Django "application". The + rule is that if a database with the same name as the application exists, + use it. Otherwise use the default. Returns a string suitible for use as a + key in the Django database settings dict """ + if application in settings['DATABASES']: + return application + + return 'default' + + +class PerApplicationRouter(object): + """ Django database router for redirecting different applications to their + own database """ + + def _db_per_app(self, model, **_): + """ If a database with the same name as the application exists, use it. + Otherwise use the default """ + return get_db_label(model._meta.app_label) # pylint: disable=W0212 + + def db_for_read(self, model, **hints): + """ Called when Django wants to find out what database to read from """ + return self._db_per_app(model, **hints) + + def db_for_write(self, model, **hints): + """ Called when Django wants to find out what database to write to """ + return self._db_per_app(model, **hints) + + def allow_relation(self, obj1, obj2, **_): + """ Called when Django wants to determine what relations to allow. Only + allow relations within an app """ + # pylint: disable=W0212 + return obj1._meta.app_label == obj2._meta.app_label + # pylint: enable=W0212 + + def allow_syncdb(self, *_): + """ Called when Django wants to determine which models to sync to a + given database. Take the cowards way out and sync all models to all + databases to allow for easy migrations. """ + return True + + class _OptionContainer(object): """ Container for options loaded at import-time to configure databases """ @@ -131,6 +207,7 @@ class _OptionContainer(object): default="/etc/bcfg2-web.conf", action=Bcfg2.Options.ConfigFileAction, help='Web interface configuration file'), + # default database options Bcfg2.Options.Option( cf=('database', 'engine'), default='sqlite3', help='Database engine', dest='db_engine'), @@ -148,11 +225,40 @@ class _OptionContainer(object): cf=('database', 'port'), help='Database port', dest='db_port'), Bcfg2.Options.Option( cf=('database', 'schema'), help='Database schema', - dest='db_schema'), + dest='db_schema', default='public'), Bcfg2.Options.Option( cf=('database', 'options'), help='Database options', dest='db_opts', type=Bcfg2.Options.Types.comma_dict, default=dict()), + # reporting database options + Bcfg2.Options.Option( + cf=('database', 'reporting_engine'), + help='Reporting database engine', dest='reporting_db_engine'), + Bcfg2.Options.Option( + cf=('database', 'reporting_name'), + default='<repository>/etc/reporting.sqlite', + help="Reporting database name", dest="reporting_db_name"), + Bcfg2.Options.Option( + cf=('database', 'reporting_user'), + help='Reporting database username', dest='reporting_db_user'), + Bcfg2.Options.Option( + cf=('database', 'reporting_password'), + help='Reporting database password', dest='reporting_db_password'), + Bcfg2.Options.Option( + cf=('database', 'reporting_host'), + help='Reporting database host', dest='reporting_db_host'), + Bcfg2.Options.Option( + cf=('database', 'reporting_port'), + help='Reporting database port', dest='reporting_db_port'), + Bcfg2.Options.Option( + cf=('database', 'reporting_schema'), + help='Reporting database schema', dest='reporting_db_schema', + default='public'), + Bcfg2.Options.Option( + cf=('database', 'reporting_options'), + help='Reporting database options', dest='reporting_db_opts', + type=Bcfg2.Options.Types.comma_dict, default=dict()), + # Django options Bcfg2.Options.Option( cf=('reporting', 'timezone'), help='Django timezone'), Bcfg2.Options.BooleanOption( diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index 0f7995e0f..11eaeebd1 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -21,7 +21,7 @@ class TermiosFormatter(logging.Formatter): def __init__(self, fmt=None, datefmt=None): logging.Formatter.__init__(self, fmt, datefmt) - if sys.stdout.isatty(): + if hasattr(sys.stdout, 'isatty') and sys.stdout.isatty(): # now get termios info try: self.width = struct.unpack('hhhh', diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py index 8b97f1da8..8b941f2bb 100644 --- a/src/lib/Bcfg2/Options/Actions.py +++ b/src/lib/Bcfg2/Options/Actions.py @@ -7,7 +7,27 @@ from Bcfg2.Options.Parser import get_parser __all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] -class ComponentAction(argparse.Action): +class FinalizableAction(argparse.Action): + """ A FinalizableAction requires some additional action to be taken + when storing the value, and as a result must be finalized if the + default value is used.""" + + def __init__(self, *args, **kwargs): + argparse.Action.__init__(self, *args, **kwargs) + self._final = False + + def finalize(self, parser, namespace): + """ Finalize a default value by calling the action callable. """ + if not self._final: + self.__call__(parser, namespace, getattr(namespace, self.dest, + self.default)) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + self._final = True + + +class ComponentAction(FinalizableAction): """ ComponentAction automatically imports classes and modules based on the value of the option, and automatically collects options from the loaded classes and modules. It cannot be used by @@ -84,8 +104,7 @@ class ComponentAction(argparse.Action): if self.mapping: if 'choices' not in kwargs: kwargs['choices'] = self.mapping.keys() - self._final = False - argparse.Action.__init__(self, *args, **kwargs) + FinalizableAction.__init__(self, *args, **kwargs) def _import(self, module, name): """ Import the given name from the given module, handling @@ -123,17 +142,10 @@ class ComponentAction(argparse.Action): break if cls: get_parser().add_component(cls) - else: + elif not self.fail_silently: print("Could not load component %s" % name) return cls - def finalize(self, parser, namespace): - """ Finalize a default value by loading the components given - in it. This lets a default be specified with a list of - strings instead of a list of classes. """ - if not self._final: - self.__call__(parser, namespace, self.default) - def __call__(self, parser, namespace, values, option_string=None): if values is None: result = None @@ -146,18 +158,19 @@ class ComponentAction(argparse.Action): result.append(cls) else: result = self._load_component(values) - self._final = True - setattr(namespace, self.dest, result) + FinalizableAction.__call__(self, parser, namespace, result, + option_string=option_string) -class ConfigFileAction(argparse.Action): +class ConfigFileAction(FinalizableAction): """ ConfigFileAction automatically loads and parses a supplementary config file (e.g., ``bcfg2-web.conf`` or ``bcfg2-lint.conf``). """ def __call__(self, parser, namespace, values, option_string=None): - get_parser().add_config_file(self.dest, values) - setattr(namespace, self.dest, values) + parser.add_config_file(self.dest, values, reparse=False) + FinalizableAction.__call__(self, parser, namespace, values, + option_string=option_string) class PluginsAction(ComponentAction): diff --git a/src/lib/Bcfg2/Options/Common.py b/src/lib/Bcfg2/Options/Common.py index 9ba08eb87..620a7604c 100644 --- a/src/lib/Bcfg2/Options/Common.py +++ b/src/lib/Bcfg2/Options/Common.py @@ -94,7 +94,7 @@ class Common(object): #: Log to syslog syslog = BooleanOption( - cf=('logging', 'syslog'), help="Log to syslog") + cf=('logging', 'syslog'), help="Log to syslog", default=True) #: Server location location = Option( @@ -107,20 +107,16 @@ class Common(object): '-x', '--password', cf=('communication', 'password'), metavar='<password>', help="Communication Password") - #: Path to SSL key - ssl_key = PathOption( - '--ssl-key', cf=('communication', 'key'), dest="key", - help='Path to SSL key', default="/etc/pki/tls/private/bcfg2.key") - - #: Path to SSL certificate - ssl_cert = PathOption( - cf=('communication', 'certificate'), dest="cert", - help='Path to SSL certificate', default="/etc/pki/tls/certs/bcfg2.crt") - #: Path to SSL CA certificate ssl_ca = PathOption( cf=('communication', 'ca'), help='Path to SSL CA Cert') + #: Communication protocol + protocol = Option( + cf=('communication', 'protocol'), default='xmlrpc/tlsv1', + choices=['xmlrpc/ssl', 'xmlrpc/tlsv1'], + help='Communication protocol to use.') + #: Default Path paranoid setting default_paranoid = Option( cf=('mdata', 'paranoid'), dest="default_paranoid", default='true', diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py index be7e7c646..3874f810d 100644 --- a/src/lib/Bcfg2/Options/Options.py +++ b/src/lib/Bcfg2/Options/Options.py @@ -10,7 +10,18 @@ from Bcfg2.Options import Types from Bcfg2.Compat import ConfigParser -__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument"] +__all__ = ["Option", "BooleanOption", "PathOption", "PositionalArgument", + "_debug"] + + +def _debug(msg): + """ Option parsing happens before verbose/debug have been set -- + they're options, after all -- so option parsing verbosity is + enabled by changing this to True. The verbosity here is primarily + of use to developers. """ + if os.environ.get('BCFG2_OPTIONS_DEBUG', '0') == '1': + print(msg) + #: A dict that records a mapping of argparse action name (e.g., #: "store_true") to the argparse Action class for it. See @@ -158,6 +169,10 @@ class Option(object): the appropriate default value in the appropriate format.""" for parser, action in self.actions.items(): if hasattr(action, "finalize"): + if parser: + _debug("Finalizing %s for %s" % (self, parser)) + else: + _debug("Finalizing %s" % self) action.finalize(parser, namespace) def from_config(self, cfp): @@ -181,23 +196,25 @@ class Option(object): exclude.update(o.cf[1] for o in parser.option_list if o.cf and o.cf[0] == self.cf[0]) - return dict([(o, cfp.get(self.cf[0], o)) - for o in fnmatch.filter(cfp.options(self.cf[0]), - self.cf[1]) - if o not in exclude]) + rv = dict([(o, cfp.get(self.cf[0], o)) + for o in fnmatch.filter(cfp.options(self.cf[0]), + self.cf[1]) + if o not in exclude]) else: - return dict() + rv = dict() else: + if self.type: + rtype = self.type + else: + rtype = lambda x: x try: - val = cfp.getboolean(*self.cf) + rv = rtype(cfp.getboolean(*self.cf)) except ValueError: - val = cfp.get(*self.cf) + rv = rtype(cfp.get(*self.cf)) except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return None - if self.type: - return self.type(val) - else: - return val + rv = None + _debug("Setting %s from config file(s): %s" % (self, rv)) + return rv def default_from_config(self, cfp): """ Set the default value of this option from the config file @@ -208,9 +225,13 @@ class Option(object): """ if self.env and self.env in os.environ: self.default = os.environ[self.env] + _debug("Setting the default of %s from environment: %s" % + (self, self.default)) else: val = self.from_config(cfp) if val is not None: + _debug("Setting the default of %s from config: %s" % + (self, val)) self.default = val def _get_default(self): @@ -250,13 +271,17 @@ class Option(object): self.parsers.append(parser) if self.args: # cli option + _debug("Adding %s to %s as a CLI option" % (self, parser)) action = parser.add_argument(*self.args, **self._kwargs) if not self._dest: self._dest = action.dest if self._default: action.default = self._default self.actions[parser] = action - # else, config file-only option + else: + # else, config file-only option + _debug("Adding %s to %s as a config file-only option" % + (self, parser)) class PathOption(Option): @@ -281,6 +306,26 @@ class PathOption(Option): Option.__init__(self, *args, **kwargs) +class _BooleanOptionAction(argparse.Action): + """ BooleanOptionAction sets a boolean value in the following ways: + - if None is passed, store the default + - if the option_string is not None, then the option was passed on the + command line, thus store the opposite of the default (this is the + argparse store_true and store_false behavior) + - if a boolean value is passed, use that + + Defined here instead of :mod:`Bcfg2.Options.Actions` because otherwise + there is a circular import Options -> Actions -> Parser -> Options """ + + def __call__(self, parser, namespace, values, option_string=None): + if values is None: + setattr(namespace, self.dest, self.default) + elif option_string is not None: + setattr(namespace, self.dest, not self.default) + else: + setattr(namespace, self.dest, bool(values)) + + class BooleanOption(Option): """ Shortcut for boolean options. The default is False, but this can easily be overridden: @@ -292,11 +337,10 @@ class BooleanOption(Option): "--dwim", default=True, help="Do What I Mean")] """ def __init__(self, *args, **kwargs): - if 'default' in kwargs and kwargs['default']: - kwargs.setdefault('action', 'store_false') - else: - kwargs.setdefault('action', 'store_true') - kwargs.setdefault('default', False) + kwargs.setdefault('action', _BooleanOptionAction) + kwargs.setdefault('nargs', 0) + kwargs.setdefault('default', False) + Option.__init__(self, *args, **kwargs) diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index 80f966246..677a69e4c 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -5,7 +5,7 @@ import sys import argparse from Bcfg2.version import __version__ from Bcfg2.Compat import ConfigParser -from Bcfg2.Options import Option, PathOption, BooleanOption +from Bcfg2.Options import Option, PathOption, BooleanOption, _debug __all__ = ["setup", "OptionParserException", "Parser", "get_parser"] @@ -37,6 +37,7 @@ class Parser(argparse.ArgumentParser): #: Option for specifying the path to the Bcfg2 config file configfile = PathOption('-C', '--config', + env="BCFG2_CONFIG_FILE", help="Path to configuration file", default="/etc/bcfg2.conf") @@ -121,14 +122,16 @@ class Parser(argparse.ArgumentParser): """ Add a component (and all of its options) to the parser. """ if component not in self.components: + _debug("Adding component %s to %s" % (component, self)) self.components.append(component) if hasattr(component, "options"): self.add_options(getattr(component, "options")) - def _set_defaults(self): + def _set_defaults_from_config(self): """ Set defaults from the config file for all options that can come from the config file, but haven't yet had their default set """ + _debug("Setting defaults on all options") for opt in self.option_list: if opt not in self._defaults_set: opt.default_from_config(self._cfp) @@ -138,15 +141,15 @@ class Parser(argparse.ArgumentParser): """ populate the namespace with default values for any options that aren't already in the namespace (i.e., options without CLI arguments) """ + _debug("Parsing config file-only options") for opt in self.option_list[:]: if not opt.args and opt.dest not in self.namespace: value = opt.default if value: - for parser, action in opt.actions.items(): - if parser is None: - action(self, self.namespace, value) - else: - action(parser, parser.namespace, value) + for _, action in opt.actions.items(): + _debug("Setting config file-only option %s to %s" % + (opt, value)) + action(self, self.namespace, value) else: setattr(self.namespace, opt.dest, value) @@ -155,6 +158,7 @@ class Parser(argparse.ArgumentParser): additional post-processing step. (Mostly :class:`Bcfg2.Options.Actions.ComponentAction` subclasses.) """ + _debug("Finalizing options") for opt in self.option_list[:]: opt.finalize(self.namespace) @@ -162,20 +166,23 @@ class Parser(argparse.ArgumentParser): """ Delete all options from the namespace except for a few predefined values and config file options. """ self.parsed = False + _debug("Resetting namespace") for attr in dir(self.namespace): if (not attr.startswith("_") and attr not in ['uri', 'version', 'name'] and attr not in self._config_files): + _debug("Deleting %s" % attr) delattr(self.namespace, attr) def add_config_file(self, dest, cfile, reparse=True): """ Add a config file, which triggers a full reparse of all options. """ if dest not in self._config_files: + _debug("Adding new config file %s for %s" % (cfile, dest)) self._reset_namespace() self._cfp.read([cfile]) self._defaults_set = [] - self._set_defaults() + self._set_defaults_from_config() if reparse: self._parse_config_options() self._config_files.append(dest) @@ -188,6 +195,7 @@ class Parser(argparse.ArgumentParser): (I.e., the argument list that was initially parsed.) :type argv: list """ + _debug("Reparsing all options") self._reset_namespace() self.parse(argv or self.argv) @@ -200,15 +208,19 @@ class Parser(argparse.ArgumentParser): :func:`Bcfg2.Options.Parser.reparse`. :type argv: list """ + _debug("Parsing options") if argv is None: argv = sys.argv[1:] if self.parsed and self.argv == argv: + _debug("Returning already parsed namespace") return self.namespace self.argv = argv # phase 1: get and read config file + _debug("Option parsing phase 1: Get and read main config file") bootstrap_parser = argparse.ArgumentParser(add_help=False) self.configfile.add_to_parser(bootstrap_parser) + self.configfile.default_from_config(self._cfp) bootstrap = bootstrap_parser.parse_known_args(args=self.argv)[0] # check whether the specified bcfg2.conf exists @@ -219,6 +231,7 @@ class Parser(argparse.ArgumentParser): # phase 2: re-parse command line for early options; currently, # that's database options + _debug("Option parsing phase 2: Parse early options") if not self._early: early_opts = argparse.Namespace() early_parser = Parser(add_help=False, namespace=early_opts, @@ -232,35 +245,62 @@ class Parser(argparse.ArgumentParser): early_components.append(component) early_parser.add_component(component) early_parser.parse(self.argv) + _debug("Early parsing complete, calling hooks") for component in early_components: if hasattr(component, "component_parsed_hook"): + _debug("Calling component_parsed_hook on %s" % component) getattr(component, "component_parsed_hook")(early_opts) # phase 3: re-parse command line, loading additional # components, until all components have been loaded. On each # iteration, set defaults from config file/environment # variables + _debug("Option parsing phase 3: Main parser loop") + # _set_defaults_from_config must be called before _parse_config_options + # This is due to a tricky interaction between the two methods: + # + # (1) _set_defaults_from_config does what its name implies, it updates + # the "default" property of each Option based on the value that exists + # in the config. + # + # (2) _parse_config_options will look at each option and set it to the + # default value that is _currently_ defined. If the option does not + # exist in the namespace, it will be added. The method carefully + # avoids overwriting the value of an option that is already defined in + # the namespace. + # + # Thus, if _set_defaults_from_config has not been called yet when + # _parse_config_options is called, all config file options will get set + # to their hardcoded defaults. This process defines the options in the + # namespace and _parse_config_options will never look at them again. + self._set_defaults_from_config() self._parse_config_options() while not self.parsed: self.parsed = True - self._set_defaults() + self._set_defaults_from_config() self.parse_known_args(args=self.argv, namespace=self.namespace) self._parse_config_options() self._finalize() - self._parse_config_options() # phase 4: fix up <repository> macros + _debug("Option parsing phase 4: Fix up macros") repo = getattr(self.namespace, "repository", repository.default) for attr in dir(self.namespace): value = getattr(self.namespace, attr) - if not attr.startswith("_") and hasattr(value, "replace"): + if (not attr.startswith("_") and + hasattr(value, "replace") and + "<repository>" in value): setattr(self.namespace, attr, value.replace("<repository>", repo, 1)) + _debug("Fixing up macros in %s: %s -> %s" % + (attr, value, getattr(self.namespace, attr))) # phase 5: call post-parsing hooks + _debug("Option parsing phase 5: Call hooks") if not self._early: for component in self.components: if hasattr(component, "options_parsed_hook"): + _debug("Calling post-parsing hook on %s" % component) getattr(component, "options_parsed_hook")() return self.namespace diff --git a/src/lib/Bcfg2/Options/Types.py b/src/lib/Bcfg2/Options/Types.py index 2f0fd7d52..d11e54fba 100644 --- a/src/lib/Bcfg2/Options/Types.py +++ b/src/lib/Bcfg2/Options/Types.py @@ -50,6 +50,15 @@ def comma_dict(value): return result +def anchored_regex_list(value): + """ Split an option string on whitespace and compile each element as + an anchored regex """ + try: + return [re.compile('^' + x + '$') for x in re.split(r'\s+', value)] + except re.error: + raise ValueError("Not a list of regexes", value) + + def octal(value): """ Given an octal string, get an integer representation. """ return int(value, 8) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 6c1dfdccb..12c9cdaa8 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -6,6 +6,7 @@ import time import threading # pylint: disable=E0611 +from lockfile import LockFailed, LockTimeout try: from lockfile.pidlockfile import PIDLockFile from lockfile import Error as PIDFileError @@ -63,6 +64,8 @@ class ReportingCollector(object): bcfg2-admin""" self.terminate = None self.context = None + self.children = [] + self.cleanup_threshold = 25 if Bcfg2.Options.setup.debug: level = logging.DEBUG @@ -106,12 +109,24 @@ class ReportingCollector(object): self.terminate = threading.Event() atexit.register(self.shutdown) self.context = daemon.DaemonContext(detach_process=True) + iter = 0 if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") try: self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) self.context.open() + except LockFailed: + self.logger.error("Failed to daemonize: %s" % + sys.exc_info()[1]) + self.shutdown() + return + except LockTimeout: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + self.setup['daemon']) + self.shutdown() + return except PIDFileError: self.logger.error("Error writing pid file: %s" % sys.exc_info()[1]) @@ -128,6 +143,13 @@ class ReportingCollector(object): continue store_thread = ReportingStoreThread(interaction, self.storage) store_thread.start() + self.children.append(store_thread) + + iter += 1 + if iter >= self.cleanup_threshold: + self.reap_children() + iter = 0 + except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() @@ -147,3 +169,16 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() + + def reap_children(self): + """Join any non-live threads""" + newlist = [] + + self.logger.debug("Starting reap_children") + for child in self.children: + if child.isAlive(): + newlist.append(child) + else: + child.join() + self.logger.debug("Joined child thread %s" % child.getName()) + self.children = newlist diff --git a/src/lib/Bcfg2/Reporting/Compat.py b/src/lib/Bcfg2/Reporting/Compat.py index 57261970d..9113fdb91 100644 --- a/src/lib/Bcfg2/Reporting/Compat.py +++ b/src/lib/Bcfg2/Reporting/Compat.py @@ -10,9 +10,7 @@ if VERSION[0] == 1 and VERSION[1] < 6: try: # Django < 1.6 - from django.conf.urls import defaults - django_urls = defaults + from django.conf.urls.defaults import url, patterns except ImportError: # Django > 1.6 - from django.conf import urls - django_urls = urls + from django.conf.urls import url, patterns diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py index 35c09a7e1..219d74584 100755 --- a/src/lib/Bcfg2/Reporting/Reports.py +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -43,6 +43,8 @@ def print_fields(fields, client, fmt, extra=None): fdata.append(client.current_interaction.extra_count) elif field == 'bad': fdata.append((client.current_interaction.bad_count)) + elif field == 'stale': + fdata.append(client.current_interaction.isstale()) else: try: fdata.append(getattr(client, field)) diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index c223c3c73..406216861 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -4,6 +4,7 @@ The base for the original DjangoORM (DBStats) from lxml import etree from datetime import datetime +import traceback from time import strptime import Bcfg2.Options import Bcfg2.DBSettings @@ -14,6 +15,7 @@ from django.core import management from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models import FieldDoesNotExist from django.core.cache import cache +from django import db #Used by GetCurrentEntry import difflib @@ -368,7 +370,12 @@ class DjangoORM(StorageBase): self._import_interaction(interaction) except: self.logger.error("Failed to import interaction: %s" % - sys.exc_info()[1]) + traceback.format_exc().splitlines()[-1]) + finally: + self.logger.debug("%s: Closing database connection" % + self.__class__.__name__) + db.close_connection() + def validate(self): """Validate backend storage. Should be called once when loaded""" @@ -380,9 +387,9 @@ class DjangoORM(StorageBase): vrb = 1 else: vrb = 0 - management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, - interactive=False) + Bcfg2.DBSettings.sync_databases(verbosity=vrb, interactive=False) + Bcfg2.DBSettings.migrate_databases(verbosity=vrb, + interactive=False) except: msg = "Failed to update database schema: %s" % sys.exc_info()[1] self.logger.error(msg) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 0598e4d33..2d96990b1 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -3,7 +3,7 @@ import sys from django.core.exceptions import ImproperlyConfigured try: - from django.db import models, backend, connection + from django.db import models, backend, connections except ImproperlyConfigured: e = sys.exc_info()[1] print("Reports: unable to import django models: %s" % e) @@ -12,6 +12,7 @@ except ImproperlyConfigured: from django.core.cache import cache from datetime import datetime, timedelta from Bcfg2.Compat import cPickle +from Bcfg2.DBSettings import get_db_label TYPE_GOOD = 0 @@ -61,7 +62,8 @@ def _quote(value): global _our_backend if not _our_backend: try: - _our_backend = backend.DatabaseOperations(connection) + _our_backend = backend.DatabaseOperations( + connections[get_db_label('Reporting')]) except TypeError: _our_backend = backend.DatabaseOperations() return _our_backend.quote_name(value) @@ -91,8 +93,8 @@ class InteractionManager(models.Manager): maxdate -- datetime object. Most recent date to pull. (default None) """ - from django.db import connection - cursor = connection.cursor() + from django.db import connections + cursor = connections[get_db_label('Reporting')].cursor() cfilter = "expiration is null" sql = 'select ri.id, x.client_id from ' + \ diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 7edf3a949..8b197231c 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5 <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.3</span> + <span>Bcfg2 Version 1.4.0pre1</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index 33c78a5f0..6a314bd88 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -32,7 +32,7 @@ This is needed for Django versions less than 1.5 <td class='right_column_narrow'>{{ entry.bad_count }}</td> <td class='right_column_narrow'>{{ entry.modified_count }}</td> <td class='right_column_narrow'>{{ entry.extra_count }}</td> - <td class='right_column'><span {% if entry.timestamp|isstale:entry_max %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> + <td class='right_column'><span {% if entry.isstale %}class='dirty-lineitem'{% endif %}>{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</span></td> <td class='right_column_wide'> {% if entry.server %} <a href='{% add_url_filter server=entry.server %}'>{{ entry.server }}</a> diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 489682f30..4a93e77e0 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -189,19 +189,6 @@ def build_metric_list(mdict): @register.filter -def isstale(timestamp, entry_max=None): - """ - Check for a stale timestamp - - Compares two timestamps and returns True if the - difference is greater then 24 hours. - """ - if not entry_max: - entry_max = datetime.now() - return entry_max - timestamp > timedelta(hours=24) - - -@register.filter def sort_interactions_by_name(value): """ Sort an interaction list by client name @@ -318,7 +305,11 @@ def determine_client_state(entry): dirty. If the client is reporting dirty, this will figure out just _how_ dirty and adjust the color accordingly. """ + if entry.isstale(): + return "stale-lineitem" if entry.state == 'clean': + if entry.extra_count > 0: + return "extra-lineitem" return "clean-lineitem" bad_percentage = 100 * (float(entry.bad_count) / entry.total_count) diff --git a/src/lib/Bcfg2/Reporting/urls.py b/src/lib/Bcfg2/Reporting/urls.py index a9e5690be..3a40cb932 100644 --- a/src/lib/Bcfg2/Reporting/urls.py +++ b/src/lib/Bcfg2/Reporting/urls.py @@ -1,4 +1,4 @@ -from Bcfg2.Reporting.Compat.django_urls import * +from Bcfg2.Reporting.Compat import url, patterns # django compat imports from django.core.urlresolvers import reverse, NoReverseMatch from django.http import HttpResponsePermanentRedirect from Bcfg2.Reporting.utils import filteredUrls, paginatedUrls, timeviewUrls diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index d9b8213b1..0d394fcd8 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -1,5 +1,4 @@ """Helper functions for reports""" -from Bcfg2.Reporting.Compat.django_urls import * import re """List of filters provided by filteredUrls""" diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index c7c2a503f..0b8ed65cc 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -13,7 +13,7 @@ from django.http import \ from django.shortcuts import render_to_response, get_object_or_404 from django.core.urlresolvers import \ resolve, reverse, Resolver404, NoReverseMatch -from django.db import connection, DatabaseError +from django.db import DatabaseError from django.db.models import Q, Count from Bcfg2.Reporting.models import * diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 207106596..0807fb2b0 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -173,15 +173,33 @@ class Backup(AdminCmd): class Client(_ServerAdminCmd): - """ Create, delete, or list client entries """ + """ Create, modify, delete, or list client entries """ + __plugin_whitelist__ = ["Metadata"] options = _ServerAdminCmd.options + [ Bcfg2.Options.PositionalArgument( "mode", - choices=["add", "del", "list"]), - Bcfg2.Options.PositionalArgument("hostname", nargs='?')] - - __plugin_whitelist__ = ["Metadata"] + choices=["add", "del", "delete", "remove", "rm", "up", "update", + "list"]), + Bcfg2.Options.PositionalArgument("hostname", nargs='?'), + Bcfg2.Options.PositionalArgument("attributes", metavar="KEY=VALUE", + nargs='*')] + + valid_attribs = ['profile', 'uuid', 'password', 'floating', 'secure', + 'address', 'auth'] + + def get_attribs(self, setup): + """ Get attributes for adding or updating a client from the command + line """ + attr_d = {} + for i in setup.attributes: + attr, val = i.split('=', 1) + if attr not in self.valid_attribs: + print("Attribute %s unknown. Valid attributes: %s" % + (attr, self.valid_attribs)) + raise SystemExit(1) + attr_d[attr] = val + return attr_d def run(self, setup): if setup.mode != 'list' and not setup.hostname: @@ -189,23 +207,32 @@ class Client(_ServerAdminCmd): elif setup.mode == 'list' and setup.hostname: self.logger.warning("<hostname> is not honored in list mode") - if setup.mode == 'add': - try: - self.metadata.add_client(setup.hostname) - except MetadataConsistencyError: - err = sys.exc_info()[1] - self.errExit("Error adding client %s: %s" % (setup.hostname, - err)) - elif setup.mode == 'del': + if setup.mode == 'list': + for client in self.metadata.list_clients(): + print(client) + else: + include_attribs = True + if setup.mode == 'add': + func = self.metadata.add_client + action = "adding" + elif setup.mode in ['up', 'update']: + func = self.metadata.update_client + action = "updating" + elif setup.mode in ['del', 'delete', 'rm', 'remove']: + func = self.metadata.remove_client + include_attribs = False + action = "deleting" + + if include_attribs: + args = (setup.hostname, self.get_attribs(setup)) + else: + args = (setup.hostname,) try: - self.metadata.remove_client(setup.hostname) + func(*args) except MetadataConsistencyError: err = sys.exc_info()[1] - self.errExit("Error deleting client %s: %s" % (setup.hostname, - err)) - elif setup.mode == 'list': - for client in self.metadata.list_clients(): - print(client) + self.errExit("Error %s client %s: %s" % (setup.hostname, + action, err)) class Compare(AdminCmd): @@ -885,8 +912,9 @@ if HAS_DJANGO: def run(self, setup): Bcfg2.Server.models.load_models() try: - management.call_command("syncdb", interactive=False, - verbosity=setup.verbose + setup.debug) + Bcfg2.DBSettings.sync_databases( + interactive=False, + verbosity=setup.verbose + setup.debug) except ImproperlyConfigured: err = sys.exc_info()[1] self.logger.error("Django configuration problem: %s" % err) @@ -933,10 +961,10 @@ if HAS_REPORTS: def run(self, setup): verbose = setup.verbose + setup.debug try: - management.call_command("syncdb", interactive=False, - verbosity=verbose) - management.call_command("migrate", interactive=False, - verbosity=verbose) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=verbose) + Bcfg2.DBSettings.migrate_databases(interactive=False, + verbosity=verbose) except: # pylint: disable=W0702 self.errExit("%s failed: %s" % (self.__class__.__name__.title(), diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 0023e9313..769addf55 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -113,7 +113,8 @@ class BuiltinCore(NetworkCore): keyfile=Bcfg2.Options.setup.key, certfile=Bcfg2.Options.setup.cert, register=False, - ca=Bcfg2.Options.setup.ca) + ca=Bcfg2.Options.setup.ca, + protocol=Bcfg2.Options.setup.protocol) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Server startup failed: %s" % err) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 398053374..892f2832a 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,14 +19,13 @@ import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, wraps # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics try: from django.core.exceptions import ImproperlyConfigured - from django.core import management import django.conf HAS_DJANGO = True except ImportError: @@ -74,6 +73,24 @@ def sort_xml(node, key=None): node[:] = sorted_children +def close_db_connection(func): + """ Decorator that closes the Django database connection at the end of + the function. This should decorate any exposed function that + might open a database connection. """ + @wraps(func) + def inner(self, *args, **kwargs): + """ The decorated function """ + rv = func(self, *args, **kwargs) + if self._database_available: # pylint: disable=W0212 + from django import db + self.logger.debug("%s: Closing database connection" % + threading.current_thread().name) + db.close_connection() + return rv + + return inner + + class CoreInitError(Exception): """ Raised when the server core cannot be initialized. """ pass @@ -114,7 +131,8 @@ class Core(object): Bcfg2.Options.Common.repository, Bcfg2.Options.Common.filemonitor, Bcfg2.Options.BooleanOption( - cf=('server', 'fam_blocking'), default=False, + "--no-fam-blocking", cf=('server', 'fam_blocking'), + dest="fam_blocking", default=True, help='FAM blocks on startup until all events are processed'), Bcfg2.Options.BooleanOption( cf=('logging', 'performance'), dest="perflog", @@ -128,6 +146,10 @@ class Core(object): default='off', choices=['off', 'on', 'initial', 'cautious', 'aggressive'])] + #: The name of this server core. This can be overridden by core + #: implementations to provide a more specific name. + name = "Core" + def __init__(self): # pylint: disable=R0912,R0915 """ .. automethod:: _run @@ -196,6 +218,12 @@ class Core(object): self.revision = '-1' atexit.register(self.shutdown) + #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly, + #: then :mod:`atexit` calls it *again*, so it gets called + #: twice. This is potentially bad, so we use + #: :attr:`Bcfg2.Server.Core._running` as a flag to determine + #: if the core needs to be shutdown, and only do it once. + self._running = True #: Threading event to signal worker threads (e.g., #: :attr:`fam_thread`) to shutdown @@ -236,16 +264,16 @@ class Core(object): self._database_available = False if HAS_DJANGO: try: - management.call_command("syncdb", interactive=False, - verbosity=0) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.logger.error("Django configuration problem: %s" % err) + self.logger.error("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] self.logger.error("Updating database %s failed: %s" % - (Bcfg2.Options.setup.db_name, err)) + (Bcfg2.Options.setup.db_name, + sys.exc_info()[1])) def __str__(self): return self.__class__.__name__ @@ -332,7 +360,7 @@ class Core(object): This does not start plugin threads; that is done later, in :func:`Bcfg2.Server.Core.BaseCore.run` """ for plugin in Bcfg2.Options.setup.plugins: - if not plugin in self.plugins: + if plugin not in self.plugins: self.init_plugin(plugin) # Remove blacklisted plugins @@ -403,14 +431,22 @@ class Core(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ - self.logger.info("Shutting down core...") + if not self._running: + self.logger.debug("%s: Core already shut down" % self.name) + return + self.logger.info("%s: Shutting down core..." % self.name) if not self.terminate.isSet(): self.terminate.set() - self.fam.shutdown() - self.logger.info("FAM shut down") - for plugin in list(self.plugins.values()): - plugin.shutdown() - self.logger.info("All plugins shut down") + self._running = False + self.fam.shutdown() + self.logger.info("%s: FAM shut down" % self.name) + for plugin in list(self.plugins.values()): + plugin.shutdown() + self.logger.info("%s: All plugins shut down" % self.name) + if self._database_available: + from django import db + self.logger.info("%s: Closing database connection" % self.name) + db.close_connection() @property def metadata_cache_mode(self): @@ -601,9 +637,10 @@ class Core(object): del entry.attrib['realname'] return ret except: - self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('realname'), - entry.get('name'))) + self.logger.error( + "Failed binding entry %s:%s with altsrc %s: %s" % + (entry.tag, entry.get('realname'), entry.get('name'), + sys.exc_info()[1])) entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -1052,6 +1089,7 @@ class Core(object): @exposed @track_statistics() + @close_db_connection def DeclareVersion(self, address, version): """ Declare the client version. @@ -1074,6 +1112,7 @@ class Core(object): return True @exposed + @close_db_connection def GetProbes(self, address): """ Fetch probes for the client. @@ -1099,6 +1138,7 @@ class Core(object): (client, err)) @exposed + @close_db_connection def RecvProbeData(self, address, probedata): """ Receive probe data from clients. @@ -1146,6 +1186,7 @@ class Core(object): return True @exposed + @close_db_connection def AssertProfile(self, address, profile): """ Set profile for a client. @@ -1165,6 +1206,7 @@ class Core(object): return True @exposed + @close_db_connection def GetConfig(self, address): """ Build config for a client by calling :func:`BuildConfiguration`. @@ -1184,6 +1226,7 @@ class Core(object): self.critical_error("Metadata consistency failure for %s" % client) @exposed + @close_db_connection def RecvStats(self, address, stats): """ Act on statistics upload with :func:`process_statistics`. @@ -1199,6 +1242,7 @@ class Core(object): return True @exposed + @close_db_connection def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. @@ -1326,8 +1370,16 @@ class NetworkCore(Core): daemonized, etc.""" options = Core.options + [ Bcfg2.Options.Common.daemon, Bcfg2.Options.Common.syslog, - Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, - Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.Common.protocol, + Bcfg2.Options.PathOption( + '--ssl-key', cf=('communication', 'key'), dest="key", + help='Path to SSL key', + default="/etc/pki/tls/private/bcfg2.key"), + Bcfg2.Options.PathOption( + cf=('communication', 'certificate'), dest="cert", + help='Path to SSL certificate', + default="/etc/pki/tls/certs/bcfg2.crt"), Bcfg2.Options.BooleanOption( '--listen-all', cf=('server', 'listen_all'), default=False, help="Listen on all interfaces"), diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index f8b602d90..b60302871 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -173,6 +173,17 @@ def ssl_encrypt(plaintext, passwd, algorithm=None, salt=None): return b64encode("Salted__" + salt + crypted) + "\n" +def is_encrypted(val): + """ Make a best guess if the value is encrypted or not. This just + checks to see if ``val`` is a base64-encoded string whose content + starts with "Salted__", so it may have (rare) false positives. It + will not have false negatives. """ + try: + return b64decode(val).startswith("Salted__") + except: # pylint: disable=W0702 + return False + + def bruteforce_decrypt(crypted, passphrases=None, algorithm=None): """ Convenience method to decrypt the given encrypted string by trying the given passphrases or all passphrases sequentially until @@ -233,6 +244,10 @@ class DecryptError(Exception): """ Exception raised when decryption fails. """ +class EncryptError(Exception): + """ Exception raised when encryption fails. """ + + class CryptoTool(object): """ Generic decryption/encryption interface base object """ @@ -319,6 +334,8 @@ class CfgEncryptor(Encryptor): Bcfg2.Options.setup.config) def encrypt(self): + if is_encrypted(self.data): + raise EncryptError("Data is alraedy encrypted") return ssl_encrypt(self.data, self.passphrase) def get_destination_filename(self, original_filename): @@ -355,7 +372,7 @@ class CfgDecryptor(Decryptor): class PropertiesCryptoMixin(object): """ Mixin to provide some common methods for Properties crypto """ - default_xpath = '//*' + default_xpath = '//*[@encrypted]' def _get_elements(self, xdata): """ Get the list of elements to encrypt or decrypt """ @@ -425,11 +442,13 @@ class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): def encrypt(self): xdata = lxml.etree.XML(self.data, parser=XMLParser) for elt in self._get_elements(xdata): + if is_encrypted(elt.text): + raise EncryptError("Element is already encrypted: %s" % + print_xml(elt)) try: pname, passphrase = self._get_element_passphrase(elt) except PassphraseError: - self.logger.error(str(sys.exc_info()[1])) - return False + raise EncryptError(str(sys.exc_info()[1])) self.logger.debug("Encrypting %s" % print_xml(elt)) elt.text = ssl_encrypt(elt.text, passphrase).strip() elt.set("encrypted", pname) @@ -441,7 +460,6 @@ class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): """ decryptor class for Properties files """ - default_xpath = '//*[@encrypted]' def decrypt(self): decrypted_any = False @@ -640,9 +658,9 @@ class CLI(object): if data is None: try: data = getattr(tool, mode)() - except DecryptError: - self.logger.error("Failed to %s %s, skipping" % (mode, - fname)) + except (EncryptError, DecryptError): + self.logger.error("Failed to %s %s, skipping: %s" % + (mode, fname, sys.exc_info()[1])) continue if Bcfg2.Options.setup.stdout: if len(Bcfg2.Options.setup.files) > 1: diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 69463ab4c..b349d20fd 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -27,11 +27,11 @@ class GaminEvent(Event): class Gamin(FileMonitor): """ File monitor backend with `Gamin - <http://people.gnome.org/~veillard/gamin/>`_ support. """ + <http://people.gnome.org/~veillard/gamin/>`_ support. **Deprecated.** """ - #: The Gamin backend is fairly decent, particularly newer - #: releases, so it has a fairly high priority. - __priority__ = 90 + #: The Gamin backend is deprecated, but better than pseudo, so it + #: has a medium priority. + __priority__ = 50 def __init__(self): FileMonitor.__init__(self) @@ -46,6 +46,9 @@ class Gamin(FileMonitor): #: The queue used to record monitors that are added before #: :func:`start` has been called and :attr:`mon` is created. self.add_q = [] + + self.logger.warning("The Gamin file monitor backend is deprecated. " + "Please switch to a supported file monitor.") __init__.__doc__ = FileMonitor.__init__.__doc__ def start(self): diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index b8eb06aa1..c4b34a469 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -212,7 +212,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__ def shutdown(self): - if self.notifier: + if self.started and self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py index 0caf4d7ed..aee15cb5d 100644 --- a/src/lib/Bcfg2/Server/Lint/Bundler.py +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -1,12 +1,12 @@ """ ``bcfg2-lint`` plugin for :ref:`Bundler -<server-plugins-structures-bundler-index>` """ +<server-plugins-structures-bundler>` """ from Bcfg2.Server.Lint import ServerPlugin class Bundler(ServerPlugin): """ Perform various :ref:`Bundler - <server-plugins-structures-bundler-index>` checks. """ + <server-plugins-structures-bundler>` checks. """ def Run(self): self.missing_bundles() diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index e2d1ec597..fc4506c12 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -9,6 +9,7 @@ from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ import CfgPlaintextGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML @@ -76,6 +77,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for Cheetah-templated Cfg files"), Bcfg2.Options.Option( + cf=("Comments", "jinja2_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "jinja2_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Jinja2-templated Cfg files"), + Bcfg2.Options.Option( cf=("Comments", "infoxml_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for info.xml files"), @@ -235,6 +244,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): rtype = "cfg" elif isinstance(entry, CfgCheetahGenerator): rtype = "cheetah" + elif isinstance(entry, CfgJinja2Generator): + rtype = "jinja2" elif isinstance(entry, CfgInfoXML): self.check_xml(entry.infoxml.name, entry.infoxml.pnode.data, diff --git a/src/lib/Bcfg2/Server/Lint/Crypto.py b/src/lib/Bcfg2/Server/Lint/Crypto.py new file mode 100644 index 000000000..53a54031c --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Crypto.py @@ -0,0 +1,61 @@ +""" Check for data that claims to be encrypted, but is not. """ + +import os +import lxml.etree +import Bcfg2.Options +from Bcfg2.Server.Lint import ServerlessPlugin +from Bcfg2.Server.Encryption import is_encrypted + + +class Crypto(ServerlessPlugin): + """ Check for templated scripts or executables. """ + + def Run(self): + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + self.check_cfg() + if os.path.exists(os.path.join(Bcfg2.Options.setup.repository, + "Properties")): + self.check_properties() + # TODO: check all XML files + + @classmethod + def Errors(cls): + return {"unencrypted-cfg": "error", + "empty-encrypted-properties": "error", + "unencrypted-properties": "error"} + + def check_cfg(self): + """ Check for Cfg files that end in .crypt but aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Cfg")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".crypt"): + if not is_encrypted(open(fpath).read()): + self.LintError( + "unencrypted-cfg", + "%s is a .crypt file, but it is not encrypted" % + fpath) + + def check_properties(self): + """ Check for Properties data that has an ``encrypted`` attribute but + aren't encrypted """ + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, "Properties")): + for fname in files: + fpath = os.path.join(root, fname) + if self.HandlesFile(fpath) and fname.endswith(".xml"): + xdata = lxml.etree.parse(fpath) + for elt in xdata.xpath('//*[@encrypted]'): + if not elt.text: + self.LintError( + "empty-encrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but no text content: %s" % + (fpath, self.RenderXML(elt))) + elif not is_encrypted(elt.text): + self.LintError( + "unencrypted-properties", + "Element in %s has an 'encrypted' attribute, " + "but is not encrypted: %s" % + (fpath, self.RenderXML(elt))) diff --git a/src/lib/Bcfg2/Server/Lint/Jinja2.py b/src/lib/Bcfg2/Server/Lint/Jinja2.py new file mode 100755 index 000000000..333249cc2 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Jinja2.py @@ -0,0 +1,41 @@ +""" Check Jinja2 templates for syntax errors. """ + +import sys +import Bcfg2.Server.Lint +from jinja2 import Template, TemplateSyntaxError +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator + + +class Jinja2(Bcfg2.Server.Lint.ServerPlugin): + """ Check Jinja2 templates for syntax errors. """ + + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + + @classmethod + def Errors(cls): + return {"jinja2-syntax-error": "error", + "unknown-jinja2-error": "error"} + + def check_template(self, entry): + """ Generic check for all jinja2 templates """ + try: + Template(entry.data.decode(entry.encoding)) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("jinja2-syntax-error", + "Jinja2 syntax error in %s: %s" % (entry.name, err)) + except: + err = sys.exc_info()[1] + self.LintError("unknown-jinja2-error", + "Unknown Jinja2 error in %s: %s" % (entry.name, + err)) + + def check_cfg(self): + """ Check jinja2 templates in Cfg for syntax errors. """ + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + isinstance(entry, CfgJinja2Generator)): + self.check_template(entry) diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 5d9e229fa..ebf4c4954 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -123,12 +123,30 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"unknown-entry-type": "error", + return {"missing-elements": "error", + "unknown-entry-type": "error", "unknown-entry-tag": "error", "required-attrs-missing": "error", "required-attr-format": "error", "extra-attrs": "warning"} + def check_default_acl(self, path): + """ Check that a default ACL contains either no entries or minimum + required entries """ + defaults = 0 + if path.xpath("ACL[@type='default' and @scope='user' and @user='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='group' and @group='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='other']"): + defaults += 1 + if defaults > 0 and defaults < 3: + self.LintError( + "missing-elements", + "A Path must have either no default ACLs or at" + " least default:user::, default:group:: and" + " default:other::") + def check_packages(self): """ Check Packages sources for Source entries with missing attributes. """ @@ -172,7 +190,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): rules.name)) def check_bundles(self): - """ Check bundles for BoundPath entries with missing + """ Check bundles for BoundPath and BoundPackage entries with missing attrs. """ if 'Bundler' not in self.core.plugins: return @@ -183,6 +201,25 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "//*[substring(name(), 1, 5) = 'Bound']"): self.check_entry(path, bundle.name) + # ensure that abstract Path tags have either name + # or glob specified + for path in bundle.xdata.xpath("//Path"): + if ('name' not in path.attrib and + 'glob' not in path.attrib): + self.LintError( + "required-attrs-missing", + "Path tags require either a 'name' or 'glob' " + "attribute: \n%s" % self.RenderXML(path)) + # ensure that abstract Package tags have either name + # or group specified + for package in bundle.xdata.xpath("//Package"): + if ('name' not in package.attrib and + 'group' not in package.attrib): + self.LintError( + "required-attrs-missing", + "Package tags require either a 'name' or 'group' " + "attribute: \n%s" % self.RenderXML(package)) + def check_entry(self, entry, filename): """ Generic entry check. @@ -221,6 +258,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): required_attrs['major'] = is_device_mode required_attrs['minor'] = is_device_mode + if tag == 'Path': + self.check_default_acl(entry) + if tag == 'ACL' and 'scope' in required_attrs: required_attrs[entry.get('scope')] = is_username diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py new file mode 100644 index 000000000..5a80a5884 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -0,0 +1,80 @@ +""" Check for templated scripts or executables. """ + +import os +import stat +import Bcfg2.Server.Lint +from Bcfg2.Compat import any # pylint: disable=W0622 +from Bcfg2.Server.Plugin import default_path_metadata +from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \ + CfgEncryptedGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \ + CfgEncryptedCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedJinja2Generator import \ + CfgEncryptedJinja2Generator + + +class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): + """ Check for templated scripts or executables. """ + templates = [CfgGenshiGenerator, CfgCheetahGenerator, CfgJinja2Generator, + CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator, + CfgEncryptedJinja2Generator] + extensions = [".pl", ".py", ".sh", ".rb"] + + def Run(self): + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + any(isinstance(entry, t) for t in self.templates)): + self.check_template(entryset, entry) + + @classmethod + def Errors(cls): + return {"templated-script": "warning", + "templated-executable": "warning"} + + def check_template(self, entryset, entry): + """ Check a template to see if it's a script or an executable. """ + # first, check for a known script extension + ext = os.path.splitext(entryset.path)[1] + if ext in self.extensions: + self.LintError("templated-script", + "Templated script found: %s\n" + "File has a known script extension: %s\n" + "Template a config file for the script instead" % + (entry.name, ext)) + return + + # next, check for a shebang line + firstline = open(entry.name).readline() + if firstline.startswith("#!"): + self.LintError("templated-script", + "Templated script found: %s\n" + "File starts with a shebang: %s\n" + "Template a config file for the script instead" % + (entry.name, firstline)) + return + + # finally, check for executable permissions in info.xml + for entry in entryset.entries.values(): + if isinstance(entry, CfgInfoXML): + for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + try: + mode = int( + pinfo.get("mode", + default_path_metadata()['mode']), 8) + except ValueError: + # LintError will be produced by RequiredAttrs plugin + self.logger.warning("Non-octal mode: %s" % mode) + continue + if mode & stat.S_IXUSR != 0: + self.LintError( + "templated-executable", + "Templated executable found: %s\n" + "Template a config file for the executable instead" + % entry.name) + return diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py index fbd5a2893..a952da724 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -23,9 +23,11 @@ class TemplateHelper(ServerPlugin): def __init__(self, *args, **kwargs): ServerPlugin.__init__(self, *args, **kwargs) - self.reserved_keywords = dir(HelperModule("foo.py")) - self.reserved_defaults = \ - self.core.plugins['TemplateHelper'].reserved_defaults + # we instantiate a dummy helper to discover which keywords and + # defaults are reserved + dummy = HelperModule("foo.py") + self.reserved_keywords = dir(dummy) + self.reserved_defaults = dummy.reserved_defaults def Run(self): for helper in self.core.plugins['TemplateHelper'].entries.values(): diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index e38619355..0b3f1e24d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -90,6 +90,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "xml-failed-to-parse": "error", "xml-failed-to-read": "error", "xml-failed-to-verify": "error", + "xinclude-does-not-exist": "error", "input-output-error": "error"} def check_properties(self): @@ -113,9 +114,17 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): :type filename: string :returns: lxml.etree._ElementTree - the parsed data""" try: - return lxml.etree.parse(filename) - except SyntaxError: - result = self.cmd.run(["xmllint", filename]) + xdata = lxml.etree.parse(filename) + if self.files is None: + self._expand_wildcard_xincludes(xdata) + xdata.xinclude() + return xdata + except (lxml.etree.XIncludeError, SyntaxError): + cmd = ["xmllint", "--noout"] + if self.files is None: + cmd.append("--xinclude") + cmd.append(filename) + result = self.cmd.run(cmd) self.LintError("xml-failed-to-parse", "%s fails to parse:\n%s" % (filename, result.stdout + result.stderr)) @@ -125,6 +134,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def _expand_wildcard_xincludes(self, xdata): + """ a lightweight version of + :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """ + xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE + for el in xdata.findall('//' + xinclude): + name = el.get("href") + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name) + + # expand globs in xinclude, a bcfg2-specific extension + extras = glob.glob(fpath) + if not extras: + msg = "%s: %s does not exist, skipping: %s" % \ + (xdata.docinfo.URL, name, self.RenderXML(el)) + if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): + self.logger.debug(msg) + else: + self.LintError("xinclude-does-not-exist", msg) + + parent = el.getparent() + parent.remove(el) + for extra in extras: + if extra != xdata.docinfo.URL: + lxml.etree.SubElement(parent, xinclude, href=extra) + def validate(self, filename, schemafile, schema=None): """ Validate a file against the given schema. @@ -146,6 +182,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if not schema: return False datafile = self.parse(filename) + if not datafile: + return False if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py new file mode 100644 index 000000000..6383a3c99 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -0,0 +1,72 @@ +"""Ensure that all JSON files in the Bcfg2 repository are +valid. Currently, the only plugins that uses JSON are Ohai and +Properties.""" + +import os +import sys +import glob +import fnmatch +import Bcfg2.Server.Lint + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + + +class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): + """Ensure that all JSON files in the Bcfg2 repository are + valid. Currently, the only plugins that uses JSON are Ohai and + Properties. """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A list of file globs that give the path to JSON files. The + #: globs are extended :mod:`fnmatch` globs that also support + #: ``**``, which matches any number of any characters, + #: including forward slashes. + self.globs = ["Properties/*.json", "Ohai/*.json"] + self.files = self.get_files() + + def Run(self): + for path in self.files: + self.logger.debug("Validating JSON in %s" % path) + try: + json.load(open(path)) + except ValueError: + self.LintError("json-failed-to-parse", + "%s does not contain valid JSON: %s" % + (path, sys.exc_info()[1])) + + @classmethod + def Errors(cls): + return {"json-failed-to-parse": "error"} + + def get_files(self): + """Return a list of all JSON files to validate, based on + :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """ + if self.files is not None: + listfiles = lambda p: fnmatch.filter(self.files, + os.path.join('*', p)) + else: + listfiles = lambda p: glob.glob( + os.path.join(Bcfg2.Options.setup.repository, p)) + + rv = [] + for path in self.globs: + if '/**/' in path: + if self.files is not None: + rv.extend(listfiles(path)) + else: # self.files is None + fpath, fname = path.split('/**/') + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, + fpath)): + rv.extend([os.path.join(root, f) + for f in files if f == fname]) + else: + rv.extend(listfiles(path)) + return rv diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 8a793fd94..9b3e6ece2 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -13,6 +13,7 @@ import lxml.etree import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Plugins +from Bcfg2.Compat import walk_packages def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -297,11 +298,10 @@ class LintPluginAction(Bcfg2.Options.ComponentAction): bases = ['Bcfg2.Server.Lint'] def __call__(self, parser, namespace, values, option_string=None): - for plugin in getattr(Bcfg2.Options.setup, "plugins", []): - module = sys.modules[plugin.__module__] - if hasattr(module, "%sLint" % plugin.name): - print("Adding lint plugin %s" % plugin) - values.append(plugin) + plugins = getattr(Bcfg2.Options.setup, "plugins", []) + for lint_plugin in walk_packages(path=__path__): + if lint_plugin[1] in plugins: + values.append(lint_plugin[1]) Bcfg2.Options.ComponentAction.__call__(self, parser, namespace, values, option_string) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 294963669..724b34d8d 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -275,6 +275,7 @@ class ChildCore(Core): @exposed def GetConfig(self, client): """ Render the configuration for a client """ + self.metadata.update_client_list() self.logger.debug("%s: Building configuration for %s" % (self.name, client)) return lxml.etree.tostring(self.BuildConfiguration(client)) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 1cb5a7b3e..559612d1e 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -18,7 +18,7 @@ from Bcfg2.Compat import CmpMixin, wraps from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.interfaces import Generator, TemplateDataProvider from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ - PluginExecutionError + PluginExecutionError, PluginInitError try: import Bcfg2.Server.Encryption @@ -219,6 +219,18 @@ class DatabaseBacked(Plugin): .. private-include: _must_lock """ + def __init__(self, core): + Plugin.__init__(self, core) + use_db = getattr(Bcfg2.Options.setup, "%s_db" % self.name.lower(), + False) + if use_db and not HAS_DJANGO: + raise PluginInitError("%s is configured to use the database but " + "Django libraries are not found" % self.name) + elif use_db and not self.core.database_available: + raise PluginInitError("%s is configured to use the database but " + "the database is unavailable due to prior " + "errors" % self.name) + @property def _use_db(self): """ Whether or not this plugin is configured to use the @@ -227,11 +239,7 @@ class DatabaseBacked(Plugin): False) if use_db and HAS_DJANGO and self.core.database_available: return True - elif not use_db: - return False else: - self.logger.error("%s: use_database is true but django not found" % - self.name) return False @property @@ -267,7 +275,8 @@ class PluginDatabaseModel(object): inherit from. This is just a mixin; models must also inherit from django.db.models.Model to be valid Django models.""" - class Meta: # pylint: disable=C0111,W0232 + class Meta(object): # pylint: disable=W0232 + """ Model metadata options """ app_label = "Server" @@ -638,7 +647,13 @@ class XMLFileBacked(FileBacked): if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): self.logger.debug(msg) else: - self.logger.warning(msg) + self.logger.error(msg) + # add a FAM monitor for this path. this isn't perfect + # -- if there's an xinclude of "*.xml", we'll watch + # the literal filename "*.xml". but for non-globbing + # filenames, it works fine. + if fpath not in self.extra_monitors: + self.add_monitor(fpath) parent = el.getparent() parent.remove(el) @@ -748,9 +763,6 @@ class StructFile(XMLFileBacked): err)) if HAS_CRYPTO and self.encryption: - lax_decrypt = self.xdata.get( - "lax_decryption", - str(Bcfg2.Options.setup.lax_decryption)).lower() == "true" for el in self.xdata.xpath("//*[@encrypted]"): try: el.text = self._decrypt(el).encode('ascii', @@ -759,10 +771,14 @@ class StructFile(XMLFileBacked): self.logger.info("%s: Decrypted %s to gibberish, skipping" % (self.name, el.tag)) except Bcfg2.Server.Encryption.EVPError: + lax_decrypt = self.xdata.get( + "lax_decryption", + str(Bcfg2.Options.setup.lax_decryption)).lower() == \ + "true" msg = "Failed to decrypt %s element in %s" % (el.tag, self.name) if lax_decrypt: - self.logger.warning(msg) + self.logger.debug(msg) else: raise PluginExecutionError(msg) Index.__doc__ = XMLFileBacked.Index.__doc__ @@ -774,16 +790,11 @@ class StructFile(XMLFileBacked): passes = Bcfg2.Options.setup.passphrases try: passphrase = passes[element.get("encrypted")] - try: - return Bcfg2.Server.Encryption.ssl_decrypt(element.text, - passphrase) - except Bcfg2.Server.Encryption.EVPError: - # error is raised below - pass + return Bcfg2.Server.Encryption.ssl_decrypt(element.text, + passphrase) except KeyError: - # bruteforce_decrypt raises an EVPError with a sensible - # error message, so we just let it propagate up the stack - return Bcfg2.Server.Encryption.bruteforce_decrypt(element.text) + raise Bcfg2.Server.Encryption.EVPError("No passphrase named '%s'" % + element.get("encrypted")) raise Bcfg2.Server.Encryption.EVPError("Failed to decrypt") def _include_element(self, item, metadata, *args): @@ -818,7 +829,8 @@ class StructFile(XMLFileBacked): """ stream = self.template.generate( **get_xml_template_data(self, metadata)).filter(removecomment) - return lxml.etree.XML(stream.render('xml', strip_whitespace=False), + return lxml.etree.XML(stream.render('xml', + strip_whitespace=False).encode(), parser=Bcfg2.Server.XMLParser) def _match(self, item, metadata, *args): @@ -935,7 +947,7 @@ class InfoXML(StructFile): _include_tests = copy.copy(StructFile._include_tests) _include_tests['Path'] = lambda el, md, entry, *args: \ - entry.get("name") == el.get("name") + entry.get('realname', entry.get('name')) == el.get("name") def Match(self, metadata, entry): # pylint: disable=W0221 """ Implementation of diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 622b69c79..c45d6fa84 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -216,6 +216,10 @@ class Metadata(object): """ raise NotImplementedError + def update_client_list(self): + """ Re-read the cached list of clients """ + raise NotImplementedError + class Connector(object): """ Connector plugins augment client metadata instances with diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 8b9330c9b..41ee57b6d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -4,31 +4,30 @@ import os import re import sys import copy -import Bcfg2.Server -import Bcfg2.Server.Plugin +import fnmatch +import lxml.etree +from Bcfg2.Server.Plugin import StructFile, Plugin, Structure, \ + StructureValidator, XMLDirectoryBacked, Generator from genshi.template import TemplateError -class BundleFile(Bcfg2.Server.Plugin.StructFile): +class BundleFile(StructFile): """ Representation of a bundle XML file """ bundle_name_re = re.compile(r'^(?P<name>.*)\.(xml|genshi)$') def __init__(self, filename, should_monitor=False): - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, - should_monitor=should_monitor) + StructFile.__init__(self, filename, should_monitor=should_monitor) if self.name.endswith(".genshi"): self.logger.warning("Bundler: %s: Bundle filenames ending with " ".genshi are deprecated; add the Genshi XML " "namespace to a .xml bundle instead" % self.name) - __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ def Index(self): - Bcfg2.Server.Plugin.StructFile.Index(self) + StructFile.Index(self) if self.xdata.get("name"): self.logger.warning("Bundler: %s: Explicitly specifying bundle " "names is deprecated" % self.name) - Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ @property def bundle_name(self): @@ -37,9 +36,10 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): os.path.basename(self.name)).group("name") -class Bundler(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): +class Bundler(Plugin, + Structure, + StructureValidator, + XMLDirectoryBacked): """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' @@ -47,18 +47,30 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, patterns = re.compile(r'^.*\.(?:xml|genshi)$') def __init__(self, core): - Bcfg2.Server.Plugin.Plugin.__init__(self, core) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) + Plugin.__init__(self, core) + Structure.__init__(self) + StructureValidator.__init__(self) + XMLDirectoryBacked.__init__(self, self.data) #: Bundles by bundle name, rather than filename self.bundles = dict() def HandleEvent(self, event): - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - + XMLDirectoryBacked.HandleEvent(self, event) self.bundles = dict([(b.bundle_name, b) for b in self.entries.values()]) + def validate_structures(self, metadata, structures): + """ Translate <Path glob='...'/> entries into <Path name='...'/> + entries """ + for struct in structures: + for pathglob in struct.xpath("//Path[@glob]"): + for plugin in self.core.plugins_by_type(Generator): + for match in fnmatch.filter(plugin.Entries['Path'].keys(), + pathglob.get("glob")): + lxml.etree.SubElement(pathglob.getparent(), + "Path", name=match) + pathglob.getparent().remove(pathglob) + def BuildStructures(self, metadata): bundleset = [] bundles = copy.copy(metadata.bundles) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index e2a2f696a..849c75f70 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,6 +1,7 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: @@ -25,7 +26,6 @@ class CfgEncryptedGenerator(CfgGenerator): CfgGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") - __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): CfgGenerator.handle_event(self, event) @@ -35,11 +35,13 @@ class CfgEncryptedGenerator(CfgGenerator): try: self.data = bruteforce_decrypt(self.data) except EVPError: - raise PluginExecutionError("Failed to decrypt %s" % self.name) - handle_event.__doc__ = CfgGenerator.handle_event.__doc__ + msg = "Cfg: Failed to decrypt %s" % self.name + if Bcfg2.Options.setup.lax_decryption: + self.logger.debug(msg) + else: + raise PluginExecutionError(msg) def get_data(self, entry, metadata): if self.data is None: raise PluginExecutionError("Failed to decrypt %s" % self.name) return CfgGenerator.get_data(self, entry, metadata) - get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py new file mode 100644 index 000000000..c8da84ae0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedJinja2Generator.py @@ -0,0 +1,25 @@ +""" Handle encrypted Jinja2 templates (.crypt.jinja2 or +.jinja2.crypt files)""" + +from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator \ + import CfgEncryptedGenerator + + +class CfgEncryptedJinja2Generator(CfgJinja2Generator, CfgEncryptedGenerator): + """ CfgEncryptedJinja2Generator lets you encrypt your Jinja2 + :ref:`server-plugins-generators-cfg` files on the server """ + + #: handle .crypt.jinja2 or .jinja2.crypt files + __extensions__ = ['jinja2.crypt', 'crypt.jinja2'] + + #: Override low priority from parent class + __priority__ = 0 + + def handle_event(self, event): + CfgEncryptedGenerator.handle_event(self, event) + handle_event.__doc__ = CfgEncryptedGenerator.handle_event.__doc__ + + def get_data(self, entry, metadata): + return CfgJinja2Generator.get_data(self, entry, metadata) + get_data.__doc__ = CfgJinja2Generator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py new file mode 100644 index 000000000..e36ee78aa --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgJinja2Generator.py @@ -0,0 +1,52 @@ +""" The CfgJinja2Generator allows you to use the `Jinja2 +<http://jinja.pocoo.org/>`_ templating system to generate +:ref:`server-plugins-generators-cfg` files. """ + +import Bcfg2.Options +from Bcfg2.Server.Plugin import PluginExecutionError, \ + DefaultTemplateDataProvider, get_template_data +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +try: + from jinja2 import Template + HAS_JINJA2 = True +except ImportError: + HAS_JINJA2 = False + + +class DefaultJinja2DataProvider(DefaultTemplateDataProvider): + """ Template data provider for Jinja2 templates. Jinja2 and + Genshi currently differ over the value of the ``path`` variable, + which is why this is necessary. """ + + def get_template_data(self, entry, metadata, template): + rv = DefaultTemplateDataProvider.get_template_data(self, entry, + metadata, template) + rv['path'] = rv['name'] + return rv + + +class CfgJinja2Generator(CfgGenerator): + """ The CfgJinja2Generator allows you to use the `Jinja2 + <http://jinja.pocoo.org/>`_ templating system to generate + :ref:`server-plugins-generators-cfg` files. """ + + #: Handle .jinja2 files + __extensions__ = ['jinja2'] + + #: Low priority to avoid matching host- or group-specific + #: .crypt.jinja2 files + __priority__ = 50 + + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) + if not HAS_JINJA2: + raise PluginExecutionError("Jinja2 is not available") + __init__.__doc__ = CfgGenerator.__init__.__doc__ + + def get_data(self, entry, metadata): + template = Template(self.data.decode(Bcfg2.Options.setup.encoding)) + return template.render( + get_template_data(entry, metadata, self.name, + default=DefaultJinja2DataProvider())) + get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index e9698f526..8cc3f7b21 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -34,7 +34,6 @@ class CfgPrivateKeyCreator(XMLCfgCreator): pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) self.cmd = Executor() - __init__.__doc__ = XMLCfgCreator.__init__.__doc__ def _gen_keypair(self, metadata, spec=None): """ Generate a keypair according to the given client medata diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index d2b982349..5dc3d98eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -872,8 +872,7 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, """ The Cfg plugin provides a repository to describe configuration file contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client - machines. - """ + machines. """ __author__ = 'bcfg-dev@mcs.anl.gov' es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 3d3ef8f8c..b30a9acea 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -31,4 +31,4 @@ class Decisions(Bcfg2.Server.Plugin.Plugin, self.blacklist = DecisionFile(os.path.join(self.data, "blacklist.xml")) def GetDecisions(self, metadata, mode): - return getattr(self, mode).get_decision(metadata) + return getattr(self, mode).get_decisions(metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 78f86f28e..1d15656af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -674,6 +674,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if attribs is None: attribs = dict() if self._use_db: + if attribs: + msg = "Metadata does not support setting client attributes " +\ + "with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: client = MetadataClientModel.objects.get(hostname=client_name) except MetadataClientModel.DoesNotExist: @@ -681,7 +686,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = MetadataClientModel(hostname=client_name) # pylint: enable=E1102 client.save() - self.clients = self.list_clients() + self.update_client_list() return client else: try: @@ -734,7 +739,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, attribs, alias=True) def list_clients(self): - """ List all clients in client database """ + """ List all clients in client database. + + Making ``self.clients`` a property and reading the client list + dynamically from the database on every call to + ``self.clients`` can result in very high rates of database + reads, so we cache the ``list_clients()`` results to reduce + the database load. When the database is in use, the client + list is reread periodically with + :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """ if self._use_db: return set([c.hostname for c in MetadataClientModel.objects.all()]) else: @@ -785,13 +798,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.warning(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) client.delete() - self.clients = self.list_clients() + self.update_client_list() else: return self._remove_xdata(self.clients_xml, "Client", client_name) def _handle_clients_xml_event(self, _): # pylint: disable=R0912 """ handle all events for clients.xml and files xincluded from clients.xml """ + # disable metadata builds during parsing. this prevents + # clients from getting bogus metadata during the brief time it + # takes to rebuild the clients.xml data + self.states['clients.xml'] = False + xdata = self.clients_xml.xdata self.clients = [] self.clientgroups = {} @@ -853,9 +871,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.clientgroups[clname].append(profile) except KeyError: self.clientgroups[clname] = [profile] + self.update_client_list() + self.cache.expire() self.states['clients.xml'] = True - if self._use_db: - self.clients = self.list_clients() def _get_condition(self, element): """ Return a predicate that returns True if a client meets @@ -883,7 +901,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ + # disable metadata builds during parsing. this prevents + # clients from getting bogus metadata during the brief time it + # takes to rebuild the groups.xml data + self.states['groups.xml'] = False + self.groups = {} + self.group_membership = dict() + self.negated_groups = dict() + self.ordered_groups = [] # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of @@ -908,10 +934,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if grp.get('default', 'false') == 'true': self.default = grp.get('name') - self.group_membership = dict() - self.negated_groups = dict() - self.ordered_groups = [] - # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is # infinitely recursive, so "all" elements really means _all_ @@ -944,6 +966,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership.setdefault(gname, []) self.group_membership[gname].append( self._aggregate_conditions(conditions)) + self.cache.expire() self.states['groups.xml'] = True def HandleEvent(self, event): @@ -1447,6 +1470,32 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return True # pylint: enable=R0911,R0912 + def update_client_list(self): + """ Re-read the client list from the database (if the database is in + use) """ + if self._use_db: + self.logger.debug("Metadata: Re-reading client list from database") + old = set(self.clients) + self.clients = self.list_clients() + + # we could do this with set.symmetric_difference(), but we + # want detailed numbers of added/removed clients for + # logging + new = set(self.clients) + added = new - old + removed = old - new + self.logger.debug("Metadata: Added %s clients: %s" % + (len(added), added)) + self.logger.debug("Metadata: Removed %s clients: %s" % + (len(removed), removed)) + + for client in added.union(removed): + self.cache.expire(client) + + def start_client_run(self, metadata): + """ Hook to reread client list if the database is in use """ + self.update_client_list() + def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index ba7baab11..c5fb46c97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin try: import json -except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): import simplejson as json PROBECODE = """#!/bin/sh diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index dba56eed2..3d5c68e3f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -69,12 +69,11 @@ class AptSource(Source): else: return ["%sPackages.gz" % self.rawurl] - def read_files(self): + def read_files(self): # pylint: disable=R0912 bdeps = dict() + brecs = dict() bprov = dict() - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') + self.essentialpkgs = set() for fname in self.files: if not self.rawurl: barch = [x @@ -86,6 +85,7 @@ class AptSource(Source): barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() + brecs[barch] = dict() bprov[barch] = dict() try: reader = gzip.GzipFile(fname) @@ -100,9 +100,10 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: self.essentialpkgs.add(pkgname) - elif words[0] in depfnames: + elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): if '|' in dep: @@ -113,17 +114,24 @@ class AptSource(Source): barch, vindex) vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) + + if words[0] == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) + if words[0] == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': for pkg in words[1].split(','): dname = pkg.rstrip().strip() if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 8b20df58a..004e27874 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -289,7 +289,7 @@ class Collection(list, Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package): + def get_deps(self, package, recs=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -299,9 +299,14 @@ class Collection(list, Debuggable): :type package: string :returns: list of strings, but see :ref:`pkg-objects` """ + recommended = None + if recs and package in recs: + recommended = recs[package] + for source in self: if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) + return source.get_deps(self.metadata, package, recommended) + return [] def get_essential(self): @@ -465,7 +470,8 @@ class Collection(list, Debuggable): return list(complete.difference(initial)) @track_statistics() - def complete(self, packagelist): # pylint: disable=R0912,R0914 + def complete(self, packagelist, # pylint: disable=R0912,R0914 + recommended=None): """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the @@ -529,7 +535,7 @@ class Collection(list, Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current) + deps = self.get_deps(current, recommended) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py new file mode 100644 index 000000000..e393cabfe --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -0,0 +1,86 @@ +""" pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """ + +import lzma +import tarfile + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Source import Source + + +class PkgngCollection(Collection): + """ Handle collections of pkgng sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`, + overrides nothing, and defers all operations to :class:`PacSource` + """ + + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + # we define an __init__ that just calls the parent __init__, + # so that we can set the docstring on __init__ to something + # different from the parent __init__ -- namely, the parent + # __init__ docstring, minus everything after ``.. -----``, + # which we use to delineate the actual docs from the + # .. autoattribute hacks we have to do to get private + # attributes included in sphinx 1.0 """ + Collection.__init__(self, metadata, sources, cachepath, basepath, + debug=debug) + __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + + +class PkgngSource(Source): + """ Handle pkgng sources """ + + #: PkgngSource sets the ``type`` on Package entries to "pkgng" + ptype = 'pkgng' + + @property + def urls(self): + """ A list of URLs to the base metadata file for each + repository described by this source. """ + if not self.rawurl: + rv = [] + for part in self.components: + for arch in self.arches: + rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" % + (self.url, self.version, arch, part)) + return rv + else: + return ["%s/packagesite.txz" % self.rawurl] + + def read_files(self): + bdeps = dict() + for fname in self.files: + if not self.rawurl: + abi = [x + for x in fname.split('@') + if x.startswith('freebsd:')][0][8:] + barch = ':'.join(abi.split(':')[1:]) + else: + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + barch = self.arches[0] + if barch not in bdeps: + bdeps[barch] = dict() + try: + tar = tarfile.open(fileobj=lzma.LZMAFile(fname)) + reader = tar.extractfile('packagesite.yaml') + except: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') + pkg = json.loads(line) + pkgname = pkg['name'] + self.pkgnames.add(pkgname) + if 'deps' in pkg: + bdeps[barch][pkgname] = pkg['deps'].keys() + self.process_files(bdeps, dict()) + read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 4b6130f72..24db2963d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -246,6 +246,10 @@ class Source(Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + #: A dict of ``<package name>`` -> ``<list of recommended + #: symbols>``. This will not necessarily be populated. + self.recommends = dict() + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) @@ -310,7 +314,7 @@ class Source(Debuggable): # pylint: disable=R0902 :raises: cPickle.UnpicklingError - If the saved data is corrupt """ data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, - self.essentialpkgs) = cPickle.load(data) + self.essentialpkgs, self.recommends) = cPickle.load(data) def save_state(self): """ Save state to :attr:`cachefile`. If caching and @@ -318,7 +322,7 @@ class Source(Debuggable): # pylint: disable=R0902 does not need to be implemented. """ cache = open(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides, - self.essentialpkgs), cache, 2) + self.essentialpkgs, self.recommends), cache, 2) cache.close() @track_statistics() @@ -513,13 +517,14 @@ class Source(Debuggable): # pylint: disable=R0902 as its final step.""" pass - def process_files(self, dependencies, provides): + def process_files(self, dependencies, # pylint: disable=R0912,W0102 + provides, recommends=dict()): """ Given dicts of depends and provides generated by :func:`read_files`, this generates :attr:`deps` and :attr:`provides` and calls :func:`save_state` to save the cached data to disk. - Both arguments are dicts of dicts of lists. Keys are the + All arguments are dicts of dicts of lists. Keys are the arches of packages contained in this source; values are dicts whose keys are package names and values are lists of either dependencies for each package the symbols provided by each @@ -531,14 +536,20 @@ class Source(Debuggable): # pylint: disable=R0902 :param provides: A dict of symbols provided by packages in this repository. :type provides: dict; see above. + :param recommends: A dict of recommended dependencies + found for this source. + :type recommends: dict; see above. """ self.deps['global'] = dict() + self.recommends['global'] = dict() self.provides['global'] = dict() for barch in dependencies: self.deps[barch] = dict() + self.recommends[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() + rset = set() for barch in dependencies: if pkgname not in dependencies[barch]: dependencies[barch][pkgname] = [] @@ -548,6 +559,18 @@ class Source(Debuggable): # pylint: disable=R0902 else: for barch in dependencies: self.deps[barch][pkgname] = dependencies[barch][pkgname] + + for barch in recommends: + if pkgname not in recommends[barch]: + recommends[barch][pkgname] = [] + rset.add(tuple(recommends[barch][pkgname])) + if len(rset) == 1: + self.recommends['global'][pkgname] = rset.pop() + else: + for barch in recommends: + self.recommends[barch][pkgname] = \ + recommends[barch][pkgname] + provided = set() for bprovided in list(provides.values()): provided.update(set(bprovided)) @@ -655,17 +678,24 @@ class Source(Debuggable): # pylint: disable=R0902 """ return ['global'] + [a for a in self.arches if a in metadata.groups] - def get_deps(self, metadata, package): + def get_deps(self, metadata, package, recommended=None): """ Get a list of the dependencies of the given package. :param package: The name of the symbol :type package: string :returns: list of strings """ + recs = [] + if ((recommended is None and self.recommended) or + (recommended and recommended.lower() == 'true')): + for arch in self.get_arches(metadata): + if package in self.recommends[arch]: + recs.extend(self.recommends[arch][package]) + for arch in self.get_arches(metadata): if package in self.deps[arch]: - return self.deps[arch][package] - return [] + recs.extend(self.deps[arch][package]) + return recs def get_provides(self, metadata, package): """ Get a list of all symbols provided by the given package. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index b98d3f419..f26ded4c5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -63,6 +63,7 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from lockfile import FileLock from Bcfg2.Utils import Executor +from distutils.spawn import find_executable # pylint: disable=E0611 # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -89,7 +90,9 @@ try: import yum try: import json - except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 + except (ImportError, AttributeError): import simplejson as json HAS_YUM = True except ImportError: @@ -340,25 +343,21 @@ class YumCollection(Collection): @property def helper(self): - """ The full path to :file:`bcfg2-yum-helper`. First, we - check in the config file to see if it has been explicitly - specified; next we see if it's in $PATH (which we do by making - a call to it; I wish there was a way to do this without - forking, but apparently not); finally we check in /usr/sbin, - the default location. """ + """The full path to :file:`bcfg2-yum-helper`. First, we check in the + config file to see if it has been explicitly specified; next + we see if it's in $PATH; finally we default to /usr/sbin, the + default location. """ + # pylint: disable=W0212 if not self._helper: - # pylint: disable=W0212 self.__class__._helper = Bcfg2.Options.setup.yum_helper if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH - try: - self.debug_log("Checking for bcfg2-yum-helper in $PATH") - self.cmd.run(['bcfg2-yum-helper']) - self.__class__._helper = 'bcfg2-yum-helper' - except OSError: + self.debug_log("Checking for bcfg2-yum-helper in $PATH") + self.__class__._helper = find_executable('bcfg2-yum-helper') + if not self.__class__._helper: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" - # pylint: enable=W0212 - return self._helper + return self.__class__._helper + # pylint: enable=W0212 @property def use_yum(self): @@ -417,6 +416,25 @@ class YumCollection(Collection): yumconf.write(open(self.cfgfile, 'w')) + def get_arch(self): + """ If 'arch' for each source is the same, return that arch, otherwise + None. + + This helps bcfg2-yum-helper when the client arch is + incompatible with the bcfg2 server's arch. + + In case multiple arches are found, punt back to the default behavior. + """ + arches = set() + for source in self: + for url_map in source.url_map: + if url_map['arch'] in self.metadata.groups: + arches.add(url_map['arch']) + if len(arches) == 1: + return arches.pop() + else: + return None + def get_config(self, raw=False): # pylint: disable=W0221 """ Get the yum configuration for this collection. @@ -839,7 +857,7 @@ class YumCollection(Collection): return new @track_statistics() - def complete(self, packagelist): + def complete(self, packagelist, recommended=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -857,7 +875,7 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist) + return Collection.complete(self, packagelist, recommended) lock = FileLock(os.path.join(self.cachefile, "lock")) slept = 0 @@ -872,10 +890,12 @@ class YumCollection(Collection): if packagelist: try: - result = self.call_helper( - "complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) + helper_dict = dict(packages=list(packagelist), + groups=list(self.get_relevant_groups())) + arch = self.get_arch() + if arch is not None: + helper_dict['arch'] = arch + result = self.call_helper("complete", helper_dict) except ValueError: # error reported by call_helper() return set(), packagelist diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 49f64bdf3..d11ac60fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -101,7 +101,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, cf=("packages", "backends"), dest="packages_backends", help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, - action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), + action=PackagesBackendAction, + default=['Yum', 'Apt', 'Pac', 'Pkgng']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", @@ -319,8 +320,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, structures.append(indep) @track_statistics() - def _build_packages(self, metadata, independent, structures, - collection=None): + def _build_packages(self, metadata, independent, # pylint: disable=R0914 + structures, collection=None): """ Perform dependency resolution and build the complete list of packages that need to be included in the specification by :func:`validate_structures`, based on the initial list of @@ -357,10 +358,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, initial = set() to_remove = [] groups = [] + recommended = dict() + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) + + if pkg.get("recommended"): + recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -399,7 +405,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: - pcache[pkey] = collection.complete(base) + pcache[pkey] = collection.complete(base, recommended) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 9f2375fcd..21d50ace6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -10,7 +10,7 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import unicode # pylint: disable=W0622 +from Bcfg2.Compat import unicode, any # pylint: disable=W0622 import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics @@ -51,8 +51,10 @@ def load_django_models(): try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -431,7 +433,13 @@ class Probes(Bcfg2.Server.Plugin.Probing, options = [ Bcfg2.Options.BooleanOption( cf=('probes', 'use_database'), dest="probes_db", - help="Use database capabilities of the Probes plugin")] + help="Use database capabilities of the Probes plugin"), + Bcfg2.Options.Option( + cf=('probes', 'allowed_groups'), dest="probes_allowed_groups", + help="Whitespace-separated list of group name regexps to which " + "probes can assign a client", + default=[re.compile('.*')], + type=Bcfg2.Options.Types.anchored_regex_list)] options_parsed_hook = staticmethod(load_django_models) def __init__(self, core): @@ -480,7 +488,13 @@ class Probes(Bcfg2.Server.Plugin.Probing, for line in dlines[:]: match = self.groupline_re.match(line) if match: - groups.append(match.group("groupname")) + newgroup = match.group("groupname") + if self._group_allowed(newgroup): + groups.append(newgroup) + else: + self.logger.warning( + "Disallowed group assignment %s from %s" % + (newgroup, client.hostname)) dlines.remove(line) return (groups, ProbeData("\n".join(dlines))) @@ -489,3 +503,10 @@ class Probes(Bcfg2.Server.Plugin.Probing, def get_additional_data(self, metadata): return self.probestore.get_data(metadata.hostname) + + def _group_allowed(self, group): + """ Determine if the named group can be set as a probe group + by checking the regexes listed in the [probes] groups_allowed + setting """ + return any(r.match(group) + for r in Bcfg2.Options.setup.probes_allowed_groups) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 87cee7029..28400f6d2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -13,8 +13,10 @@ from Bcfg2.Server.Plugin import PluginExecutionError try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -161,7 +163,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) PropertyFile.__init__(self, name) - __init__.__doc__ = Bcfg2.Server.Plugin.StructFile.__init__.__doc__ def _write(self): open(self.name, "wb").write( @@ -169,7 +170,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): xml_declaration=False, pretty_print=True).decode('UTF-8')) return True - _write.__doc__ = PropertyFile._write.__doc__ def validate_data(self): """ ensure that the data in this object validates against the @@ -192,7 +192,6 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): self.name) else: return True - validate_data.__doc__ = PropertyFile.validate_data.__doc__ def get_additional_data(self, metadata): if Bcfg2.Options.setup.automatch: diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 8b8ada852..282de8247 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -54,7 +54,7 @@ class Reporting(Statistics, Threaded, PullSource): self.logger.error(msg) raise PluginInitError(msg) - def start_threads(self): + # This must be loaded here for bcfg2-admin try: self.transport = Bcfg2.Options.setup.reporting_transport() except TransportError: @@ -63,6 +63,10 @@ class Reporting(Statistics, Threaded, PullSource): if self.debug_flag: self.transport.set_debug(self.debug_flag) + def start_threads(self): + """Nothing to do here""" + pass + def set_debug(self, debug): rv = Statistics.set_debug(self, debug) if self.transport is not None: diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index b2a16e52e..b752650f0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -20,8 +20,8 @@ class Svn(Bcfg2.Server.Plugin.Version): Bcfg2.Options.Option( cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", type=lambda v: v.replace("-", "_"), - choices=dir(pysvn.wc_conflict_choice), - default=pysvn.wc_conflict_choice.postpone, + choices=dir(pysvn.wc_conflict_choice), # pylint: disable=E1101 + default=pysvn.wc_conflict_choice.postpone, # pylint: disable=E1101 help="SVN conflict resolution method"), Bcfg2.Options.Option( cf=("svn", "user"), dest="svn_user", help="SVN username"), diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 5e6846a44..6ad5b5635 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -72,7 +72,7 @@ class SSLServer(SocketServer.TCPServer, object): def __init__(self, listen_all, server_address, RequestHandlerClass, keyfile=None, certfile=None, reqCert=False, ca=None, - timeout=None, protocol='xmlrpc/ssl'): + timeout=None, protocol='xmlrpc/tlsv1'): """ :param listen_all: Listen on all interfaces :type listen_all: bool @@ -333,7 +333,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, """ Component XMLRPCServer. """ def __init__(self, listen_all, server_address, RequestHandlerClass=None, - keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', + keyfile=None, certfile=None, ca=None, protocol='xmlrpc/tlsv1', timeout=10, logRequests=False, register=True, allow_none=True, encoding=None): """ diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 35d4cfa0a..196d77273 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.3" +__version__ = "1.4.0pre1" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 |