diff options
Diffstat (limited to 'src/lib/Bcfg2')
93 files changed, 2949 insertions, 880 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index d30708e83..ad718749e 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,6 +1,7 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ +import copy import time import fnmatch import logging @@ -97,8 +98,8 @@ class Frame(object): self.logger.warning(deprecated) experimental = [tool.name for tool in self.tools if tool.experimental] if experimental: - self.logger.warning("Loaded experimental tool drivers:") - self.logger.warning(experimental) + self.logger.info("Loaded experimental tool drivers:") + self.logger.info(experimental) # find entries not handled by any tools self.unhandled = [entry for struct in config @@ -207,7 +208,15 @@ class Frame(object): # take care of important entries first if not self.dryrun: - for parent in self.config.findall(".//Path/.."): + parent_map = dict((c, p) + for p in self.config.getiterator() + for c in p) + for cfile in self.config.findall(".//Path"): + if (cfile.get('name') not in self.__important__ or + cfile.get('type') != 'file' or + cfile not in self.whitelist): + continue + parent = parent_map[cfile] if ((parent.tag == "Bundle" and ((self.setup['bundle'] and parent.get("name") not in self.setup['bundle']) or @@ -216,15 +225,9 @@ class Frame(object): (parent.tag == "Independent" and (self.setup['bundle'] or self.setup['skipindep']))): continue - for cfile in parent.findall("./Path"): - if (cfile.get('name') not in self.__important__ or - cfile.get('type') != 'file' or - cfile not in self.whitelist): - continue - tools = [t for t in self.tools - if t.handlesEntry(cfile) and t.canVerify(cfile)] - if not tools: - continue + tools = [t for t in self.tools + if t.handlesEntry(cfile) and t.canVerify(cfile)] + if tools: if (self.setup['interactive'] and not self.promptFilter("Install %s: %s? (y/N):", [cfile])): self.whitelist.remove(cfile) @@ -326,11 +329,13 @@ class Frame(object): if bundle.tag != 'Bundle': continue bmodified = len([item for item in bundle - if item in self.whitelist]) + if item in self.whitelist or + item in self.modified]) actions = [a for a in bundle.findall('./Action') if (a.get('timing') != 'post' and (bmodified or a.get('when') == 'always'))] - # now we process all "always actions" + # now we process all "pre" and "both" actions that are either + # always or the bundle has been modified if self.setup['interactive']: self.promptFilter(iprompt, actions) self.DispatchInstallCalls(actions) @@ -520,7 +525,7 @@ class Frame(object): container = Bcfg2.Client.XML.SubElement(stats, ename) for item in data: item.set('qtext', '') - container.append(item) + container.append(copy.deepcopy(item)) item.text = None timeinfo = Bcfg2.Client.XML.Element("OpStamps") diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index da4412b1d..0166e4c00 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -32,10 +32,17 @@ class Action(Bcfg2.Client.Tools.Tool): def RunAction(self, entry): """This method handles command execution and status return.""" + shell = False + shell_string = '' + if entry.get('shell', 'false') == 'true': + shell = True + shell_string = '(in shell) ' + if not self.setup['dryrun']: if self.setup['interactive']: - prompt = ('Run Action %s, %s: (y/N): ' % - (entry.get('name'), entry.get('command'))) + prompt = ('Run Action %s%s, %s: (y/N): ' % + (shell_string, entry.get('name'), + entry.get('command'))) # flush input buffer while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: @@ -48,8 +55,9 @@ class Action(Bcfg2.Client.Tools.Tool): self.logger.debug("Action: Deferring execution of %s due " "to build mode" % entry.get('command')) return False - self.logger.debug("Running Action %s" % (entry.get('name'))) - rv = self.cmd.run(entry.get('command')) + self.logger.debug("Running Action %s %s" % + (shell_string, entry.get('name'))) + rv = self.cmd.run(entry.get('command'), shell=shell) self.logger.debug("Action: %s got return code %s" % (entry.get('command'), rv.retval)) entry.set('rc', str(rv.retval)) diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index 1fce5515b..4833f3f68 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -19,26 +19,22 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): def get_svc_command(self, service, action): return "/sbin/service %s %s" % (service.get('name'), action) - def VerifyService(self, entry, _): - """Verify Service status for entry.""" - entry.set('target_status', entry.get('status')) - if entry.get('status') == 'ignore': - return True - + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" rv = self.cmd.run("/sbin/chkconfig --list %s " % entry.get('name')) if rv.success: srvdata = rv.stdout.splitlines()[0].split() else: # service not installed - entry.set('current_status', 'off') + entry.set('current_bootstatus', 'service not installed') return False if len(srvdata) == 2: # This is an xinetd service - if entry.get('status') == srvdata[1]: + if bootstatus == srvdata[1]: return True else: - entry.set('current_status', srvdata[1]) + entry.set('current_bootstatus', srvdata[1]) return False try: @@ -47,46 +43,81 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): except IndexError: onlevels = [] - pstatus = self.check_service(entry) - if entry.get('status') == 'on': - status = (len(onlevels) > 0 and pstatus) + if bootstatus == 'on': + current_bootstatus = (len(onlevels) > 0) else: - status = (len(onlevels) == 0 and not pstatus) + current_bootstatus = (len(onlevels) == 0) + return current_bootstatus + + def VerifyService(self, entry, _): + """Verify Service status for entry.""" + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: + return True + current_bootstatus = self.verify_bootstatus(entry, bootstatus) - if not status: + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) if entry.get('status') == 'on': - entry.set('current_status', 'off') - else: - entry.set('current_status', 'on') - return status + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: + entry.set('current_status', 'on') + else: + entry.set('current_status', 'off') + + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" - rcmd = "/sbin/chkconfig %s %s" - self.cmd.run("/sbin/chkconfig --add %s" % (entry.attrib['name'])) + self.cmd.run("/sbin/chkconfig --add %s" % (entry.get('name'))) self.logger.info("Installing Service %s" % (entry.get('name'))) - rv = True - if (entry.get('status') == 'off' or - self.setup["servicemode"] == "build"): - rv &= self.cmd.run((rcmd + " --level 0123456") % - (entry.get('name'), - entry.get('status'))).success - if entry.get("current_status") == "on" and \ - self.setup["servicemode"] != "disabled": - rv &= self.stop_service(entry).success + bootstatus = self.get_bootstatus(entry) + if bootstatus is not None: + if bootstatus == 'on': + # make sure service is enabled on boot + bootcmd = '/sbin/chkconfig %s %s' % \ + (entry.get('name'), bootstatus) + elif bootstatus == 'off': + # make sure service is disabled on boot + bootcmd = '/sbin/chkconfig %s %s' % (entry.get('name'), + bootstatus) + bootcmdrv = self.cmd.run(bootcmd).success + if self.setup['servicemode'] == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv + buildmode = self.setup['servicemode'] == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv else: - rv &= self.cmd.run(rcmd % (entry.get('name'), - entry.get('status'))).success - if entry.get("current_status") == "off" and \ - self.setup["servicemode"] != "disabled": - rv &= self.start_service(entry).success - return rv + # when bootstatus is 'None', status == 'ignore' + return True def FindExtra(self): """Locate extra chkconfig Services.""" allsrv = [line.split()[0] - for line in self.cmd.run("/sbin/chkconfig", - "--list").stdout.splitlines() + for line in + self.cmd.run("/sbin/chkconfig --list").stdout.splitlines() if ":on" in line] self.logger.debug('Found active services:') self.logger.debug(allsrv) diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index d916b1662..b544e44d4 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -18,13 +18,11 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): svcre = \ re.compile(r'/etc/.*/(?P<action>[SK])(?P<sequence>\d+)(?P<name>\S+)') - # implement entry (Verify|Install) ops - def VerifyService(self, entry, _): - """Verify Service status for entry.""" - - if entry.get('status') == 'ignore': - return True + def get_svc_command(self, service, action): + return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" rawfiles = glob.glob("/etc/rc*.d/[SK]*%s" % (entry.get('name'))) files = [] @@ -54,9 +52,9 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): continue if match.group('name') == entry.get('name'): files.append(filename) - if entry.get('status') == 'off': + if bootstatus == 'off': if files: - entry.set('current_status', 'on') + entry.set('current_bootstatus', 'on') return False else: return True @@ -72,12 +70,47 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): return False return True else: - entry.set('current_status', 'off') + entry.set('current_bootstatus', 'off') return False + def VerifyService(self, entry, _): + """Verify Service status for entry.""" + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: + return True + current_bootstatus = self.verify_bootstatus(entry, bootstatus) + + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: + entry.set('current_status', 'on') + else: + entry.set('current_status', 'off') + + return current_bootstatus and current_svcstatus + def InstallService(self, entry): - """Install Service for entry.""" + """Install Service entry.""" self.logger.info("Installing Service %s" % (entry.get('name'))) + bootstatus = self.get_bootstatus(entry) + + # check if init script exists try: os.stat('/etc/init.d/%s' % entry.get('name')) except OSError: @@ -85,20 +118,41 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): entry.get('name')) return False - if entry.get('status') == 'off': - self.cmd.run("/usr/sbin/invoke-rc.d %s stop" % (entry.get('name'))) - return self.cmd.run("/usr/sbin/update-rc.d -f %s remove" % - entry.get('name')).success + if bootstatus is not None: + seqcmdrv = True + if bootstatus == 'on': + # make sure service is enabled on boot + bootcmd = '/usr/sbin/update-rc.d %s defaults' % \ + entry.get('name') + if entry.get('sequence'): + seqcmd = '/usr/sbin/update-rc.d -f %s remove' % \ + entry.get('name') + seqcmdrv = self.cmd.run(seqcmd) + start_sequence = int(entry.get('sequence')) + kill_sequence = 100 - start_sequence + bootcmd = '%s %d %d' % (bootcmd, start_sequence, + kill_sequence) + elif bootstatus == 'off': + # make sure service is disabled on boot + bootcmd = '/usr/sbin/update-rc.d -f %s remove' % \ + entry.get('name') + bootcmdrv = self.cmd.run(bootcmd) + if self.setup['servicemode'] == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv and seqcmdrv + buildmode = self.setup['servicemode'] == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv and seqcmdrv else: - command = "/usr/sbin/update-rc.d %s defaults" % (entry.get('name')) - if entry.get('sequence'): - if not self.cmd.run("/usr/sbin/update-rc.d -f %s remove" % - entry.get('name')).success: - return False - start_sequence = int(entry.get('sequence')) - kill_sequence = 100 - start_sequence - command = "%s %d %d" % (command, start_sequence, kill_sequence) - return self.cmd.run(command).success + # when bootstatus is 'None', status == 'ignore' + return True def FindExtra(self): """Find Extra Debian Service entries.""" @@ -116,6 +170,3 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): # Extra service removal is nonsensical # Extra services need to be reflected in the config return - - def get_svc_command(self, service, action): - return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py new file mode 100644 index 000000000..8506f4bc7 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -0,0 +1,296 @@ +""" Augeas driver """ + +import sys +import Bcfg2.Client.XML +from augeas import Augeas +from Bcfg2.Client.Tools.POSIX.base import POSIXTool +from Bcfg2.Client.Tools.POSIX.File import POSIXFile + + +class AugeasCommand(object): + """ Base class for all Augeas command objects """ + + def __init__(self, command, augeas_obj, logger): + self._augeas = augeas_obj + self.command = command + self.entry = self.command.getparent() + self.logger = logger + + def get_path(self, attr="path"): + """ Get a fully qualified path from the name of the parent entry and + the path given in this command tag. + + @param attr: The attribute to get the relative path from + @type attr: string + @returns: string - the fully qualified Augeas path + + """ + return "/files/%s/%s" % (self.entry.get("name").strip("/"), + self.command.get(attr).lstrip("/")) + + def _exists(self, path): + """ Return True if a path exists in Augeas, False otherwise. + + Note that a False return can mean many things: A file that + doesn't exist, a node within the file that doesn't exist, no + lens to parse the file, etc. """ + return len(self._augeas.match(path)) > 1 + + def _verify_exists(self, path=None): + """ Verify that the given path exists, with friendly debug + logging. + + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path exists + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying that '%s' exists" % path) + return self._exists(path) + + def _verify_not_exists(self, path=None): + """ Verify that the given path does not exist, with friendly + debug logging. + + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path does not exist. + (I.e., True if it does not exist, False if it does + exist.) + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying that '%s' does not exist" % path) + return not self._exists(path) + + def _verify_set(self, expected, path=None): + """ Verify that the given path is set to the given value, with + friendly debug logging. + + @param expected: The expected value of the node. + @param path: The path to verify existence of. Defaults to the + result of + :func:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand.getpath`. + @type path: string + @returns: bool - Whether or not the path matches the expected value. + + """ + if path is None: + path = self.get_path() + self.logger.debug("Augeas: Verifying '%s' == '%s'" % (path, expected)) + actual = self._augeas.get(path) + if actual == expected: + return True + else: + self.logger.debug("Augeas: '%s' failed verification: '%s' != '%s'" + % (path, actual, expected)) + return False + + def __str__(self): + return Bcfg2.Client.XML.tostring(self.command) + + def verify(self): + """ Verify that the command has been applied. """ + raise NotImplementedError + + def install(self): + """ Run the command. """ + raise NotImplementedError + + +class Remove(AugeasCommand): + """ Augeas ``rm`` command """ + def verify(self): + return self._verify_not_exists() + + def install(self): + self.logger.debug("Augeas: Removing %s" % self.get_path()) + return self._augeas.remove(self.get_path()) + + +class Move(AugeasCommand): + """ Augeas ``move`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.source = self.get_path("source") + self.dest = self.get_path("destination") + + def verify(self): + return (self._verify_not_exists(self.source), + self._verify_exists(self.dest)) + + def install(self): + self.logger.debug("Augeas: Moving %s to %s" % (self.source, self.dest)) + return self._augeas.move(self.source, self.dest) + + +class Set(AugeasCommand): + """ Augeas ``set`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.value = self.command.get("value") + + def verify(self): + return self._verify_set(self.value) + + def install(self): + self.logger.debug("Augeas: Setting %s to %s" % (self.get_path(), + self.value)) + return self._augeas.set(self.get_path(), self.value) + + +class Clear(Set): + """ Augeas ``clear`` command """ + def __init__(self, command, augeas_obj, logger): + Set.__init__(self, command, augeas_obj, logger) + self.value = None + + +class SetMulti(AugeasCommand): + """ Augeas ``setm`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.sub = self.command.get("sub") + self.value = self.command.get("value") + self.base = self.get_path("base") + + def verify(self): + return all(self._verify_set(self.value, + path="%s/%s" % (path, self.sub)) + for path in self._augeas.match(self.base)) + + def install(self): + return self._augeas.setm(self.base, self.sub, self.value) + + +class Insert(AugeasCommand): + """ Augeas ``ins`` command """ + def __init__(self, command, augeas_obj, logger): + AugeasCommand.__init__(self, command, augeas_obj, logger) + self.label = self.command.get("label") + self.where = self.command.get("where", "before") + self.before = self.where == "before" + + def verify(self): + return self._verify_exists("%s/../%s" % (self.get_path(), self.label)) + + def install(self): + self.logger.debug("Augeas: Inserting new %s %s %s" % + (self.label, self.where, self.get_path())) + return self._augeas.insert(self.get_path(), self.label, self.before) + + +class POSIXAugeas(POSIXTool): + """ Handle <Path type='augeas'...> entries. See + :ref:`client-tools-augeas`. """ + __req__ = ['name', 'mode', 'owner', 'group'] + + def __init__(self, logger, setup, config): + POSIXTool.__init__(self, logger, setup, config) + self._augeas = dict() + # file tool for setting initial values of files that don't + # exist + self.filetool = POSIXFile(logger, setup, config) + + def get_augeas(self, entry): + """ Get an augeas object for the given entry. """ + if entry.get("name") not in self._augeas: + aug = Augeas() + if entry.get("lens"): + self.logger.debug("Augeas: Adding %s to include path for %s" % + (entry.get("name"), entry.get("lens"))) + incl = "/augeas/load/%s/incl" % entry.get("lens") + ilen = len(aug.match(incl)) + if ilen == 0: + self.logger.error("Augeas: Lens %s does not exist" % + entry.get("lens")) + else: + aug.set("%s[%s]" % (incl, ilen + 1), entry.get("name")) + aug.load() + self._augeas[entry.get("name")] = aug + return self._augeas[entry.get("name")] + + def fully_specified(self, entry): + return len(entry.getchildren()) != 0 + + def get_commands(self, entry): + """ Get a list of commands to verify or install. + + @param entry: The entry to get commands from. + @type entry: lxml.etree._Element + @param unverified: Only get commands that failed verification. + @type unverified: bool + @returns: list of + :class:`Bcfg2.Client.Tools.POSIX.Augeas.AugeasCommand` + objects representing the commands. + """ + rv = [] + for cmd in entry.iterchildren(): + if cmd.tag == "Initial": + continue + if cmd.tag in globals(): + rv.append(globals()[cmd.tag](cmd, self.get_augeas(entry), + self.logger)) + else: + err = "Augeas: Unknown command %s in %s" % (cmd.tag, + entry.get("name")) + self.logger.error(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) + return rv + + def verify(self, entry, modlist): + rv = True + for cmd in self.get_commands(entry): + try: + if not cmd.verify(): + err = "Augeas: Command has not been applied to %s: %s" % \ + (entry.get("name"), cmd) + self.logger.debug(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), + err])) + rv = False + cmd.command.set("verified", "false") + else: + cmd.command.set("verified", "true") + except: # pylint: disable=W0702 + err = "Augeas: Unexpected error verifying %s: %s: %s" % \ + (entry.get("name"), cmd, sys.exc_info()[1]) + self.logger.error(err) + entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) + rv = False + cmd.command.set("verified", "false") + return POSIXTool.verify(self, entry, modlist) and rv + + def install(self, entry): + rv = True + if entry.get("current_exists", "true") == "false": + initial = entry.find("Initial") + if initial is not None: + self.logger.debug("Augeas: Setting initial data for %s" % + entry.get("name")) + file_entry = Bcfg2.Client.XML.Element("Path", + **dict(entry.attrib)) + file_entry.text = initial.text + self.filetool.install(file_entry) + # re-parse the file + self.get_augeas(entry).load() + for cmd in self.get_commands(entry): + try: + cmd.install() + except: # pylint: disable=W0702 + self.logger.error( + "Failure running Augeas command on %s: %s: %s" % + (entry.get("name"), cmd, sys.exc_info()[1])) + rv = False + try: + self.get_augeas(entry).save() + except: # pylint: disable=W0702 + self.logger.error("Failure saving Augeas changes to %s: %s" % + (entry.get("name"), sys.exc_info()[1])) + rv = False + return POSIXTool.install(self, entry) and rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 168c35c98..b1bde1057 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -53,6 +53,10 @@ class POSIXFile(POSIXTool): def verify(self, entry, modlist): ondisk = self._exists(entry) tempdata, is_binary = self._get_data(entry) + if isinstance(tempdata, str) and str != unicode: + tempdatasize = len(tempdata) + else: + tempdatasize = len(tempdata.encode(self.setup['encoding'])) different = False content = None @@ -61,7 +65,7 @@ class POSIXFile(POSIXTool): # they're clearly different different = True content = "" - elif len(tempdata) != ondisk[stat.ST_SIZE]: + elif tempdatasize != ondisk[stat.ST_SIZE]: # next, see if the size of the target file is different # from the size of the desired content different = True @@ -72,6 +76,9 @@ class POSIXFile(POSIXTool): # for everything else try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), + encoding=self.setup['encoding']).read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) @@ -89,7 +96,7 @@ class POSIXFile(POSIXTool): def _write_tmpfile(self, entry): """ Write the file data to a temp file """ - filedata, _ = self._get_data(entry) + filedata = self._get_data(entry)[0] # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions # protections on that directory, and also to avoid issues with @@ -105,7 +112,11 @@ class POSIXFile(POSIXTool): (os.path.dirname(entry.get('name')), err)) return False try: - os.fdopen(newfd, 'w').write(filedata) + if isinstance(filedata, str) and str != unicode: + os.fdopen(newfd, 'w').write(filedata) + else: + os.fdopen(newfd, 'wb').write( + filedata.encode(self.setup['encoding'])) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " @@ -146,8 +157,8 @@ class POSIXFile(POSIXTool): return POSIXTool.install(self, entry) and rv - def _get_diffs(self, entry, interactive=False, sensitive=False, - is_binary=False, content=None): + def _get_diffs(self, entry, interactive=False, # pylint: disable=R0912 + sensitive=False, is_binary=False, content=None): """ generate the necessary diffs for entry """ if not interactive and sensitive: return @@ -163,6 +174,8 @@ class POSIXFile(POSIXTool): # prompts for -I and the reports try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), encoding='utf-8').read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 7708c4f72..8d64cf84d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -47,8 +47,11 @@ class POSIX(Bcfg2.Client.Tools.Tool): mname = submodule[1].rsplit('.', 1)[-1] if mname == 'base': continue - module = getattr(__import__(submodule[1]).Client.Tools.POSIX, - mname) + try: + module = getattr(__import__(submodule[1]).Client.Tools.POSIX, + mname) + except ImportError: + continue hdlr = getattr(module, "POSIX" + mname) if POSIXTool in hdlr.__mro__: # figure out what entry type this handler handles diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 16fe0acb5..3243bbf50 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -232,6 +232,11 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: defacl = None + if not acls: + self.logger.debug("POSIX: Removed ACLs from %s" % + entry.get("name")) + return True + for aclkey, perms in acls.items(): atype, scope, qualifier = aclkey if atype == "default": @@ -390,7 +395,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): acl_str.append("user") elif scope == posix1e.ACL_GROUP: acl_str.append("group") - acl_str.append(qualifier) + if qualifier is None: + acl_str.append('') + else: + acl_str.append(qualifier) acl_str.append(self._acl_perm2string(perms)) return ":".join(acl_str) @@ -525,7 +533,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if entry.get("secontext") == "__default__": try: wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + selinux.matchpathcon( + path, ondisk[stat.ST_MODE])[1].split(":")[2] except OSError: errors.append("%s has no default SELinux context" % entry.get("name")) @@ -686,7 +695,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ os.makedirs helpfully creates all parent directories for us, but it sets permissions according to umask, which is probably wrong. we need to find out which directories were - created and set permissions on those + created and try to set permissions on those (http://trac.mcs.anl.gov/projects/bcfg2/ticket/1125 and http://trac.mcs.anl.gov/projects/bcfg2/ticket/1134) """ created = [] @@ -706,22 +715,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): (path, err)) rv = False - # we need to make sure that we give +x to everyone who needs - # it. E.g., if the file that's been distributed is 0600, we - # can't make the parent directories 0600 also; that'd be - # pretty useless. They need to be 0700. + # set auto-created directories to mode 755 and use best effort for + # permissions. If you need something else, you should specify it in + # your config. tmpentry = copy.deepcopy(entry) - newmode = int(entry.get('mode'), 8) - for i in range(0, 3): - if newmode & (6 * pow(8, i)): - newmode |= 1 * pow(8, i) - tmpentry.set('mode', oct_mode(newmode)) + tmpentry.set('mode', '0755') for acl in tmpentry.findall('ACL'): acl.set('perms', oct_mode(self._norm_acl_perms(acl.get('perms')) | ACL_MAP['x'])) for cpath in created: - rv &= self._set_perms(tmpentry, path=cpath) + self._set_perms(tmpentry, path=cpath) return rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 8226392f9..6d18cd176 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -146,7 +146,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group")] + if entry.get("name") in g[3] and g[0] != entry.get("group") + and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ @@ -189,14 +190,18 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): else: for attr, idx in self.attr_mapping[entry.tag].items(): val = str(self.existing[entry.tag][entry.get("name")][idx]) - entry.set("current_%s" % attr, val) + entry.set("current_%s" % + attr, val.decode(self.setup['encoding'])) if attr in ["uid", "gid"]: if entry.get(attr) is None: # no uid/gid specified, so we let the tool # automatically determine one -- i.e., it always # verifies continue - if val != entry.get(attr): + entval = entry.get(attr) + if not isinstance(entval, str): + entval = entval.encode('utf-8') + if val != entval: errors.append("%s for %s %s is incorrect. Current %s is " "%s, but should be %s" % (attr.title(), entry.tag, entry.get("name"), @@ -249,7 +254,6 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('gid'): cmd.extend(['-g', entry.get('gid')]) elif entry.tag == 'POSIXUser': - cmd.append('-m') if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 4b78581f7..e0c913dcd 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -21,21 +21,38 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): '-s']).stdout.splitlines() if 'started' in line] + def get_default_svcs(self): + """Return a list of services in the 'default' runlevel.""" + return [line.split()[0] + for line in self.cmd.run(['/sbin/rc-update', + 'show']).stdout.splitlines() + if 'default' in line] + + def verify_bootstatus(self, entry, bootstatus): + """Verify bootstatus for entry.""" + # get a list of all started services + allsrv = self.get_default_svcs() + # set current_bootstatus attribute + if entry.get('name') in allsrv: + entry.set('current_bootstatus', 'on') + else: + entry.set('current_bootstatus', 'off') + if bootstatus == 'on': + return entry.get('name') in allsrv + else: + return entry.get('name') not in allsrv + def VerifyService(self, entry, _): """ Verify Service status for entry. Assumes we run in the "default" runlevel. """ - if entry.get('status') == 'ignore': + entry.set('target_status', entry.get('status')) # for reporting + bootstatus = self.get_bootstatus(entry) + if bootstatus is None: return True - - # get a list of all started services - allsrv = self.get_enabled_svcs() - - # check if service is enabled - result = self.cmd.run(["/sbin/rc-update", "show", "default"]).stdout - is_enabled = entry.get("name") in result + current_bootstatus = self.verify_bootstatus(entry, bootstatus) # check if init script exists try: @@ -45,39 +62,58 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): entry.get('name')) return False - # check if service is enabled - is_running = entry.get('name') in allsrv - - if entry.get('status') == 'on' and not (is_enabled and is_running): - entry.set('current_status', 'off') - return False - - elif entry.get('status') == 'off' and (is_enabled or is_running): + if entry.get('status') == 'ignore': + # 'ignore' should verify + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True + + if svcstatus: entry.set('current_status', 'on') - return False + else: + entry.set('current_status', 'off') - return True + return current_bootstatus and current_svcstatus def InstallService(self, entry): - """ - Install Service entry - - """ + """Install Service entry.""" self.logger.info('Installing Service %s' % entry.get('name')) - if entry.get('status') == 'on': - if entry.get('current_status') == 'off': - self.start_service(entry) - # make sure it's enabled - cmd = '/sbin/rc-update add %s default' - return self.cmd.run(cmd % entry.get('name')).success - elif entry.get('status') == 'off': - if entry.get('current_status') == 'on': - self.stop_service(entry) - # make sure it's disabled - cmd = '/sbin/rc-update del %s default' - return self.cmd.run(cmd % entry.get('name')).success - - return False + bootstatus = self.get_bootstatus(entry) + if bootstatus is not None: + if bootstatus == 'on': + # make sure service is enabled on boot + bootcmd = '/sbin/rc-update add %s default' + elif bootstatus == 'off': + # make sure service is disabled on boot + bootcmd = '/sbin/rc-update del %s default' + bootcmdrv = self.cmd.run(bootcmd % entry.get('name')).success + if self.setup['servicemode'] == 'disabled': + # 'disabled' means we don't attempt to modify running svcs + return bootcmdrv + buildmode = self.setup['servicemode'] == 'build' + if (entry.get('status') == 'on' and not buildmode) and \ + entry.get('current_status') == 'off': + svccmdrv = self.start_service(entry) + elif (entry.get('status') == 'off' or buildmode) and \ + entry.get('current_status') == 'on': + svccmdrv = self.stop_service(entry) + else: + svccmdrv = True # ignore status attribute + return bootcmdrv and svccmdrv + else: + # when bootstatus is 'None', status == 'ignore' + return True def FindExtra(self): """Locate extra rc-update services.""" diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 1ab867215..aca5dbbc7 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -1,14 +1,15 @@ """VCS support.""" # TODO: -# * git_write_index # * add svn support # * integrate properly with reports missing = [] +import errno import os import shutil import sys +import stat # python-dulwich git imports try: @@ -26,6 +27,38 @@ except ImportError: import Bcfg2.Client.Tools +def cleanup_mode(mode): + """Cleanup a mode value. + + This will return a mode that can be stored in a tree object. + + :param mode: Mode to clean up. + """ + if stat.S_ISLNK(mode): + return stat.S_IFLNK + elif stat.S_ISDIR(mode): + return stat.S_IFDIR + elif dulwich.index.S_ISGITLINK(mode): + return dulwich.index.S_IFGITLINK + ret = stat.S_IFREG | int('644', 8) + ret |= (mode & int('111', 8)) + return ret + + +def index_entry_from_stat(stat_val, hex_sha, flags, mode=None): + """Create a new index entry from a stat value. + + :param stat_val: POSIX stat_result instance + :param hex_sha: Hex sha of the object + :param flags: Index flags + """ + if mode is None: + mode = cleanup_mode(stat_val.st_mode) + return (stat_val.st_ctime, stat_val.st_mtime, stat_val.st_dev, + stat_val.st_ino, mode, stat_val.st_uid, + stat_val.st_gid, stat_val.st_size, hex_sha, flags) + + class VCS(Bcfg2.Client.Tools.Tool): """VCS support.""" __handles__ = [('Path', 'vcs')] @@ -47,11 +80,24 @@ class VCS(Bcfg2.Client.Tools.Tool): self.logger.info("Repository %s does not exist" % entry.get('name')) return False - cur_rev = repo.head() - if cur_rev != entry.get('revision'): + try: + expected_rev = entry.get('revision') + cur_rev = repo.head() + except: + return False + + try: + client, path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) + remote_refs = client.fetch_pack(path, (lambda x: None), None, None, None) + if expected_rev in remote_refs: + expected_rev = remote_refs[expected_rev] + except: + pass + + if cur_rev != expected_rev: self.logger.info("At revision %s need to go to revision %s" % - (cur_rev, entry.get('revision'))) + (cur_rev.strip(), expected_rev.strip())) return False return True @@ -71,45 +117,64 @@ class VCS(Bcfg2.Client.Tools.Tool): destname) return False - destr = dulwich.repo.Repo.init(destname, mkdir=True) + dulwich.file.ensure_dir_exists(destname) + destr = dulwich.repo.Repo.init(destname) cl, host_path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) remote_refs = cl.fetch(host_path, destr, determine_wants=destr.object_store.determine_wants_all, progress=sys.stdout.write) - destr.refs['refs/heads/master'] = entry.get('revision') - dtree = destr[entry.get('revision')].tree - obj_store = destr.object_store - for fname, mode, sha in obj_store.iter_tree_contents(dtree): - fullpath = os.path.join(destname, fname) - try: - f = open(os.path.join(destname, fname), 'wb') - except IOError: - dir = os.path.split(fullpath)[0] - os.makedirs(dir) - f = open(os.path.join(destname, fname), 'wb') - f.write(destr[sha].data) - f.close() - os.chmod(os.path.join(destname, fname), mode) + + if entry.get('revision') in remote_refs: + destr.refs['HEAD'] = remote_refs[entry.get('revision')] + else: + destr.refs['HEAD'] = entry.get('revision') + + dtree = destr['HEAD'].tree + index = dulwich.index.Index(destr.index_path()) + for fname, mode, sha in destr.object_store.iter_tree_contents(dtree): + full_path = os.path.join(destname, fname) + dulwich.file.ensure_dir_exists(os.path.dirname(full_path)) + + if stat.S_ISLNK(mode): + src_path = destr[sha].as_raw_string() + try: + os.symlink(src_path, full_path) + except OSError: + e = sys.exc_info()[1] + if e.errno == errno.EEXIST: + os.unlink(full_path) + os.symlink(src_path, full_path) + else: + raise + else: + file = open(full_path, 'wb') + file.write(destr[sha].as_raw_string()) + file.close() + os.chmod(full_path, mode) + + st = os.lstat(full_path) + index[fname] = index_entry_from_stat(st, sha, 0) + + index.write() return True - # FIXME: figure out how to write the git index properly - #iname = "%s/.git/index" % entry.get('name') - #f = open(iname, 'w+') - #entries = obj_store[sha].iteritems() - #try: - # dulwich.index.write_index(f, entries) - #finally: - # f.close() def Verifysvn(self, entry, _): """Verify svn repositories""" + headrev = pysvn.Revision( pysvn.opt_revision_kind.head ) client = pysvn.Client() try: cur_rev = str(client.info(entry.get('name')).revision.number) + server = client.info2(entry.get('sourceurl'), headrev, recurse=False) + if server: + server_rev = str(server[0][1].rev.number) except: self.logger.info("Repository %s does not exist" % entry.get('name')) return False + if entry.get('revision') == 'latest' and cur_rev == server_rev: + return True + if cur_rev != entry.get('revision'): self.logger.info("At revision %s need to go to revision %s" % (cur_rev, entry.get('revision'))) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index c5a5ee4d6..703b8ff57 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -519,6 +519,22 @@ class SvcTool(Tool): """ return '/etc/init.d/%s %s' % (service.get('name'), action) + def get_bootstatus(self, service): + """ Return the bootstatus attribute if it exists. + + :param service: The service entry + :type service: lxml.etree._Element + :returns: string or None - Value of bootstatus if it exists. If + bootstatus is unspecified and status is not *ignore*, + return value of status. If bootstatus is unspecified + and status is *ignore*, return None. + """ + if service.get('bootstatus') is not None: + return service.get('bootstatus') + elif service.get('status') != 'ignore': + return service.get('status') + return None + def start_service(self, service): """ Start a service. @@ -578,13 +594,14 @@ class SvcTool(Tool): if not self.handlesEntry(entry): continue + estatus = entry.get('status') restart = entry.get("restart", "true").lower() - if (restart == "false" or + if (restart == "false" or estatus == 'ignore' or (restart == "interactive" and not self.setup['interactive'])): continue success = False - if entry.get('status') == 'on': + if estatus == 'on': if self.setup['servicemode'] == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 91d4ac5c6..4ba06abae 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -5,9 +5,29 @@ # pylint: disable=E0611,W0611,W0613,C0103 try: - from lxml.etree import Element, SubElement, XML, tostring + from lxml.etree import Element, SubElement, tostring, XMLParser from lxml.etree import XMLSyntaxError as ParseError + from lxml.etree import XML as _XML + from Bcfg2.Compat import wraps driver = 'lxml' + + # libxml2 2.9.0+ doesn't parse 10M+ documents by default: + # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html + try: + _parser = XMLParser(huge_tree=True) + except TypeError: + _parser = XMLParser() + + @wraps(_XML) + def XML(val, **kwargs): + """ unicode strings w/encoding declaration are not supported in + recent lxml.etree, so we try to read XML, and if it fails we try + encoding the string. """ + kwargs.setdefault('parser', _parser) + try: + return _XML(val, **kwargs) + except ValueError: + return _XML(val.encode(), **kwargs) except ImportError: # lxml not available from xml.parsers.expat import ExpatError as ParseError diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 3bc261f2f..6d1cb9d40 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -21,9 +21,9 @@ def prompt(msg): try: ans = input(msg) return ans in ['y', 'Y'] + except UnicodeEncodeError: + ans = input(msg.encode('utf-8')) + return ans in ['y', 'Y'] except EOFError: - # python 2.4.3 on CentOS doesn't like ^C for some reason - return False - except: - print("Error while reading input: %s" % sys.exc_info()[1]) - return False + # handle ^C on rhel-based platforms + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index d034c0777..049236e03 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -79,10 +79,7 @@ except NameError: def u_str(string, encoding=None): """ print to file compatibility """ if sys.hexversion >= 0x03000000: - if encoding is not None: - return string.encode(encoding) - else: - return string + return string else: if encoding is not None: return unicode(string, encoding) diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py index b4674d72f..69d40ea37 100755 --- a/src/lib/Bcfg2/Encryption.py +++ b/src/lib/Bcfg2/Encryption.py @@ -3,6 +3,7 @@ handling encryption in Bcfg2. See :ref:`server-encryption` for more details. """ import os +import sys from M2Crypto import Rand from M2Crypto.EVP import Cipher, EVPError from Bcfg2.Compat import StringIO, md5, b64encode, b64decode @@ -114,13 +115,21 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): :returns: string - The decrypted data """ # base64-decode the data - data = b64decode(data) + try: + data = b64decode(data) + except TypeError: + # we do not include the data in the error message, because one + # of the common causes of this is data that claims to be + # encrypted but is not. we don't want to include a plaintext + # secret in the error logs. + raise TypeError("Could not decode base64 data: %s" % + sys.exc_info()[1]) salt = data[8:16] - # pylint: disable=E1101 + # pylint: disable=E1101,E1121 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) - # pylint: enable=E1101 + # pylint: enable=E1101,E1121 key = hashes[0] + hashes[1] iv = hashes[2] @@ -146,11 +155,11 @@ def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): if salt is None: salt = Rand.rand_bytes(8) - # pylint: disable=E1101 + # pylint: disable=E1101,E1121 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) - # pylint: enable=E1101 + # pylint: enable=E1101,E1121 key = hashes[0] + hashes[1] iv = hashes[2] diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 243c4ed2a..206c63d4f 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -312,6 +312,15 @@ def list_split(c_string): return [] +def list_split_anchored_regex(c_string): + """ like list_split but split on whitespace and compile each element as + anchored regex """ + try: + return [re.compile('^' + x + '$') for x in re.split(r'\s+', c_string)] + except re.error: + raise ValueError("Not a list of regexes", c_string) + + def colon_split(c_string): """ split an option string on colons, returning a list """ if c_string: @@ -319,6 +328,28 @@ def colon_split(c_string): return [] +def dict_split(c_string): + """ split an option string on commas, optionally surrounded by + whitespace and split the resulting items again on equals signs, + returning a dict """ + result = dict() + if c_string: + items = re.split(r'\s*,\s*', c_string) + for item in items: + if r'=' in item: + key, value = item.split(r'=', 1) + try: + result[key] = get_bool(value) + except ValueError: + try: + result[key] = get_int(value) + except ValueError: + result[key] = value + else: + result[item] = True + return result + + def get_bool(val): """ given a string value of a boolean configuration option, return an actual bool (True or False) """ @@ -619,6 +650,12 @@ SERVER_CHILDREN = \ cf=('server', 'children'), cook=get_int, long_arg=True) +SERVER_PROBE_ALLOWED_GROUPS = \ + Option('Whitespace-separated list of group names (as regex) to which ' + 'probes can assign a client by writing "group:" to stdout.', + default=[re.compile('.*')], + cf=('probes', 'allowed_groups'), + cook=list_split_anchored_regex) # database options DB_ENGINE = \ @@ -651,6 +688,15 @@ DB_PORT = \ default='', cf=('database', 'port'), deprecated_cf=('statistics', 'database_port')) +DB_OPTIONS = \ + Option('Database options', + default=dict(), + cf=('database', 'options'), + cook=dict_split) +DB_SCHEMA = \ + Option('Database schema', + default='public', + cf=('database', 'schema')) # Django options WEB_CFILE = \ @@ -1193,7 +1239,9 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, authentication=SERVER_AUTHENTICATION, perflog=LOG_PERFORMANCE, perflog_interval=PERFLOG_INTERVAL, - children=SERVER_CHILDREN) + children=SERVER_CHILDREN, + client_timeout=CLIENT_TIMEOUT, + probe_allowed_groups=SERVER_PROBE_ALLOWED_GROUPS) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, @@ -1233,9 +1281,9 @@ DRIVER_OPTIONS = \ yum_verify_fail_action=CLIENT_YUM_VERIFY_FAIL_ACTION, yum_verify_flags=CLIENT_YUM_VERIFY_FLAGS, posix_uid_whitelist=CLIENT_POSIX_UID_WHITELIST, - posix_gid_whitelist=CLIENT_POSIX_UID_WHITELIST, + posix_gid_whitelist=CLIENT_POSIX_GID_WHITELIST, posix_uid_blacklist=CLIENT_POSIX_UID_BLACKLIST, - posix_gid_blacklist=CLIENT_POSIX_UID_BLACKLIST) + posix_gid_blacklist=CLIENT_POSIX_GID_BLACKLIST) CLIENT_COMMON_OPTIONS = \ dict(extra=CLIENT_EXTRA_DISPLAY, @@ -1285,6 +1333,8 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, db_password=DB_PASSWORD, db_host=DB_HOST, db_port=DB_PORT, + db_options=DB_OPTIONS, + db_schema=DB_SCHEMA, time_zone=DJANGO_TIME_ZONE, django_debug=DJANGO_DEBUG, web_prefix=DJANGO_WEB_PREFIX) diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index f6db66a93..34080da6b 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -104,7 +104,6 @@ class RetryMethod(xmlrpclib._Method): err = sys.exc_info()[1] msg = err except: - raise etype, err = sys.exc_info()[:2] msg = "Unknown failure: %s (%s)" % (err, etype.__name__) if msg: diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index df82248d0..52700f917 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -20,10 +20,38 @@ from Bcfg2.Reporting.Transport.DirectStore import DirectStore from Bcfg2.Reporting.Storage import load_storage_from_config, \ StorageError, StorageImportError + class ReportingError(Exception): """Generic reporting exception""" pass + +class ReportingStoreThread(threading.Thread): + """Thread for calling the storage backend""" + def __init__(self, interaction, storage, group=None, target=None, + name=None, args=(), kwargs=None): + """Initialize the thread with a reference to the interaction + as well as the storage engine to use""" + threading.Thread.__init__(self, group, target, name, args, + kwargs or dict()) + self.interaction = interaction + self.storage = storage + self.logger = logging.getLogger('bcfg2-report-collector') + + def run(self): + """Call the database storage procedure (aka import)""" + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + traceback.format_exc().splitlines()[-1]) + + class ReportingCollector(object): """The collecting process for reports""" @@ -35,6 +63,8 @@ class ReportingCollector(object): self.encoding = setup['encoding'] self.terminate = None self.context = None + self.children = [] + self.cleanup_threshold = 25 if setup['debug']: level = logging.DEBUG @@ -77,12 +107,12 @@ class ReportingCollector(object): (self.storage.__class__.__name__, traceback.format_exc().splitlines()[-1])) - def run(self): """Startup the processing and go!""" self.terminate = threading.Event() atexit.register(self.shutdown) - self.context = daemon.DaemonContext() + self.context = daemon.DaemonContext(detach_process=True) + iter = 0 if self.setup['daemon']: self.logger.debug("Daemonizing") @@ -103,15 +133,16 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - try: - start = time.time() - self.storage.import_interaction(interaction) - self.logger.info("Imported interaction for %s in %ss" % - (interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - raise + + store_thread = ReportingStoreThread(interaction, self.storage) + store_thread.start() + self.children.append(store_thread) + + iter += 1 + if iter >= self.cleanup_threshold: + self.reap_children() + iter = 0 + except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() @@ -125,7 +156,22 @@ class ReportingCollector(object): # this wil be missing if called from bcfg2-admin self.terminate.set() if self.transport: - self.transport.shutdown() + try: + self.transport.shutdown() + except OSError: + pass if self.storage: self.storage.shutdown() + def reap_children(self): + """Join any non-live threads""" + newlist = [] + + self.logger.debug("Starting reap_children") + for child in self.children: + if child.isAlive(): + newlist.append(child) + else: + child.join() + self.logger.debug("Joined child thread %s" % child.getName()) + self.children = newlist diff --git a/src/lib/Bcfg2/Reporting/Compat.py b/src/lib/Bcfg2/Reporting/Compat.py new file mode 100644 index 000000000..9113fdb91 --- /dev/null +++ b/src/lib/Bcfg2/Reporting/Compat.py @@ -0,0 +1,16 @@ +""" Compatibility imports for Django. """ + +from django import VERSION +from django.db import transaction + +# Django 1.6 deprecated commit_on_success() and introduced atomic() with +# similar semantics. +if VERSION[0] == 1 and VERSION[1] < 6: + transaction.atomic = transaction.commit_on_success + +try: + # Django < 1.6 + from django.conf.urls.defaults import url, patterns +except ImportError: + # Django > 1.6 + from django.conf.urls import url, patterns diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index aea5e9d4b..ef1e92103 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -18,12 +18,12 @@ from django.core import management from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models import FieldDoesNotExist from django.core.cache import cache -from django.db import transaction #Used by GetCurrentEntry import difflib from Bcfg2.Compat import b64decode from Bcfg2.Reporting.models import * +from Bcfg2.Reporting.Compat import transaction class DjangoORM(StorageBase): @@ -256,7 +256,7 @@ class DjangoORM(StorageBase): entry.tag) return None - @transaction.commit_on_success + @transaction.atomic def _import_interaction(self, interaction): """Real import function""" hostname = interaction['hostname'] diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 598e1c6ec..fc9523067 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -88,7 +88,7 @@ class InteractionManager(models.Manager): Returns the ids of most recent interactions for clients as of a date. Arguments: - maxdate -- datetime object. Most recent date to pull. (dafault None) + maxdate -- datetime object. Most recent date to pull. (default None) """ from django.db import connection diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index c73339911..7edf3a949 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -1,4 +1,8 @@ {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} <?xml version="1.0"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> @@ -25,8 +29,9 @@ <div id="header"> <a href="http://bcfg2.org"><img src='{% to_media_url bcfg2_logo.png %}' - height='115' width='300' alt='Bcfg2' style='float:left; height: 115px' /></a> - </div> + height='115' width='300' alt='Bcfg2' + style='float:left; height: 115px' /></a> + </div> <div id="document"> <div id="content"><div id="contentwrapper"> @@ -46,26 +51,26 @@ <li>Overview</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_summary %}">Summary</a></li> - <li><a href="{% url reports_history %}">Recent Interactions</a></li> - <li><a href="{% url reports_timing %}">Timing</a></li> + <li><a href="{% url "reports_summary" %}">Summary</a></li> + <li><a href="{% url "reports_history" %}">Recent Interactions</a></li> + <li><a href="{% url "reports_timing" %}">Timing</a></li> </ul> <ul class='menu-level1'> <li>Clients</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_grid_view %}">Grid View</a></li> - <li><a href="{% url reports_detailed_list %}">Detailed List</a></li> - <li><a href="{% url reports_client_manage %}">Manage</a></li> + <li><a href="{% url "reports_grid_view" %}">Grid View</a></li> + <li><a href="{% url "reports_detailed_list" %}">Detailed List</a></li> + <li><a href="{% url "reports_client_manage" %}">Manage</a></li> </ul> <ul class='menu-level1'> <li>Entries Configured</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_common_problems %}">Common problems</a></li> - <li><a href="{% url reports_item_list "bad" %}">Bad</a></li> - <li><a href="{% url reports_item_list "modified" %}">Modified</a></li> - <li><a href="{% url reports_item_list "extra" %}">Extra</a></li> + <li><a href="{% url "reports_common_problems" %}">Common problems</a></li> + <li><a href="{% url "reports_item_list" "bad" %}">Bad</a></li> + <li><a href="{% url "reports_item_list" "modified" %}">Modified</a></li> + <li><a href="{% url "reports_item_list" "extra" %}">Extra</a></li> </ul> {% comment %} TODO @@ -88,7 +93,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.1</span> + <span>Bcfg2 Version 1.3.3</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 4608ce6f1..e890589a7 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -1,24 +1,28 @@ {% extends "base.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Client {{client.name}}{% endblock %} {% block extra_header_info %} <style type="text/css"> .node_data { - border: 1px solid #98DBCC; - margin: 10px; - padding-left: 18px; + border: 1px solid #98DBCC; + margin: 10px; + padding-left: 18px; } .node_data td { - padding: 1px 20px 1px 2px; + padding: 1px 20px 1px 2px; } span.history_links { - font-size: 90%; - margin-left: 50px; + font-size: 90%; + margin-left: 50px; } span.history_links a { - font-size: 90%; + font-size: 90%; } </style> {% endblock %} @@ -30,12 +34,12 @@ span.history_links a { {% block content %} <div class='detail_header'> <h2>{{client.name}}</h2> - <a href='{% url reports_client_manage %}#{{ client.name }}'>[manage]</a> - <span class='history_links'><a href="{% url reports_client_history client.name %}">View History</a> | Jump to + <a href='{% url "reports_client_manage" %}#{{ client.name }}'>[manage]</a> + <span class='history_links'><a href="{% url "reports_client_history" client.name %}">View History</a> | Jump to <select id="quick" name="quick" onchange="javascript:pageJump('quick');"> <option value="" selected="selected">--- Time ---</option> {% for i in client.interactions.all|slice:":25" %} - <option value="{% url reports_client_detail_pk hostname=client.name, pk=i.id %}">{{i.timestamp|date:"c"}}</option> + <option value="{% url "reports_client_detail_pk" hostname=client.name pk=i.id %}">{{i.timestamp|date:"c"}}</option> {% endfor %} </select></span> </div> @@ -110,7 +114,7 @@ span.history_links a { {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{entry.entry_type}}</td> - <td><a href="{% url reports_item entry.class_name entry.pk interaction.pk %}"> + <td><a href="{% url "reports_item" entry.class_name entry.pk interaction.pk %}"> {{entry.name}}</a></td> </tr> {% endfor %} @@ -129,7 +133,7 @@ span.history_links a { {% for failure in interaction.failures.all %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{failure.entry_type}}</td> - <td><a href="{% url reports_item failure.class_name failure.pk interaction.pk %}"> + <td><a href="{% url "reports_item" failure.class_name failure.pk interaction.pk %}"> {{failure.name}}</a></td> </tr> {% endfor %} @@ -140,11 +144,11 @@ span.history_links a { {% if entry_list %} <div class="entry_list recent_history_wrapper"> <div class="entry_list_head" style="border-bottom: 2px solid #98DBCC;"> - <h4 style="display: inline"><a href="{% url reports_client_history client.name %}">Recent Interactions</a></h4> + <h4 style="display: inline"><a href="{% url "reports_client_history" client.name %}">Recent Interactions</a></h4> </div> <div class='recent_history_box'> {% include "widgets/interaction_list.inc" %} - <div style='padding-left: 5px'><a href="{% url reports_client_history client.name %}">more...</a></div> + <div style='padding-left: 5px'><a href="{% url "reports_client_history" client.name %}">more...</a></div> </div> </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index fd9a545ce..33c78a5f0 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Detailed Client Listing{% endblock %} {% block pagebanner %}Clients - Detailed View{% endblock %} @@ -21,7 +25,7 @@ </tr> {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> - <td class='left_column'><a href='{% url Bcfg2.Reporting.views.client_detail hostname=entry.client.name, pk=entry.id %}'>{{ entry.client.name }}</a></td> + <td class='left_column'><a href='{% url "Bcfg2.Reporting.views.client_detail" hostname=entry.client.name pk=entry.id %}'>{{ entry.client.name }}</a></td> <td class='right_column' style='width:75px'><a href='{% add_url_filter state=entry.state %}' class='{{entry|determine_client_state}}'>{{ entry.state }}</a></td> <td class='right_column_narrow'>{{ entry.good_count }}</td> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index d9c415c20..eba83670b 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block extra_header_info %} {% endblock%} @@ -17,9 +21,9 @@ <td class='{{ inter|determine_client_state }}'> <a href="{% spaceless %} {% if not timestamp %} - {% url reports_client_detail inter.client.name %} + {% url "reports_client_detail" inter.client.name %} {% else %} - {% url reports_client_detail_pk inter.client.name,inter.id %} + {% url "reports_client_detail_pk" inter.client.name inter.id %} {% endif %} {% endspaceless %}">{{ inter.client.name }}</a> </td> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html index 443ec8ccb..03918aad7 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/manage.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html @@ -1,4 +1,8 @@ {% extends "base.html" %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block extra_header_info %} {% endblock%} @@ -24,10 +28,10 @@ <td><span id="{{ client.name }}"> </span> <span id="ttag-{{ client.name }}"> </span> <span id="s-ttag-{{ client.name }}"> </span> - <a href="{% url reports_client_detail client.name %}">{{ client.name }}</a></td> + <a href='{% url "reports_client_detail" client.name %}'>{{ client.name }}</a></td> <td>{% firstof client.expiration 'Active' %}</td> <td> - <form method="post" action="{% url reports_client_manage %}"> + <form method="post" action='{% url "reports_client_manage" %}'> <div> {# here for no reason other then to validate #} <input type="hidden" name="client_name" value="{{ client.name }}" /> <input type="hidden" name="client_action" value="{% if client.expiration %}unexpire{% else %}expire{% endif %}" /> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html index 57191ec39..91f37d7dc 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/common.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -1,5 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% load url from future %} {% block title %}Bcfg2 - Common Problems{% endblock %} @@ -29,9 +30,9 @@ {% for item in type_list %} <tr class='{% cycle listview,listview_alt %}'> <td>{{ item.ENTRY_TYPE }}</td> - <td><a href="{% url reports_entry item.class_name, item.pk %}">{{ item.name }}</a></td> + <td><a href='{% url "reports_entry" item.class_name item.pk %}'>{{ item.name }}</a></td> <td>{{ item.num_entries }}</td> - <td><a href="{% url reports_item item.ENTRY_TYPE, item.pk %}">{{ item.short_list|join:"," }}</a></td> + <td><a href='{% url "reports_item" item.ENTRY_TYPE item.pk %}'>{{ item.short_list|join:"," }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html index e940889ab..e3befb0eb 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Entry Status{% endblock %} @@ -17,10 +21,10 @@ {% for item, inters in items %} {% for inter in inters %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href='{% url reports_client_detail hostname=inter.client.name %}'>{{inter.client.name}}</a></td> - <td><a href='{% url reports_client_detail_pk hostname=inter.client.name, pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> + <td><a href='{% url "reports_client_detail" hostname=inter.client.name %}'>{{inter.client.name}}</a></td> + <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> <td>{{ item.get_state_display }}</td> - <td style='white-space: nowrap'><a href="{% url reports_item entry_type=item.class_name pk=item.pk %}">({{item.pk}}) {{item.short_list|join:","}}</a></td> + <td style='white-space: nowrap'><a href='{% url "reports_item" entry_type=item.class_name pk=item.pk %}'>({{item.pk}}) {{item.short_list|join:","}}</a></td> </tr> {% endfor %} {% endfor %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index 259414399..b03d48045 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -1,6 +1,10 @@ {% extends "base.html" %} {% load split %} {% load syntax_coloring %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Element Details{% endblock %} @@ -9,20 +13,20 @@ {% block extra_header_info %} <style type="text/css"> #table_list_header { - font-size: 100%; + font-size: 100%; } table.entry_list { - width: auto; + width: auto; } div.information_wrapper { - margin: 15px; + margin: 15px; } div.diff_wrapper { - overflow: auto; + overflow: auto; } div.entry_list h3 { - font-size: 90%; - padding: 5px; + font-size: 90%; + padding: 5px; } </style> {% endblock%} @@ -131,9 +135,9 @@ div.entry_list h3 { {% if associated_list %} <table class="entry_list" cellpadding="3"> {% for inter in associated_list %} - <tr><td><a href="{% url reports_client_detail inter.client.name %}" + <tr><td><a href='{% url "reports_client_detail" inter.client.name %}' >{{inter.client.name}}</a></td> - <td><a href="{% url reports_client_detail_pk hostname=inter.client.name,pk=inter.id %}" + <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.id %}' >{{inter.timestamp}}</a></td> </tr> {% endfor %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html index 864392754..0e4812e85 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Element Listing{% endblock %} @@ -21,9 +25,9 @@ <tr style='text-align: left' ><th>Name</th><th>Count</th><th>Reason</th></tr> {% for entry in type_data %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_entry entry.class_name entry.pk %}">{{entry.name}}</a></td> + <td><a href='{% url "reports_entry" entry.class_name entry.pk %}'>{{entry.name}}</a></td> <td>{{entry.num_entries}}</td> - <td><a href="{% url reports_item entry.class_name entry.pk %}">{{entry.short_list|join:","}}</a></td> + <td><a href='{% url "reports_item" entry.class_name entry.pk %}'>{{entry.short_list|join:","}}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html index b9847cf96..ffafd52e0 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/summary.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Client Summary{% endblock %} {% block pagebanner %}Clients - Summary{% endblock %} @@ -30,7 +34,7 @@ hide_tables[{{ forloop.counter0 }}] = "table_{{ summary.name }}"; <table id='table_{{ summary.name }}' class='entry_list'> {% for node in summary.nodes|sort_interactions_by_name %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_client_detail_pk hostname=node.client.name,pk=node.id %}">{{ node.client.name }}</a></td> + <td><a href='{% url "reports_client_detail_pk" hostname=node.client.name pk=node.id %}'>{{ node.client.name }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html index ff775ded5..8ac5e49bb 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/timing.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Performance Metrics{% endblock %} {% block pagebanner %}Performance Metrics{% endblock %} @@ -12,7 +16,7 @@ <div class='client_list_box'> {% if metrics %} <table cellpadding="3"> - <tr id='table_list_header' class='listview'> + <tr id='table_list_header' class='listview'> <td>Name</td> <td>Parse</td> <td>Probe</td> @@ -21,15 +25,15 @@ <td>Config</td> <td>Total</td> </tr> - {% for metric in metrics|dictsort:"name" %} + {% for metric in metrics|dictsort:"name" %} <tr class='{% cycle listview,listview_alt %}'> <td><a style='font-size: 100%' - href="{% url reports_client_detail hostname=metric.name %}">{{ metric.name }}</a></td> + href='{% url "reports_client_detail" hostname=metric.name %}'>{{ metric.name }}</a></td> {% for mitem in metric|build_metric_list %} <td>{{ mitem }}</td> {% endfor %} - </tr> - {% endfor %} + </tr> + {% endfor %} </table> {% else %} <p>No metric data available</p> diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index f5f2e7528..489682f30 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -5,9 +5,8 @@ from django import template from django.conf import settings from django.core.urlresolvers import resolve, reverse, \ Resolver404, NoReverseMatch -from django.template.loader import get_template, \ - get_template_from_string,TemplateDoesNotExist -from django.utils.encoding import smart_unicode, smart_str +from django.template.loader import get_template_from_string +from django.utils.encoding import smart_str from django.utils.safestring import mark_safe from datetime import datetime, timedelta from Bcfg2.Reporting.utils import filter_list @@ -133,19 +132,22 @@ def filter_navigator(context): del myargs[filter] filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(lambda x, y: cmp(x[0], y[0])) + filters.sort(key=lambda x: x[0]) myargs = kwargs.copy() - selected=True + selected = True if 'group' in myargs: del myargs['group'] - selected=False - groups = [('---', reverse(view, args=args, kwargs=myargs) + qs, selected)] + selected = False + groups = [('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)] for group in Group.objects.values('name'): myargs['group'] = group['name'] - groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, - group['name'] == kwargs.get('group', ''))) - + groups.append((group['name'], + reverse(view, args=args, kwargs=myargs) + qs, + group['name'] == kwargs.get('group', ''))) + return {'filters': filters, 'groups': groups} except (Resolver404, NoReverseMatch, ValueError, KeyError): pass @@ -205,7 +207,7 @@ def sort_interactions_by_name(value): Sort an interaction list by client name """ inters = list(value) - inters.sort(lambda a, b: cmp(a.client.name, b.client.name)) + inters.sort(key=lambda a: a.client.name) return inters @@ -223,7 +225,7 @@ class AddUrlFilter(template.Node): filter_value = self.filter_value.resolve(context, True) if filter_value: filter_name = smart_str(self.filter_name) - filter_value = smart_unicode(filter_value) + filter_value = smart_str(filter_value) kwargs[filter_name] = filter_value # These two don't make sense if filter_name == 'server' and 'hostname' in kwargs: @@ -306,6 +308,7 @@ def to_media_url(parser, token): return MediaTag(filter_value) + @register.filter def determine_client_state(entry): """ @@ -338,10 +341,11 @@ def do_qs(parser, token): try: tag, name, value = token.split_contents() except ValueError: - raise template.TemplateSyntaxError, "%r tag requires exactly two arguments" \ - % token.contents.split()[0] + raise template.TemplateSyntaxError("%r tag requires exactly two arguments" + % token.contents.split()[0]) return QsNode(name, value) + class QsNode(template.Node): def __init__(self, name, value): self.name = template.Variable(name) @@ -359,7 +363,7 @@ class QsNode(template.Node): return '' except KeyError: if settings.TEMPLATE_DEBUG: - raise Exception, "'qs' tag requires context['request']" + raise Exception("'qs' tag requires context['request']") return '' except: return '' @@ -380,6 +384,7 @@ def sort_link(parser, token): return SortLinkNode(sort_key, text) + class SortLinkNode(template.Node): __TMPL__ = "{% load bcfg2_tags %}<a href='{% qs 'sort' key %}'>{{ text }}</a>" @@ -420,4 +425,3 @@ class SortLinkNode(template.Node): raise raise return '' - diff --git a/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py index 2712d6395..22700689f 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py +++ b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py @@ -1,11 +1,8 @@ -import sys from django import template -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_str from django.utils.html import conditional_escape from django.utils.safestring import mark_safe -from Bcfg2.Compat import u_str - register = template.Library() # pylint: disable=E0611 @@ -33,9 +30,9 @@ def syntaxhilight(value, arg="diff", autoescape=None): if colorize: try: - output = u_str('<style type="text/css">') \ - + smart_unicode(HtmlFormatter().get_style_defs('.highlight')) \ - + u_str('</style>') + output = smart_str('<style type="text/css">') \ + + smart_str(HtmlFormatter().get_style_defs('.highlight')) \ + + smart_str('</style>') lexer = get_lexer_by_name(arg) output += highlight(value, lexer, HtmlFormatter()) @@ -43,6 +40,7 @@ def syntaxhilight(value, arg="diff", autoescape=None): except: return value else: - return mark_safe(u_str('<div class="note-box">Tip: Install pygments ' - 'for highlighting</div><pre>%s</pre>') % value) + return mark_safe(smart_str( + '<div class="note-box">Tip: Install pygments ' + 'for highlighting</div><pre>%s</pre>') % value) syntaxhilight.needs_autoescape = True diff --git a/src/lib/Bcfg2/Reporting/urls.py b/src/lib/Bcfg2/Reporting/urls.py index 8330fef7b..3a40cb932 100644 --- a/src/lib/Bcfg2/Reporting/urls.py +++ b/src/lib/Bcfg2/Reporting/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls.defaults import * +from Bcfg2.Reporting.Compat import url, patterns # django compat imports from django.core.urlresolvers import reverse, NoReverseMatch from django.http import HttpResponsePermanentRedirect from Bcfg2.Reporting.utils import filteredUrls, paginatedUrls, timeviewUrls diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index 619501d8b..0d394fcd8 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -1,5 +1,4 @@ """Helper functions for reports""" -from django.conf.urls.defaults import * import re """List of filters provided by filteredUrls""" diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 6cba7bf8c..c7c2a503f 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -338,6 +338,8 @@ def client_detail(request, hostname=None, pk=None): for label in etypes.values(): edict[label] = [] for ekind in inter.entry_types: + if ekind == 'failures': + continue for ent in getattr(inter, ekind).all(): edict[etypes[ent.state]].append(ent) context['entry_types'] = edict diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 316c2f86c..ab7e56f33 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -5,7 +5,6 @@ better. """ import os import sys import socket -import select import signal import logging import ssl @@ -183,7 +182,6 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): Adds support for HTTP authentication. """ - logger = logging.getLogger("Bcfg2.SSLServer.XMLRPCRequestHandler") def authenticate(self): @@ -228,22 +226,22 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): return False return True - ### need to override do_POST here def do_POST(self): try: max_chunk_size = 10 * 1024 * 1024 size_remaining = int(self.headers["content-length"]) L = [] while size_remaining: - try: - select.select([self.rfile.fileno()], [], [], 3) - except select.error: - print("got select timeout") - raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size).decode('utf-8')) + chunk = self.rfile.read(chunk_size).decode('utf-8') + if not chunk: + break + L.append(chunk) size_remaining -= len(L[-1]) data = ''.join(L) + if data is None: + return # response has been sent + response = self.server._marshaled_dispatch(self.client_address, data) if sys.hexversion >= 0x03000000: @@ -251,6 +249,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): except: # pylint: disable=W0702 try: self.send_response(500) + self.send_header("Content-length", "0") self.end_headers() except: (etype, msg) = sys.exc_info()[:2] @@ -306,14 +305,11 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def finish(self): # shut down the connection - if not self.wfile.closed: - try: - self.wfile.flush() - self.wfile.close() - except socket.error: - err = sys.exc_info()[1] - self.logger.warning("Error closing connection: %s" % err) - self.rfile.close() + try: + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.finish(self) + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, @@ -431,8 +427,6 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, self.handle_request() except socket.timeout: pass - except select.error: - pass except: self.logger.error("Got unexpected error in handle_request", exc_info=1) diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py index 570e993ed..325b7ae6e 100644 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -5,9 +5,22 @@ import Bcfg2.Server.Admin from Bcfg2.Server.Plugin import MetadataConsistencyError +def get_attribs(args): + """ Get a list of attributes to set on a client when adding/updating it """ + attr_d = {} + for i in args[2:]: + attr, val = i.split('=', 1) + if attr not in ['profile', 'uuid', 'password', 'floating', 'secure', + 'address', 'auth']: + print("Attribute %s unknown" % attr) + raise SystemExit(1) + attr_d[attr] = val + return attr_d + + class Client(Bcfg2.Server.Admin.MetadataCore): """ Create, delete, or list client entries """ - __usage__ = "[options] [add|del|list] [attr=val]" + __usage__ = "[options] [add|del|update|list] [attr=val]" __plugin_whitelist__ = ["Metadata"] def __call__(self, args): @@ -16,21 +29,22 @@ class Client(Bcfg2.Server.Admin.MetadataCore): "Usage: %s" % self.__usage__) if args[0] == 'add': try: - self.metadata.add_client(args[1]) + self.metadata.add_client(args[1], get_attribs(args)) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in adding client: %s" % err) - raise SystemExit(1) + self.errExit("Error adding client: %s" % sys.exc_info()[1]) + elif args[0] in ['update', 'up']: + try: + self.metadata.update_client(args[1], get_attribs(args)) + except MetadataConsistencyError: + self.errExit("Error updating client: %s" % sys.exc_info()[1]) elif args[0] in ['delete', 'remove', 'del', 'rm']: try: self.metadata.remove_client(args[1]) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in deleting client: %s" % err) - raise SystemExit(1) + self.errExit("Error deleting client: %s" % + sys.exc_info()[1]) elif args[0] in ['list', 'ls']: for client in self.metadata.list_clients(): print(client) else: - print("No command specified") - raise SystemExit(1) + self.errExit("No command specified") diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index c56dd0a8f..d7285284a 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -115,7 +115,6 @@ class Compare(Bcfg2.Server.Admin.Mode): return identical def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0: self.errExit("No argument specified.\n" "Please see bcfg2-admin compare help for usage.") @@ -145,5 +144,4 @@ class Compare(Bcfg2.Server.Admin.Mode): (old, new) = args return self.compareSpecifications(new, old) except IndexError: - print(self.__call__.__doc__) - raise SystemExit(1) + self.errExit(self.__call__.__doc__) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 6175d8ed0..153d7bea6 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -20,6 +20,8 @@ from Bcfg2.Compat import input # pylint: disable=W0622 CONFIG = '''[server] repository = %s plugins = %s +# Uncomment the following to listen on all interfaces +#listen_all = true [statistics] sendmailpath = %s @@ -31,7 +33,7 @@ sendmailpath = %s # 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. #name = # Or path to database file if using sqlite3. -#<repository>/bcfg2.sqlite is default path if left empty +#<repository>/etc/bcfg2.sqlite is default path if left empty #user = # Not used with sqlite3. #password = @@ -78,7 +80,7 @@ CLIENTS = '''<Clients version="3.0"> ''' # Mapping of operating system names to groups -OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), +OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/CentOS', 'redhat'), ('SUSE/SLES', 'suse'), ('Mandrake', 'mandrake'), ('Debian', 'debian'), @@ -234,8 +236,9 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_server(self): """Ask for the server name.""" - newserver = safe_input("Input the server location [%s]: " % - self.data['server_uri']) + newserver = safe_input( + "Input the server location (the server listens on a single " + "interface by default) [%s]: " % self.data['server_uri']) if newserver != '': self.data['server_uri'] = newserver diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 93e42305c..37ca74894 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -20,9 +20,8 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): "Please see bcfg2-admin minestruct help for usage.") try: (opts, args) = getopt.getopt(args, 'f:g:h') - except: - self.log.error(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) client = args[0] output = sys.stdout @@ -33,8 +32,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: output = open(optarg, 'w') except IOError: - self.log.error("Failed to open file: %s" % (optarg)) - raise SystemExit(1) + self.errExit("Failed to open file: %s" % (optarg)) elif opt == '-g': groups = optarg.split(':') @@ -43,10 +41,9 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) - except: - self.log.error("Failed to find extra entry info for client %s" % - client) - raise SystemExit(1) + except: # pylint: disable=W0702 + self.errExit("Failed to find extra entry info for client %s" % + client) root = lxml.etree.Element("Base") self.log.info("Found %d extra entries" % (len(extra))) add_point = root diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 8001425df..459fcec65 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -32,9 +32,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): use_stdin = False try: opts, gargs = getopt.getopt(args, 'vfIs') - except: - print(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) for opt in opts: if opt[0] == '-v': self.log = True diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 6e313e84b..eb97123f7 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -19,11 +19,11 @@ sys.path.pop() # Set DJANGO_SETTINGS_MODULE appropriately. os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name -from django.db import transaction from Bcfg2.Reporting.models import Client, Interaction, \ Performance, Bundle, Group, FailureEntry, PathEntry, \ PackageEntry, ServiceEntry, ActionEntry +from Bcfg2.Reporting.Compat import transaction def printStats(fn): @@ -79,8 +79,7 @@ class Reports(Bcfg2.Server.Admin.Mode): def __call__(self, args): if len(args) == 0 or args[0] == '-h': - print(self.__usage__) - raise SystemExit(0) + self.errExit(self.__usage__) # FIXME - dry run @@ -101,9 +100,7 @@ class Reports(Bcfg2.Server.Admin.Mode): management.call_command("syncdb", verbosity=vrb) management.call_command("migrate", verbosity=vrb) except: - print("Update failed: %s" % - traceback.format_exc().splitlines()[-1]) - raise SystemExit(1) + self.errExit("Update failed: %s" % sys.exc_info()[1]) elif args[0] == 'purge': expired = False client = None @@ -124,24 +121,22 @@ class Reports(Bcfg2.Server.Admin.Mode): maxdate = datetime.datetime.now() - \ datetime.timedelta(days=int(args[i + 1])) except: - self.log.error("Invalid number of days: %s" % - args[i + 1]) - raise SystemExit(-1) + self.errExit("Invalid number of days: %s" % + args[i + 1]) i = i + 1 elif args[i] == '--expired': expired = True i = i + 1 if expired: if state: - self.log.error("--state is not valid with --expired") - raise SystemExit(-1) + self.errExit("--state is not valid with --expired") self.purge_expired(maxdate) else: self.purge(client, maxdate, state) else: - print("Unknown command: %s" % args[0]) + self.errExit("Unknown command: %s" % args[0]) - @transaction.commit_on_success + @transaction.atomic def scrub(self): ''' Perform a thorough scrub and cleanup of the database ''' @@ -155,8 +150,7 @@ class Reports(Bcfg2.Server.Admin.Mode): (start_count - cls.objects.count(), cls.__class__.__name__)) except: print("Failed to prune %s: %s" % - (cls.__class__.__name__, - traceback.format_exc().splitlines()[-1])) + (cls.__class__.__name__, sys.exc_info()[1])) def django_command_proxy(self, command): '''Call a django command''' @@ -180,8 +174,7 @@ class Reports(Bcfg2.Server.Admin.Mode): cobj = Client.objects.get(name=client) ipurge = ipurge.filter(client=cobj) except Client.DoesNotExist: - self.log.error("Client %s not in database" % client) - raise SystemExit(-1) + self.errExit("Client %s not in database" % client) self.log.debug("Filtering by client: %s" % client) if maxdate: diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index c2d279391..fcb240352 100644 --- a/src/lib/Bcfg2/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -27,7 +27,6 @@ class Snapshots(Bcfg2.Server.Admin.Mode): self.cfile = self.configfile def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0 or args[0] == '-h': print(self.__usage__) raise SystemExit(0) diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py index 4ba840b86..eb417966d 100644 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py @@ -3,6 +3,7 @@ import Bcfg2.settings import Bcfg2.Options import Bcfg2.Server.Admin import Bcfg2.Server.models +from django.core.exceptions import ImproperlyConfigured from django.core.management import setup_environ, call_command @@ -22,10 +23,7 @@ class Syncdb(Bcfg2.Server.Admin.Mode): call_command("syncdb", interactive=False, verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.log.error("Django configuration problem: %s" % err) - raise SystemExit(1) + self.errExit("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] - self.log.error("Database update failed: %s" % err) - raise SystemExit(1) + self.errExit("Database update failed: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py index 1d9d25f16..2cbd7eaf6 100644 --- a/src/lib/Bcfg2/Server/Admin/Viz.py +++ b/src/lib/Bcfg2/Server/Admin/Viz.py @@ -102,6 +102,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): dotpipe.stdin.write('\tcolor="lightblue";\n') dotpipe.stdin.write('\tBundle [ shape="septagon" ];\n') dotpipe.stdin.write('\tGroup [shape="ellipse"];\n') + dotpipe.stdin.write('\tGroup Category [shape="trapezium"];\n') dotpipe.stdin.write('\tProfile [style="bold", shape="ellipse"];\n') dotpipe.stdin.write('\tHblock [label="Host1|Host2|Host3", ' 'shape="record"];\n') diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index be556bed4..036129a1b 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -4,7 +4,6 @@ import sys import Bcfg2.Options import Bcfg2.Proxy import Bcfg2.Server.Admin -from Bcfg2.Compat import xmlrpclib class Xcmd(Bcfg2.Server.Admin.Mode): @@ -31,27 +30,15 @@ class Xcmd(Bcfg2.Server.Admin.Mode): ca=setup['ca'], timeout=setup['timeout']) if len(setup['args']) == 0: - print("Usage: xcmd <xmlrpc method> <optional arguments>") - return + self.errExit("Usage: xcmd <xmlrpc method> <optional arguments>") cmd = setup['args'][0] args = () if len(setup['args']) > 1: args = tuple(setup['args'][1:]) try: data = getattr(proxy, cmd)(*args) - except xmlrpclib.Fault: - flt = sys.exc_info()[1] - if flt.faultCode == 7: - print("Unknown method %s" % cmd) - return - elif flt.faultCode == 20: - return - else: - raise except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - print("Proxy Error: %s" % err) - return + self.errExit("Proxy Error: %s" % sys.exc_info()[1]) if data is not None: print(data) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index e69a92b64..93da767c7 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -31,7 +31,8 @@ class Core(BaseCore): daemon_args = dict(uid=self.setup['daemon_uid'], gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8)) + umask=int(self.setup['umask'], 8), + detach_process=True) if self.setup['daemon']: daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], acquire_timeout=5) @@ -109,7 +110,6 @@ class Core(BaseCore): keyfile=self.setup['key'], certfile=self.setup['cert'], register=False, - timeout=1, ca=self.setup['ca'], protocol=self.setup['protocol']) except: # pylint: disable=W0702 diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index c246860c1..c2cf6b7a4 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -2,14 +2,14 @@ implementations inherit from. """ import os -import sys -import time +import pwd import atexit -import select -import signal import logging -import inspect +import select +import sys import threading +import time +import inspect import lxml.etree import Bcfg2.settings import Bcfg2.Server @@ -200,6 +200,10 @@ class BaseCore(object): # load plugins Bcfg2.settings.read_config(repo=self.datastore) + # mapping of group name => plugin name to record where groups + # that are created by Connector plugins came from + self._dynamic_groups = dict() + #: Whether or not it's possible to use the Django database #: backend for plugins that have that capability self._database_available = False @@ -224,11 +228,11 @@ class BaseCore(object): verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.logger.error("Django configuration problem: %s" % err) + self.logger.error("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] - self.logger.error("Database update failed: %s" % err) + self.logger.error("Database update failed: %s" % + sys.exc_info()[1]) if do_chown and self._database_available: try: @@ -243,14 +247,6 @@ class BaseCore(object): #: The CA that signed the server cert self.ca = setup['ca'] - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - self.shutdown() - os._exit(1) # pylint: disable=W0212 - - signal.signal(signal.SIGINT, hdlr) - #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -271,6 +267,20 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + def expire_caches_by_type(self, base_cls, key=None): + """ Expire caches for all + :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that + are instances of ``base_cls``. + + :param base_cls: The base plugin interface class to match (see + :mod:`Bcfg2.Server.Plugin.interfaces`) + :type base_cls: type + :param key: The cache key to expire + """ + for plugin in self.plugins_by_type(base_cls): + if isinstance(plugin, Bcfg2.Server.Plugin.Caching): + plugin.expire_cache(key) + def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -297,11 +307,12 @@ class BaseCore(object): self.logger.debug("Performance logging thread starting") while not self.terminate.isSet(): self.terminate.wait(self.setup['perflog_interval']) - for name, stats in self.get_statistics(None).items(): - self.logger.info("Performance statistics: " - "%s min=%.06f, max=%.06f, average=%.06f, " - "count=%d" % ((name, ) + stats)) - self.logger.debug("Performance logging thread terminated") + if not self.terminate.isSet(): + for name, stats in self.get_statistics(None).items(): + self.logger.info("Performance statistics: " + "%s min=%.06f, max=%.06f, average=%.06f, " + "count=%d" % ((name, ) + stats)) + self.logger.info("Performance logging thread terminated") def _file_monitor_thread(self): """ The thread that runs the @@ -318,11 +329,12 @@ class BaseCore(object): else: if not self.fam.pending(): terminate.wait(15) + if self.fam.pending(): + self._update_vcs_revision() self.fam.handle_event_set(self.lock) except: continue - self._update_vcs_revision() - self.logger.debug("File monitor thread terminated") + self.logger.info("File monitor thread terminated") @track_statistics() def _update_vcs_revision(self): @@ -438,14 +450,14 @@ class BaseCore(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ - self.logger.debug("Shutting down core...") + self.logger.info("Shutting down core...") if not self.terminate.isSet(): self.terminate.set() self.fam.shutdown() - self.logger.debug("FAM shut down") + self.logger.info("FAM shut down") for plugin in list(self.plugins.values()): plugin.shutdown() - self.logger.debug("All plugins shut down") + self.logger.info("All plugins shut down") @property def metadata_cache_mode(self): @@ -636,10 +648,10 @@ class BaseCore(object): del entry.attrib['realname'] return ret except: - entry.set('name', oldname) self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('name'), - entry.get('altsrc'))) + (entry.tag, entry.get('realname'), + entry.get('name'))) + entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -734,7 +746,27 @@ class BaseCore(object): if event.code2str() == 'deleted': return self.setup.reparse() - self.metadata_cache.expire() + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + + def block_for_fam_events(self, handle_events=False): + """ Block until all fam events have been handleed, optionally + handling events as well. (Setting ``handle_events=True`` is + useful for local server cores that don't spawn an event + handling thread.)""" + slept = 0 + log_interval = 3 + if handle_events: + self.fam.handle_events_in_interval(1) + slept += 1 + if self.setup['fam_blocking']: + time.sleep(1) + slept += 1 + while self.fam.pending() != 0: + time.sleep(1) + slept += 1 + if slept % log_interval == 0: + self.logger.debug("Sleeping to handle FAM events...") + self.logger.debug("Slept %s seconds while handling FAM events" % slept) def run(self): """ Run the server core. This calls :func:`_daemonize`, @@ -758,6 +790,11 @@ class BaseCore(object): os.chmod(piddir, 493) # 0775 if not self._daemonize(): return False + + # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so + # this is necessary to make that work when privileges are + # dropped + os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] else: os.umask(int(self.setup['umask'], 8)) @@ -780,12 +817,9 @@ class BaseCore(object): self.shutdown() raise - if self.setup['fam_blocking']: - time.sleep(1) - while self.fam.pending() != 0: - time.sleep(1) - - self.set_debug(None, self.debug_flag) + if self.debug_flag: + self.set_debug(None, self.debug_flag) + self.block_for_fam_events() self._block() def _daemonize(self): @@ -844,17 +878,52 @@ class BaseCore(object): imd = self.metadata_cache.get(client_name, None) if not imd: self.logger.debug("Building metadata for %s" % client_name) - imd = self.metadata.get_initial_metadata(client_name) + try: + imd = self.metadata.get_initial_metadata(client_name) + except MetadataConsistencyError: + self.critical_error( + "Client metadata resolution error for %s: %s" % + (client_name, sys.exc_info()[1])) connectors = self.plugins_by_type(Connector) for conn in connectors: - grps = conn.get_additional_groups(imd) - self.metadata.merge_additional_groups(imd, grps) + groups = conn.get_additional_groups(imd) + groupnames = [] + for group in groups: + if hasattr(group, "name"): + groupname = group.name + if groupname in self._dynamic_groups: + if self._dynamic_groups[groupname] == conn.name: + self.metadata.groups[groupname] = group + else: + self.logger.warning( + "Refusing to clobber dynamic group %s " + "defined by %s" % + (self._dynamic_groups[groupname], + groupname)) + elif groupname in self.metadata.groups: + # not recorded as a dynamic group, but + # present in metadata.groups -- i.e., a + # static group + self.logger.warning( + "Refusing to clobber predefined group %s" % + groupname) + else: + self.metadata.groups[groupname] = group + self._dynamic_groups[groupname] = conn.name + groupnames.append(groupname) + else: + groupnames.append(group) + + self.metadata.merge_additional_groups(imd, groupnames) for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) imd.query.by_name = self.build_metadata if self.metadata_cache_mode in ['cautious', 'aggressive']: self.metadata_cache[client_name] = imd + else: + self.logger.debug("Using cached metadata object for %s" % + client_name) return imd def process_statistics(self, client_name, statistics): @@ -882,6 +951,7 @@ class BaseCore(object): state.get('state'))) self.client_run_hook("end_statistics", meta) + @track_statistics() def resolve_client(self, address, cleanup_cache=False, metadata=True): """ Given a client address, get the client hostname and optionally metadata. @@ -934,15 +1004,19 @@ class BaseCore(object): raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR, "Critical failure: %s" % message) + def _get_rmi_objects(self): + """ Get a dict (name: object) of all objects that may have RMI + calls. Currently, that includes all plugins and the FAM. """ + rv = {self.fam.__class__.__name__: self.fam} + rv.update(self.plugins) + return rv + def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ rmi = dict() - for pname, pinst in list(self.plugins.items()): + for pname, pinst in self._get_rmi_objects().items(): for mname in pinst.__rmi__: rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) - famname = self.fam.__class__.__name__ - for mname in self.fam.__rmi__: - rmi["%s.%s" % (famname, mname)] = getattr(self.fam, mname) return rmi def _resolve_exposed_method(self, method_name): @@ -1033,6 +1107,7 @@ class BaseCore(object): for plugin in self.plugins_by_type(Probing): for probe in plugin.GetProbes(metadata): resp.append(probe) + self.logger.debug("Sending probe list to %s" % client) return lxml.etree.tostring(resp, xml_declaration=False).decode('UTF-8') except: @@ -1058,7 +1133,7 @@ class BaseCore(object): # that's created for RecvProbeData doesn't get cached. # I.e., the next metadata object that's built, after probe # data is processed, is cached. - self.metadata_cache.expire(client) + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) try: xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) @@ -1255,9 +1330,14 @@ class BaseCore(object): self.logger.info("Core: debug = %s" % debug) levels = self._loglevels[self.debug_flag] for handler in logging.root.handlers: - level = levels.get(handler.name, levels['default']) - self.logger.debug("Setting %s log handler to %s" % - (handler.name, logging.getLevelName(level))) + try: + level = levels.get(handler.name, levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.name, logging.getLevelName(level))) + except AttributeError: + level = levels['default'] + self.logger.debug("Setting unknown log handler %s to %s" % + (handler, logging.getLevelName(level))) handler.setLevel(level) return self.debug_flag diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index e430e3160..52c3906fa 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -237,7 +237,8 @@ class FileMonitor(Debuggable): except: # pylint: disable=W0702 err = sys.exc_info()[1] LOGGER.error("Error in handling of event %s for %s: %s" % - (event.code2str(), event.filename, err)) + (event.code2str(), event.filename, err), + exc_info=1) def handle_event_set(self, lock=None): """ Handle all pending events. diff --git a/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py index 0ee204abe..a03d2c919 100644 --- a/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from django.conf.urls.defaults import * +from Bcfg2.Reporting.Compat.django_urls import * from django.contrib.auth.decorators import login_required from django.core.urlresolvers import reverse from django.views.generic.create_update import create_object, update_object, delete_object diff --git a/src/lib/Bcfg2/Server/Hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/urls.py index 01fe97d4f..4a0c33f98 100644 --- a/src/lib/Bcfg2/Server/Hostbase/urls.py +++ b/src/lib/Bcfg2/Server/Hostbase/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls.defaults import * +from Bcfg2.Reporting.Compat.django_urls import * from django.conf import settings from django.views.generic.simple import direct_to_template from django.contrib import admin diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 7c3b2d9cc..f028e225e 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -143,10 +143,11 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_xml(os.path.join(self.metadata.data, "groups.xml"), self.metadata.groups_xml.data, "metadata") - if self.has_all_xincludes("clients.xml"): - self.check_xml(os.path.join(self.metadata.data, "clients.xml"), - self.metadata.clients_xml.data, - "metadata") + if hasattr(self.metadata, "clients_xml"): + if self.has_all_xincludes("clients.xml"): + self.check_xml(os.path.join(self.metadata.data, "clients.xml"), + self.metadata.clients_xml.data, + "metadata") def check_cfg(self): """ Check Cfg files and ``info.xml`` files for required diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index 7edeb8a49..1ecb6da42 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -37,6 +37,12 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): err = sys.exc_info()[1] self.LintError("genshi-syntax-error", "Genshi syntax error: %s" % err) + except: + etype, err = sys.exc_info()[:2] + self.LintError( + "genshi-syntax-error", + "Unexpected Genshi error on %s: %s: %s" % + (entry.name, etype.__name__, err)) def check_tgenshi(self): """ Check templates in TGenshi for syntax errors. """ diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 6ffdd33a0..e49779a10 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -76,7 +76,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): permissions=dict(name=is_filename, owner=is_username, group=is_username, mode=is_octal_mode), vcs=dict(vcstype=lambda v: (v != 'Path' and - hasattr(Bcfg2.Client.Tools.VCS, + hasattr(Bcfg2.Client.Tools.VCS.VCS, "Install%s" % v)), revision=None, sourceurl=None)), Service={"__any__": dict(name=None), diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py new file mode 100644 index 000000000..fca9d14a9 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -0,0 +1,75 @@ +""" Check for templated scripts or executables. """ + +import os +import stat +import Bcfg2.Server.Lint +from Bcfg2.Compat import any # pylint: disable=W0622 +from Bcfg2.Server.Plugin import DEFAULT_FILE_METADATA +from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenshiGenerator import \ + CfgEncryptedGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedCheetahGenerator import \ + CfgEncryptedCheetahGenerator + + +class TemplateAbuse(Bcfg2.Server.Lint.ServerPlugin): + """ Check for templated scripts or executables. """ + templates = [CfgGenshiGenerator, CfgCheetahGenerator, + CfgEncryptedGenshiGenerator, CfgEncryptedCheetahGenerator] + extensions = [".pl", ".py", ".sh", ".rb"] + + def Run(self): + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if (self.HandlesFile(entry.name) and + any(isinstance(entry, t) for t in self.templates)): + self.check_template(entryset, entry) + + @classmethod + def Errors(cls): + return {"templated-script": "warning", + "templated-executable": "warning"} + + def check_template(self, entryset, entry): + """ Check a template to see if it's a script or an executable. """ + # first, check for a known script extension + ext = os.path.splitext(entryset.path)[1] + if ext in self.extensions: + self.LintError("templated-script", + "Templated script found: %s\n" + "File has a known script extension: %s\n" + "Template a config file for the script instead" % + (entry.name, ext)) + return + + # next, check for a shebang line + firstline = open(entry.name).readline() + if firstline.startswith("#!"): + self.LintError("templated-script", + "Templated script found: %s\n" + "File starts with a shebang: %s\n" + "Template a config file for the script instead" % + (entry.name, firstline)) + return + + # finally, check for executable permissions in info.xml + for entry in entryset.entries.values(): + if isinstance(entry, CfgInfoXML): + for pinfo in entry.infoxml.pnode.data.xpath("//FileInfo"): + try: + mode = int(pinfo.get("mode", + DEFAULT_FILE_METADATA['mode']), 8) + except ValueError: + # LintError will be produced by RequiredAttrs plugin + self.logger.warning("Non-octal mode: %s" % mode) + continue + if mode & stat.S_IXUSR != 0: + self.LintError( + "templated-executable", + "Templated executable found: %s\n" + "Template a config file for the executable instead" + % entry.name) + return diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 09f3f3d25..3efcc890d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -47,6 +47,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Decisions/*.xml": "decisions.xsd", "Packages/sources.xml": "packages.xsd", "GroupPatterns/config.xml": "grouppatterns.xsd", + "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", @@ -83,6 +84,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "xml-failed-to-parse": "error", "xml-failed-to-read": "error", "xml-failed-to-verify": "error", + "xinclude-does-not-exist": "error", "input-output-error": "error"} def check_properties(self): @@ -106,9 +108,17 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): :type filename: string :returns: lxml.etree._ElementTree - the parsed data""" try: - return lxml.etree.parse(filename) - except SyntaxError: - lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) + xdata = lxml.etree.parse(filename) + if self.files is None: + self._expand_wildcard_xincludes(xdata) + xdata.xinclude() + return xdata + except (lxml.etree.XIncludeError, SyntaxError): + cmd = ["xmllint", "--noout"] + if self.files is None: + cmd.append("--xinclude") + cmd.append(filename) + lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) self.LintError("xml-failed-to-parse", "%s fails to parse:\n%s" % (filename, lint.communicate()[0])) @@ -119,6 +129,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def _expand_wildcard_xincludes(self, xdata): + """ a lightweight version of + :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """ + xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE + for el in xdata.findall('//' + xinclude): + name = el.get("href") + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name) + + # expand globs in xinclude, a bcfg2-specific extension + extras = glob.glob(fpath) + if not extras: + msg = "%s: %s does not exist, skipping: %s" % \ + (xdata.docinfo.URL, name, self.RenderXML(el)) + if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): + self.logger.debug(msg) + else: + self.LintError("xinclude-does-not-exist", msg) + + parent = el.getparent() + parent.remove(el) + for extra in extras: + if extra != xdata.docinfo.URL: + lxml.etree.SubElement(parent, xinclude, href=extra) + def validate(self, filename, schemafile, schema=None): """ Validate a file against the given schema. @@ -140,6 +177,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if not schema: return False datafile = self.parse(filename) + if not datafile: + return False if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py new file mode 100644 index 000000000..1f55962eb --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -0,0 +1,72 @@ +"""Ensure that all JSON files in the Bcfg2 repository are +valid. Currently, the only plugins that uses JSON are Ohai and +Properties.""" + +import os +import sys +import glob +import fnmatch +import Bcfg2.Server.Lint + +try: + import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): + import simplejson as json + + +class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): + """Ensure that all JSON files in the Bcfg2 repository are + valid. Currently, the only plugins that uses JSON are Ohai and + Properties. """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + + #: A list of file globs that give the path to JSON files. The + #: globs are extended :mod:`fnmatch` globs that also support + #: ``**``, which matches any number of any characters, + #: including forward slashes. + self.globs = ["Properties/*.json", "Ohai/*.json"] + self.files = self.get_files() + + def Run(self): + for path in self.files: + self.logger.debug("Validating JSON in %s" % path) + try: + json.load(open(path)) + except ValueError: + self.LintError("json-failed-to-parse", + "%s does not contain valid JSON: %s" % + (path, sys.exc_info()[1])) + + @classmethod + def Errors(cls): + return {"json-failed-to-parse": "error"} + + def get_files(self): + """Return a list of all JSON files to validate, based on + :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """ + if self.files is not None: + listfiles = lambda p: fnmatch.filter(self.files, + os.path.join('*', p)) + else: + listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], + p)) + + rv = [] + for path in self.globs: + if '/**/' in path: + if self.files is not None: + rv.extend(listfiles(path)) + else: # self.files is None + fpath, fname = path.split('/**/') + for root, _, files in \ + os.walk(os.path.join(self.config['repo'], + fpath)): + rv.extend([os.path.join(root, f) + for f in files if f == fname]) + else: + rv.extend(listfiles(path)) + return rv diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 81fba7092..6d41bbcbb 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -2,14 +2,114 @@ :mod:`Bcfg2.Server.BuiltinCore` that uses the Python :mod:`multiprocessing` library to offload work to multiple child processes. As such, it requires Python 2.6+. + +The parent communicates with the children over +:class:`multiprocessing.Queue` objects via a +:class:`Bcfg2.Server.MultiprocessingCore.RPCQueue` object. + +A method being called via the RPCQueue must be exposed by the child by +decorating it with :func:`Bcfg2.Server.Core.exposed`. """ +import time import threading import lxml.etree import multiprocessing -from Bcfg2.Compat import Queue +import Bcfg2.Server.Plugin +from itertools import cycle +from Bcfg2.Cache import Cache +from Bcfg2.Compat import Empty, wraps from Bcfg2.Server.Core import BaseCore, exposed from Bcfg2.Server.BuiltinCore import Core as BuiltinCore +from multiprocessing.connection import Listener, Client + + +class DispatchingCache(Cache, Bcfg2.Server.Plugin.Debuggable): + """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates + cache expiration events to child nodes. """ + + #: The method to send over the pipe to expire the cache + method = "expire_metadata_cache" + + def __init__(self, *args, **kwargs): + self.rpc_q = kwargs.pop("queue") + Bcfg2.Server.Plugin.Debuggable.__init__(self) + Cache.__init__(self, *args, **kwargs) + + def expire(self, key=None): + self.rpc_q.publish(self.method, args=[key]) + Cache.expire(self, key=key) + + +class RPCQueue(Bcfg2.Server.Plugin.Debuggable): + """ An implementation of a :class:`multiprocessing.Queue` designed + for several additional use patterns: + + * Random-access reads, based on a key that identifies the data; + * Publish-subscribe, where a datum is sent to all hosts. + + The subscribers can deal with this as a normal Queue with no + special handling. + """ + poll_wait = 3.0 + + def __init__(self): + Bcfg2.Server.Plugin.Debuggable.__init__(self) + self._terminate = threading.Event() + self._queues = dict() + self._listeners = [] + + def add_subscriber(self, name): + """ Add a subscriber to the queue. This returns the + :class:`multiprocessing.Queue` object that the subscriber + should read from. """ + self._queues[name] = multiprocessing.Queue() + return self._queues[name] + + def publish(self, method, args=None, kwargs=None): + """ Publish an RPC call to the queue for consumption by all + subscribers. """ + for queue in self._queues.values(): + queue.put((None, (method, args or [], kwargs or dict()))) + + def rpc(self, dest, method, args=None, kwargs=None): + """ Make an RPC call to the named subscriber, expecting a + response. This opens a + :class:`multiprocessing.connection.Listener` and passes the + Listener address to the child as part of the RPC call, so that + the child can connect to the Listener to submit its results. + """ + listener = Listener() + self.logger.debug("Created new RPC listener at %s" % listener.address) + self._listeners.append(listener) + try: + self._queues[dest].put((listener.address, + (method, args or [], kwargs or dict()))) + conn = listener.accept() + try: + while not self._terminate.is_set(): + if conn.poll(self.poll_wait): + return conn.recv() + finally: + conn.close() + finally: + listener.close() + self._listeners.remove(listener) + + def close(self): + """ Close queues and connections. """ + self._terminate.set() + self.logger.debug("Closing RPC queues") + for name, queue in self._queues.items(): + self.logger.debug("Closing RPC queue to %s" % name) + queue.close() + + # close any listeners that are waiting for connections + self.logger.debug("Closing RPC connections") + for listener in self._listeners: + self.logger.debug("Closing RPC connection at %s" % + listener.address) + listener.close() class DualEvent(object): @@ -60,68 +160,153 @@ class ChildCore(BaseCore): those, though, if the pipe communication "protocol" were made more robust. """ - #: How long to wait while polling for new clients to build. This - #: doesn't affect the speed with which a client is built, but + #: How long to wait while polling for new RPC commands. This + #: doesn't affect the speed with which a command is processed, but #: setting it too high will result in longer shutdown times, since #: we only check for the termination event from the main process #: every ``poll_wait`` seconds. - poll_wait = 5.0 + poll_wait = 3.0 - def __init__(self, setup, pipe, terminate): + def __init__(self, name, setup, rpc_q, terminate): """ + :param name: The name of this child + :type name: string :param setup: A Bcfg2 options dict :type setup: Bcfg2.Options.OptionParser - :param pipe: The pipe to which client hostnames are added for - ChildCore objects to build configurations, and to - which client configurations are added after - having been built by ChildCore objects. - :type pipe: multiprocessing.Pipe + :param read_q: The queue the child will read from for RPC + communications from the parent process. + :type read_q: multiprocessing.Queue + :param write_q: The queue the child will write the results of + RPC calls to. + :type write_q: multiprocessing.Queue :param terminate: An event that flags ChildCore objects to shut themselves down. :type terminate: multiprocessing.Event """ BaseCore.__init__(self, setup) - #: The pipe to which client hostnames are added for ChildCore - #: objects to build configurations, and to which client - #: configurations are added after having been built by - #: ChildCore objects. - self.pipe = pipe + #: The name of this child + self.name = name #: The :class:`multiprocessing.Event` that will be monitored #: to determine when this child should shut down. self.terminate = terminate - def _daemonize(self): - return True + #: The queue used for RPC communication + self.rpc_q = rpc_q + + # override this setting so that the child doesn't try to write + # the pidfile + self.setup['daemon'] = False + + # ensure that the child doesn't start a perflog thread + self.perflog_thread = None + + self._rmi = dict() def _run(self): return True + def _daemonize(self): + return True + + def _dispatch(self, address, data): + """ Method dispatcher used for commands received from + the RPC queue. """ + if address is not None: + # if the key is None, then no response is expected. we + # make the return connection before dispatching the actual + # RPC call so that the parent is blocking for a connection + # as briefly as possible + self.logger.debug("Connecting to parent via %s" % address) + client = Client(address) + method, args, kwargs = data + func = None + rv = None + if "." in method: + if method in self._rmi: + func = self._rmi[method] + else: + self.logger.error("%s: Method %s does not exist" % (self.name, + method)) + elif not hasattr(self, method): + self.logger.error("%s: Method %s does not exist" % (self.name, + method)) + else: # method is not a plugin RMI, and exists + func = getattr(self, method) + if not func.exposed: + self.logger.error("%s: Method %s is not exposed" % (self.name, + method)) + func = None + if func is not None: + self.logger.debug("%s: Calling RPC method %s" % (self.name, + method)) + rv = func(*args, **kwargs) + if address is not None: + # if the key is None, then no response is expected + self.logger.debug("Returning data to parent via %s" % address) + client.send(rv) + def _block(self): - while not self.terminate.isSet(): + self._rmi = self._get_rmi() + while not self.terminate.is_set(): try: - if self.pipe.poll(self.poll_wait): - if not self.metadata.use_database: - # handle FAM events, in case (for instance) the - # client has just been added to clients.xml, or a - # profile has just been asserted. but really, you - # should be using the metadata database if you're - # using this core. - self.fam.handle_events_in_interval(0.1) - client = self.pipe.recv() - self.logger.debug("Building configuration for %s" % client) - config = \ - lxml.etree.tostring(self.BuildConfiguration(client)) - self.logger.debug("Returning configuration for %s to main " - "process" % client) - self.pipe.send(config) - self.logger.debug("Returned configuration for %s to main " - "process" % client) + address, data = self.rpc_q.get(timeout=self.poll_wait) + threadname = "-".join(str(i) for i in data) + rpc_thread = threading.Thread(name=threadname, + target=self._dispatch, + args=[address, data]) + rpc_thread.start() + except Empty: + pass except KeyboardInterrupt: break self.shutdown() + def shutdown(self): + BaseCore.shutdown(self) + self.logger.info("%s: Closing RPC command queue" % self.name) + self.rpc_q.close() + + while len(threading.enumerate()) > 1: + threads = [t for t in threading.enumerate() + if t != threading.current_thread()] + self.logger.info("%s: Waiting for %d thread(s): %s" % + (self.name, len(threads), + [t.name for t in threads])) + time.sleep(1) + self.logger.info("%s: All threads stopped" % self.name) + + def _get_rmi(self): + rmi = dict() + for pname, pinst in self._get_rmi_objects().items(): + for crmi in pinst.__child_rmi__: + if isinstance(crmi, tuple): + mname = crmi[1] + else: + mname = crmi + rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + return rmi + + @exposed + def expire_metadata_cache(self, client=None): + """ Expire the metadata cache for a client """ + self.metadata_cache.expire(client) + + @exposed + def RecvProbeData(self, address, _): + """ Expire the probe cache for a client """ + self.expire_caches_by_type(Bcfg2.Server.Plugin.Probing, + key=self.resolve_client(address, + metadata=False)[0]) + + @exposed + def GetConfig(self, client): + """ Render the configuration for a client """ + self.logger.debug("%s: Building configuration for %s" % + (self.name, client)) + return lxml.etree.tostring(self.BuildConfiguration(client)) + class Core(BuiltinCore): """ A multiprocessing core that delegates building the actual @@ -140,65 +325,163 @@ class Core(BuiltinCore): if setup['children'] is None: setup['children'] = multiprocessing.cpu_count() - #: A dict of child name -> one end of the - #: :class:`multiprocessing.Pipe` object used to communicate - #: with that child. (The child is given the other end of the - #: Pipe.) - self.pipes = dict() - - #: A queue that keeps track of which children are available to - #: render a configuration. A child is popped from the queue - #: when it starts to render a config, then it's pushed back on - #: when it's done. This lets us use a blocking call to - #: :func:`Queue.Queue.get` when waiting for an available - #: child. - self.available_children = Queue(maxsize=self.setup['children']) - - # sigh. multiprocessing was added in py2.6, which is when the - # camelCase methods for threading objects were deprecated in - # favor of the Pythonic under_score methods. So - # multiprocessing.Event *only* has is_set(), while - # threading.Event has *both* isSet() and is_set(). In order - # to make the core work with Python 2.4+, and with both - # multiprocessing and threading Event objects, we just - # monkeypatch self.terminate to have isSet(). + #: The flag that indicates when to stop child threads and + #: processes self.terminate = DualEvent(threading_event=self.terminate) + #: A :class:`Bcfg2.Server.MultiprocessingCore.RPCQueue` object + #: used to send or publish commands to children. + self.rpc_q = RPCQueue() + + self.metadata_cache = DispatchingCache(queue=self.rpc_q) + + #: A list of children that will be cycled through + self._all_children = [] + + #: An iterator that each child will be taken from in sequence, + #: to provide a round-robin distribution of render requests + self.children = None + def _run(self): for cnum in range(self.setup['children']): name = "Child-%s" % cnum - (mainpipe, childpipe) = multiprocessing.Pipe() - self.pipes[name] = mainpipe + self.logger.debug("Starting child %s" % name) - childcore = ChildCore(self.setup, childpipe, self.terminate) + child_q = self.rpc_q.add_subscriber(name) + childcore = ChildCore(name, self.setup, child_q, self.terminate) child = multiprocessing.Process(target=childcore.run, name=name) child.start() self.logger.debug("Child %s started with PID %s" % (name, child.pid)) - self.available_children.put(name) + self._all_children.append(name) + self.logger.debug("Started %s children: %s" % (len(self._all_children), + self._all_children)) + self.children = cycle(self._all_children) return BuiltinCore._run(self) def shutdown(self): BuiltinCore.shutdown(self) - for child in multiprocessing.active_children(): - self.logger.debug("Shutting down child %s" % child.name) - child.join(self.shutdown_timeout) - if child.is_alive(): + self.logger.info("Closing RPC command queues") + self.rpc_q.close() + + def term_children(): + """ Terminate all remaining multiprocessing children. """ + for child in multiprocessing.active_children(): self.logger.error("Waited %s seconds to shut down %s, " "terminating" % (self.shutdown_timeout, child.name)) child.terminate() - else: - self.logger.debug("Child %s shut down" % child.name) - self.logger.debug("All children shut down") + + timer = threading.Timer(self.shutdown_timeout, term_children) + timer.start() + while len(multiprocessing.active_children()): + self.logger.info("Waiting for %s child(ren): %s" % + (len(multiprocessing.active_children()), + [c.name + for c in multiprocessing.active_children()])) + time.sleep(1) + timer.cancel() + self.logger.info("All children shut down") + + while len(threading.enumerate()) > 1: + threads = [t for t in threading.enumerate() + if t != threading.current_thread()] + self.logger.info("Waiting for %s thread(s): %s" % + (len(threads), [t.name for t in threads])) + time.sleep(1) + self.logger.info("Shutdown complete") + + def _get_rmi(self): + child_rmi = dict() + for pname, pinst in self._get_rmi_objects().items(): + for crmi in pinst.__child_rmi__: + if isinstance(crmi, tuple): + parentname, childname = crmi + else: + parentname = childname = crmi + child_rmi["%s.%s" % (pname, parentname)] = \ + "%s.%s" % (pname, childname) + + rmi = BuiltinCore._get_rmi(self) + for method in rmi.keys(): + if method in child_rmi: + rmi[method] = self._child_rmi_wrapper(method, + rmi[method], + child_rmi[method]) + return rmi + + def _child_rmi_wrapper(self, method, parent_rmi, child_rmi): + """ Returns a callable that dispatches a call to the given + child RMI to child processes, and calls the parent RMI locally + (i.e., in the parent process). """ + @wraps(parent_rmi) + def inner(*args, **kwargs): + """ Function that dispatches an RMI call to child + processes and to the (original) parent function. """ + self.logger.debug("Dispatching RMI call to %s to children: %s" % + (method, child_rmi)) + self.rpc_q.publish(child_rmi, args=args, kwargs=kwargs) + return parent_rmi(*args, **kwargs) + + return inner + + @exposed + def set_debug(self, address, debug): + self.rpc_q.set_debug(debug) + self.rpc_q.publish("set_debug", args=[address, debug]) + self.metadata_cache.set_debug(debug) + return BuiltinCore.set_debug(self, address, debug) + + @exposed + def RecvProbeData(self, address, probedata): + rv = BuiltinCore.RecvProbeData(self, address, probedata) + # we don't want the children to actually process probe data, + # so we don't send the data, just the fact that we got some. + self.rpc_q.publish("RecvProbeData", args=[address, None]) + return rv @exposed def GetConfig(self, address): client = self.resolve_client(address)[0] - childname = self.available_children.get() - self.logger.debug("Building configuration on child %s" % childname) - pipe = self.pipes[childname] - pipe.send(client) - config = pipe.recv() - self.available_children.put_nowait(childname) - return config + childname = self.children.next() + self.logger.debug("Building configuration for %s on %s" % (client, + childname)) + return self.rpc_q.rpc(childname, "GetConfig", args=[client]) + + @exposed + def get_statistics(self, address): + stats = dict() + + def _aggregate_statistics(newstats, prefix=None): + """ Aggregate a set of statistics from a child or parent + server core. This adds the statistics to the overall + statistics dict (optionally prepending a prefix, such as + "Child-1", to uniquely identify this set of statistics), + and aggregates it with the set of running totals that are + kept from all cores. """ + for statname, vals in newstats.items(): + if statname.startswith("ChildCore:"): + statname = statname[5:] + if prefix: + prettyname = "%s:%s" % (prefix, statname) + else: + prettyname = statname + stats[prettyname] = vals + totalname = "Total:%s" % statname + if totalname not in stats: + stats[totalname] = vals + else: + newmin = min(stats[totalname][0], vals[0]) + newmax = max(stats[totalname][1], vals[1]) + newcount = stats[totalname][3] + vals[3] + newmean = ((stats[totalname][2] * stats[totalname][3]) + + (vals[2] * vals[3])) / newcount + stats[totalname] = (newmin, newmax, newmean, newcount) + + stats = dict() + for childname in self._all_children: + _aggregate_statistics( + self.rpc_q.rpc(childname, "get_statistics", args=[address]), + prefix=childname) + _aggregate_statistics(BuiltinCore.get_statistics(self, address)) + return stats diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index ecd970b54..03feceb6f 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -12,6 +12,10 @@ class Debuggable(object): #: List of names of methods to be exposed as XML-RPC functions __rmi__ = ['toggle_debug', 'set_debug'] + #: How exposed XML-RPC functions should be dispatched to child + #: processes. + __child_rmi__ = __rmi__[:] + def __init__(self, name=None): """ :param name: The name of the logger object to get. If none is @@ -34,9 +38,6 @@ class Debuggable(object): :returns: bool - The new value of the debug flag """ self.debug_flag = debug - self.debug_log("%s: debug = %s" % (self.__class__.__name__, - self.debug_flag), - flag=True) return debug def toggle_debug(self): @@ -87,9 +88,27 @@ class Plugin(Debuggable): #: alphabetically by their name. sort_order = 500 + #: Whether or not to automatically create a data directory for + #: this plugin + create = True + #: List of names of methods to be exposed as XML-RPC functions __rmi__ = Debuggable.__rmi__ + #: How exposed XML-RPC functions should be dispatched to child + #: processes, if :mod:`Bcfg2.Server.MultiprocessingCore` is in + #: use. Items ``__child_rmi__`` can either be strings (in which + #: case the same function is called on child processes as on the + #: parent) or 2-tuples, in which case the first element is the + #: name of the RPC function called on the parent process, and the + #: second element is the name of the function to call on child + #: processes. Functions that are not listed in the list will not + #: be dispatched to child processes, i.e., they will only be + #: called on the parent. A function must be listed in ``__rmi__`` + #: in order to be exposed; functions listed in ``_child_rmi__`` + #: but not ``__rmi__`` will be ignored. + __child_rmi__ = Debuggable.__child_rmi__ + def __init__(self, core, datastore): """ :param core: The Bcfg2.Server.Core initializing the plugin @@ -107,7 +126,7 @@ class Plugin(Debuggable): self.Entries = {} self.core = core self.data = os.path.join(datastore, self.name) - if not os.path.exists(self.data): + if self.create and not os.path.exists(self.data): self.logger.warning("%s: %s does not exist, creating" % (self.name, self.data)) os.makedirs(self.data) @@ -132,6 +151,8 @@ class Plugin(Debuggable): self.running = False def set_debug(self, debug): + self.debug_log("%s: debug = %s" % (self.name, self.debug_flag), + flag=True) for entry in self.Entries.values(): if isinstance(entry, Debuggable): entry.set_debug(debug) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 81dc1d736..3e7d68cd8 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -16,7 +16,7 @@ from Bcfg2.Compat import CmpMixin, wraps from Bcfg2.Server.Plugin.base import Debuggable, Plugin from Bcfg2.Server.Plugin.interfaces import Generator from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ - PluginExecutionError + PluginExecutionError, PluginInitError try: import django # pylint: disable=W0611 @@ -131,6 +131,19 @@ class DatabaseBacked(Plugin): #: conform to the possible values that function can handle. option = "use_database" + def __init__(self, core, datastore): + Plugin.__init__(self, core, datastore) + use_db = self.core.setup.cfp.getboolean(self.section, + self.option, + default=False) + if use_db and not HAS_DJANGO: + raise PluginInitError("%s.%s is True but Django not found" % + (self.section, self.option)) + elif use_db and not self.core.database_available: + raise PluginInitError("%s.%s is True but the database is " + "unavailable due to prior errors" % + (self.section, self.option)) + def _section(self): """ The section to look in for :attr:`DatabaseBacked.option` """ @@ -146,10 +159,7 @@ class DatabaseBacked(Plugin): default=False) if use_db and HAS_DJANGO and self.core.database_available: return True - elif not use_db: - return False else: - self.logger.error("%s is true but django not found" % self.option) return False @property @@ -555,16 +565,12 @@ class XMLFileBacked(FileBacked): xdata = self.xdata.getroottree() else: xdata = lxml.etree.parse(fname) - included = [el for el in xdata.findall('//' + xinclude)] - for el in included: + for el in xdata.findall('//' + xinclude): name = el.get("href") if name.startswith("/"): fpath = name else: - if fname: - rel = fname - else: - rel = self.name + rel = fname or self.name fpath = os.path.join(os.path.dirname(rel), name) # expand globs in xinclude, a bcfg2-specific extension @@ -579,12 +585,13 @@ class XMLFileBacked(FileBacked): parent = el.getparent() parent.remove(el) for extra in extras: - if extra != self.name and extra not in self.extras: - self.extras.append(extra) + if extra != self.name: lxml.etree.SubElement(parent, xinclude, href=extra) - self._follow_xincludes(fname=extra) - if extra not in self.extra_monitors: - self.add_monitor(extra) + if extra not in self.extras: + self.extras.append(extra) + self._follow_xincludes(fname=extra) + if extra not in self.extra_monitors: + self.add_monitor(extra) def Index(self): self.xdata = lxml.etree.XML(self.data, base_url=self.name, @@ -606,15 +613,16 @@ class XMLFileBacked(FileBacked): def add_monitor(self, fpath): """ Add a FAM monitor to a file that has been XIncluded. This - is only done if the constructor got both a ``fam`` object and - ``should_monitor`` set to True. + is only done if the constructor got a ``fam`` object, + regardless of whether ``should_monitor`` is set to True (i.e., + whether or not the base file is monitored). :param fpath: The full path to the file to monitor :type fpath: string :returns: None """ self.extra_monitors.append(fpath) - if self.fam and self.should_monitor: + if self.fam: self.fam.AddMonitor(fpath, self) def __iter__(self): @@ -832,15 +840,10 @@ class XMLSrc(XMLFileBacked): def HandleEvent(self, _=None): """Read file upon update.""" - try: - data = open(self.name).read() - except IOError: - msg = "Failed to read file %s: %s" % (self.name, sys.exc_info()[1]) - self.logger.error(msg) - raise PluginExecutionError(msg) self.items = {} try: - xdata = lxml.etree.XML(data, parser=Bcfg2.Server.XMLParser) + xdata = lxml.etree.parse(self.name, + parser=Bcfg2.Server.XMLParser).getroot() except lxml.etree.XMLSyntaxError: msg = "Failed to parse file %s: %s" % (self.name, sys.exc_info()[1]) @@ -857,8 +860,6 @@ class XMLSrc(XMLFileBacked): self.logger.error(msg) raise PluginExecutionError(msg) - del xdata, data - def Cache(self, metadata): """Build a package dict for a given host.""" if self.cache is None or self.cache[0] != metadata: diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 0fd711be9..33f6d338c 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -220,10 +220,32 @@ class Connector(object): def get_additional_groups(self, metadata): # pylint: disable=W0613 """ Return a list of additional groups for the given client. + Each group can be either the name of a group (a string), or a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that defines other data besides just the name. Note that you + cannot return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that clobbers a group defined by another plugin; the original + group will be used instead. For instance, assume the + following in ``Metadata/groups.xml``: + + .. code-block:: xml + + <Groups> + ... + <Group name="foo" public="false"/> + </Groups> + + You could not subsequently return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + with ``public=True``; a warning would be issued, and the + original (non-public) ``foo`` group would be used. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of strings + :return: list of strings or + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` + objects. """ return list() @@ -286,6 +308,8 @@ class Statistics(Plugin): you should avoid using Statistics and use :class:`ThreadedStatistics` instead.""" + create = False + def process_statistics(self, client, xdata): """ Process the given XML statistics data for the specified client. @@ -526,6 +550,8 @@ class GoalValidator(object): class Version(Plugin): """ Version plugins interact with various version control systems. """ + create = False + #: The path to the VCS metadata file or directory, relative to the #: base of the Bcfg2 repository. E.g., for Subversion this would #: be ".svn" @@ -594,3 +620,22 @@ class ClientRunHooks(object): :returns: None """ pass + + +class Caching(object): + """ A plugin that caches more than just the data received from the + FAM. This presents a unified interface to clear the cache. """ + + def expire_cache(self, key=None): + """ Expire the cache associated with the given key. + + :param key: The key to expire the cache for. Because cache + implementations vary tremendously between plugins, + this could be any number of things, but generally + a hostname. It also may or may not be possible to + expire the cache for a single host; this interface + does not require any guarantee about that. + :type key: varies + :returns: None + """ + raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py new file mode 100644 index 000000000..147f37fbf --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -0,0 +1,217 @@ +""" Query tags from AWS via boto, optionally setting group membership """ + +import os +import re +import sys +import Bcfg2.Server.Lint +import Bcfg2.Server.Plugin +from boto import connect_ec2 +from Bcfg2.Cache import Cache +from Bcfg2.Compat import ConfigParser + + +class NoInstanceFound(Exception): + """ Raised when there's no AWS instance for a given hostname """ + + +class AWSTagPattern(object): + """ Handler for a single Tag entry """ + + def __init__(self, name, value, groups): + self.name = re.compile(name) + if value is not None: + self.value = re.compile(value) + else: + self.value = value + self.groups = groups + + def get_groups(self, tags): + """ Get groups that apply to the given tag set """ + for key, value in tags.items(): + name_match = self.name.search(key) + if name_match: + if self.value is not None: + value_match = self.value.search(value) + if value_match: + return self._munge_groups(value_match) + else: + return self._munge_groups(name_match) + break + return [] + + def _munge_groups(self, match): + """ Replace backreferences (``$1``, ``$2``) in Group tags with + their values in the regex. """ + rv = [] + sub = match.groups() + for group in self.groups: + newg = group + for idx in range(len(sub)): + newg = newg.replace('$%s' % (idx + 1), sub[idx]) + rv.append(newg) + return rv + + def __str__(self): + if self.value: + return "%s: %s=%s: %s" % (self.__class__.__name__, self.name, + self.value, self.groups) + else: + return "%s: %s: %s" % (self.__class__.__name__, self.name, + self.groups) + + +class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of AWSTags config.xml """ + __identifier__ = None + create = 'AWSTags' + + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + should_monitor=True) + self.core = core + self.tags = [] + + def Index(self): + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() + self.tags = [] + for entry in self.xdata.xpath('//Tag'): + try: + groups = [g.text for g in entry.findall('Group')] + self.tags.append(AWSTagPattern(entry.get("name"), + entry.get("value"), + groups)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to initialize pattern %s: " + "%s" % (entry.get("name"), + sys.exc_info()[1])) + + def get_groups(self, hostname, tags): + """ return a list of groups that should be added to the given + client based on patterns that match the hostname """ + ret = [] + for pattern in self.tags: + try: + ret.extend(pattern.get_groups(tags)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to process pattern %s for " + "%s" % (pattern, hostname), + exc_info=1) + return ret + + +class AWSTags(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ Query tags from AWS via boto, optionally setting group membership """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + key_id = self.core.setup.cfp.get("awstags", "access_key_id") + secret_key = self.core.setup.cfp.get("awstags", + "secret_access_key") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError( + "AWSTags is not configured in bcfg2.conf: %s" % err) + self.debug_log("%s: Connecting to EC2" % self.name) + self._ec2 = connect_ec2(aws_access_key_id=key_id, + aws_secret_access_key=secret_key) + self._tagcache = Cache() + try: + self._keep_cache = self.core.setup.cfp.getboolean("awstags", + "cache") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._keep_cache = True + + self.config = PatternFile(os.path.join(self.data, 'config.xml'), + core=core) + + def _load_instance(self, hostname): + """ Load an instance from EC2 whose private DNS name matches + the given hostname """ + self.debug_log("AWSTags: Loading instance with private-dns-name=%s" % + hostname) + filters = {'private-dns-name': hostname} + reservations = self._ec2.get_all_instances(filters=filters) + if reservations: + res = reservations[0] + if res.instances: + return res.instances[0] + raise NoInstanceFound( + "AWSTags: No instance found with private-dns-name=%s" % + hostname) + + def _get_tags_from_ec2(self, hostname): + """ Get tags for the given host from EC2. This does not use + the local caching layer. """ + self.debug_log("AWSTags: Getting tags for %s from AWS" % + hostname) + try: + return self._load_instance(hostname).tags + except NoInstanceFound: + self.debug_log(sys.exc_info()[1]) + return dict() + + def get_tags(self, metadata): + """ Get tags for the given host. This caches the tags locally + if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is + true. """ + if not self._keep_cache: + return self._get_tags_from_ec2(metadata) + + if metadata.hostname not in self._tagcache: + self._tagcache[metadata.hostname] = \ + self._get_tags_from_ec2(metadata.hostname) + return self._tagcache[metadata.hostname] + + def expire_cache(self, key=None): + self._tagcache.expire(key=key) + + def start_client_run(self, metadata): + self.expire_cache(key=metadata.hostname) + + def get_additional_data(self, metadata): + return self.get_tags(metadata) + + def get_additional_groups(self, metadata): + return self.config.get_groups(metadata.hostname, + self.get_tags(metadata)) + + +class AWSTagsLint(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + <server-plugins-connectors-awstags>` patterns for validity. """ + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index eef176cca..58f8f4430 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -38,9 +38,9 @@ if HAS_GENSHI: Bcfg2.Server.Plugin.StructFile): """ Representation of a Genshi-templated bundle XML file """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific, encoding, fam=None): TemplateFile.__init__(self, name, specific, encoding) - Bcfg2.Server.Plugin.StructFile.__init__(self, name) + Bcfg2.Server.Plugin.StructFile.__init__(self, name, fam=fam) self.logger = logging.getLogger(name) def get_xml_value(self, metadata): @@ -53,9 +53,9 @@ if HAS_GENSHI: stream = self.template.generate( metadata=metadata, repo=SETUP['repo']).filter(removecomment) - data = lxml.etree.XML(stream.render('xml', - strip_whitespace=False), - parser=Bcfg2.Server.XMLParser) + data = lxml.etree.XML( + stream.render('xml', strip_whitespace=False).encode(), + parser=Bcfg2.Server.XMLParser) bundlename = os.path.splitext(os.path.basename(self.name))[0] bundle = lxml.etree.Element('Bundle', name=bundlename) for item in self.Match(metadata, data): @@ -106,13 +106,14 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, nsmap['py'] == 'http://genshi.edgewall.org/')): if HAS_GENSHI: spec = Bcfg2.Server.Plugin.Specificity() - return BundleTemplateFile(name, spec, self.encoding) + return BundleTemplateFile(name, spec, self.encoding, + fam=self.core.fam) else: raise Bcfg2.Server.Plugin.PluginExecutionError("Genshi not " "available: %s" % name) else: - return BundleFile(name, self.fam) + return BundleFile(name, fam=self.fam) def BuildStructures(self, metadata): """Build all structures for client (metadata).""" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index 824d01023..41d5588e4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -50,27 +50,36 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): spec = self.XMLMatch(metadata) rv = [] for allow in spec.findall("Allow"): - params = '' + options = [] if allow.find("Params") is not None: - params = ",".join("=".join(p) - for p in allow.find("Params").attrib.items()) + self.logger.warning("Use of <Params> in authorized_keys.xml " + "is deprecated; use <Option> instead") + options.extend("=".join(p) + for p in allow.find("Params").attrib.items()) + + for opt in allow.findall("Option"): + if opt.get("value"): + options.append("%s=%s" % (opt.get("name"), + opt.get("value"))) + else: + options.append(opt.get("name")) pubkey_name = allow.get("from") if pubkey_name: host = allow.get("host") group = allow.get("group") + category = allow.get("category", self.category) if host: key_md = self.core.build_metadata(host) elif group: key_md = ClientMetadata("dummy", group, [group], [], set(), set(), dict(), None, None, None, None) - elif (self.category and - not metadata.group_in_category(self.category)): + elif category and not metadata.group_in_category(category): self.logger.warning("Cfg: %s ignoring Allow from %s: " "No group in category %s" % (metadata.hostname, pubkey_name, - self.category)) + category)) continue else: key_md = metadata @@ -96,6 +105,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): (metadata.hostname, lxml.etree.tostring(allow))) continue - rv.append(" ".join([params, pubkey]).strip()) + rv.append(" ".join([",".join(options), pubkey]).strip()) return "\n".join(rv) get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 3b4703ddb..cf7eae75b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,8 +1,9 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ +import Bcfg2.Server.Plugins.Cfg from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP +from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: from Bcfg2.Encryption import bruteforce_decrypt, EVPError, \ get_algorithm @@ -34,8 +35,10 @@ class CfgEncryptedGenerator(CfgGenerator): return # todo: let the user specify a passphrase by name try: - self.data = bruteforce_decrypt(self.data, setup=SETUP, - algorithm=get_algorithm(SETUP)) + self.data = bruteforce_decrypt( + self.data, + setup=Bcfg2.Server.Plugins.Cfg.SETUP, + algorithm=get_algorithm(Bcfg2.Server.Plugins.Cfg.SETUP)) except EVPError: raise PluginExecutionError("Failed to decrypt %s" % self.name) handle_event.__doc__ = CfgGenerator.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index c7b62f352..e890fdecb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -159,7 +159,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): return specificity # pylint: disable=W0221 - def create_data(self, entry, metadata, return_pair=False): + def create_data(self, entry, metadata): """ Create data for the given entry on the given client :param entry: The abstract entry to create data for. This @@ -167,15 +167,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): :type entry: lxml.etree._Element :param metadata: The client metadata to create data for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param return_pair: Return a tuple of ``(public key, private - key)`` instead of just the private key. - This is used by - :class:`Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.CfgPublicKeyCreator` - to create public keys as requested. - :type return_pair: bool :returns: string - The private key data - :returns: tuple - Tuple of ``(public key, private key)``, if - ``return_pair`` is set to True """ spec = self.XMLMatch(metadata) specificity = self.get_specificity(metadata, spec) @@ -201,11 +193,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): specificity['ext'] = '.crypt' self.write_data(privkey, **specificity) - - if return_pair: - return (pubkey, privkey) - else: - return privkey + return privkey finally: shutil.rmtree(os.path.dirname(filename)) # pylint: enable=W0221 @@ -230,7 +218,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if strict: raise PluginExecutionError(msg) else: - self.logger.warning(msg) + self.logger.info(msg) Index.__doc__ = StructFile.Index.__doc__ def _decrypt(self, element): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py index 6be438462..4bd8690ed 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py @@ -2,7 +2,11 @@ :class:`Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.CfgPrivateKeyCreator` to create SSH keys on the fly. """ +import os +import sys +import tempfile import lxml.etree +from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import StructFile, PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, CFG @@ -27,7 +31,8 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) self.cfg = CFG - __init__.__doc__ = CfgCreator.__init__.__doc__ + self.core = CFG.core + self.cmd = Executor() def create_data(self, entry, metadata): if entry.get("name").endswith(".pub"): @@ -37,25 +42,51 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): "%s: Filename does not end in .pub" % entry.get("name")) - if privkey not in self.cfg.entries: - raise CfgCreationError("Cfg: Could not find Cfg entry for %s " - "(private key for %s)" % (privkey, - self.name)) - eset = self.cfg.entries[privkey] + privkey_entry = lxml.etree.Element("Path", name=privkey) try: + self.core.Bind(privkey_entry, metadata) + except PluginExecutionError: + raise CfgCreationError("Cfg: Could not bind %s (private key for " + "%s): %s" % (privkey, self.name, + sys.exc_info()[1])) + + try: + eset = self.cfg.entries[privkey] creator = eset.best_matching(metadata, eset.get_handlers(metadata, CfgCreator)) + except KeyError: + raise CfgCreationError("Cfg: No private key defined for %s (%s)" % + (self.name, privkey)) except PluginExecutionError: raise CfgCreationError("Cfg: No privkey.xml defined for %s " "(private key for %s)" % (privkey, self.name)) - privkey_entry = lxml.etree.Element("Path", name=privkey) - pubkey = creator.create_data(privkey_entry, metadata, - return_pair=True)[0] - return pubkey - create_data.__doc__ = CfgCreator.create_data.__doc__ + specificity = creator.get_specificity(metadata) + fname = self.get_filename(**specificity) + + # if the private key didn't exist, then creating it may have + # created the private key, too. check for it first. + if os.path.exists(fname): + return open(fname).read() + else: + # generate public key from private key + fd, privfile = tempfile.mkstemp() + try: + os.fdopen(fd, 'w').write(privkey_entry.text) + cmd = ["ssh-keygen", "-y", "-f", privfile] + self.debug_log("Cfg: Extracting SSH public key from %s: %s" % + (privkey, " ".join(cmd))) + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Cfg: Failed to extract public key " + "from %s: %s" % (privkey, + result.error)) + self.write_data(result.stdout, **specificity) + return result.stdout + finally: + os.unlink(privfile) def handle_event(self, event): CfgCreator.handle_event(self, event) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index c6ac9d8dc..c6e2d0acb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -10,6 +10,7 @@ import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.Lint +from fnmatch import fnmatch from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ @@ -35,6 +36,24 @@ SETUP = None #: facility for passing it otherwise. CFG = None +_HANDLERS = [] + + +def handlers(): + """ A list of Cfg handler classes. Loading the handlers must + be done at run-time, not at compile-time, or it causes a + circular import and Bad Things Happen.""" + if not _HANDLERS: + for submodule in walk_packages(path=__path__, prefix=__name__ + "."): + mname = submodule[1].rsplit('.', 1)[-1] + module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, + mname) + hdlr = getattr(module, mname) + if issubclass(hdlr, CfgBaseFileMatcher): + _HANDLERS.append(hdlr) + _HANDLERS.sort(key=operator.attrgetter("__priority__")) + return _HANDLERS + class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, Bcfg2.Server.Plugin.Debuggable): @@ -82,6 +101,8 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, experimental = False def __init__(self, name, specific, encoding): + if not self.__specific__ and not specific: + specific = Bcfg2.Server.Plugin.Specificity(all=True) Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, encoding) Bcfg2.Server.Plugin.Debuggable.__init__(self) @@ -459,7 +480,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry_type, encoding) Bcfg2.Server.Plugin.Debuggable.__init__(self) self.specific = None - self._handlers = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): @@ -468,24 +488,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry.set_debug(debug) return rv - @property - def handlers(self): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if self._handlers is None: - self._handlers = [] - for submodule in walk_packages(path=__path__, - prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if CfgBaseFileMatcher in hdlr.__mro__: - self._handlers.append(hdlr) - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -502,7 +504,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in self.handlers: + for hdlr in handlers(): if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -582,10 +584,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, def bind_entry(self, entry, metadata): self.bind_info_to_entry(entry, metadata) - data = self._generate_data(entry, metadata) - - for fltr in self.get_handlers(metadata, CfgFilter): - data = fltr.modify_data(entry, metadata, data) + data, generator = self._generate_data(entry, metadata) + + if generator is not None: + # apply no filters if the data was created by a CfgCreator + for fltr in self.get_handlers(metadata, CfgFilter): + if fltr.specific <= generator.specific: + # only apply filters that are as specific or more + # specific than the generator used for this entry. + # Note that specificity comparison is backwards in + # this sense, since it's designed to sort from + # most specific to least specific. + data = fltr.modify_data(entry, metadata, data) if SETUP['validate']: try: @@ -694,7 +704,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, :type entry: lxml.etree._Element :param metadata: The client metadata to generate data for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: string - the data for the entry + :returns: tuple of (string, generator) - the data for the + entry and the generator used to generate it (or + None, if data was created) """ try: generator = self.best_matching(metadata, @@ -703,7 +715,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, except PluginExecutionError: # if no creators or generators exist, _create_data() # raises an appropriate exception - return self._create_data(entry, metadata) + return (self._create_data(entry, metadata), None) if entry.get('mode').lower() == 'inherit': # use on-disk permissions @@ -713,7 +725,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry.set('mode', oct_mode(stat.S_IMODE(os.stat(fname).st_mode))) try: - return generator.get_data(entry, metadata) + return (generator.get_data(entry, metadata), generator) except: msg = "Cfg: Error rendering %s: %s" % (entry.get("name"), sys.exc_info()[1]) @@ -888,12 +900,17 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): for basename, entry in list(self.core.plugins['Cfg'].entries.items()): self.check_delta(basename, entry) self.check_pubkey(basename, entry) + self.check_missing_files() + self.check_conflicting_handlers() @classmethod def Errors(cls): return {"cat-file-used": "warning", "diff-file-used": "warning", - "no-pubkey-xml": "warning"} + "no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error", + "multiple-global-handlers": "error"} def check_delta(self, basename, entry): """ check that no .cat or .diff files are in use """ @@ -927,3 +944,74 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("no-pubkey-xml", "%s has no corresponding pubkey.xml at %s" % (basename, pubkey)) + + def _list_path_components(self, path): + """ Get a list of all components of a path. E.g., + ``self._list_path_components("/foo/bar/foobaz")`` would return + ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed + to be in order.""" + rv = [] + remaining, component = os.path.split(path) + while component != '': + rv.append(component) + remaining, component = os.path.split(remaining) + return rv + + def check_conflicting_handlers(self): + """ Check that a single entryset doesn't have multiple + non-specific (i.e., 'all') handlers. """ + cfg = self.core.plugins['Cfg'] + for eset in cfg.entries.values(): + alls = [e for e in eset.entries.values() + if (e.specific.all and + issubclass(e.__class__, CfgGenerator))] + if len(alls) > 1: + self.LintError("multiple-global-handlers", + "%s has multiple global handlers: %s" % + (eset.path, ", ".join(os.path.basename(e.name) + for e in alls))) + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = set() + for hdlr in handlers(): + ignore.update(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + for fname in files: + fpath = os.path.join(root, fname) + # check against the handler ignore patterns and the + # global FAM ignore list + if (not any(fname.endswith("." + i) for i in ignore) and + not any(fnmatch(fpath, p) + for p in self.config['ignore']) and + not any(fnmatch(c, p) + for p in self.config['ignore'] + for c in self._list_path_components(fpath))): + all_files.add(fpath) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index 810b273af..d74c16e8b 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -3,7 +3,9 @@ template to dynamically set additional groups for clients. """ import os import lxml.etree +from threading import local import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Metadata import MetadataGroup try: from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile except ImportError: @@ -35,13 +37,40 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, """ GroupLogic is a connector plugin that lets you use an XML Genshi template to dynamically set additional groups for clients. """ + # perform grouplogic later than other Connector plugins, so it can + # use groups set by them + sort_order = 1000 def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), core.fam) + self._local = local() def get_additional_groups(self, metadata): - return [el.get("name") - for el in self.config.get_xml_value(metadata).findall("Group")] + if not hasattr(self._local, "building"): + # building is a thread-local set that tracks which + # machines GroupLogic is getting additional groups for. + # If a get_additional_groups() is called twice for a + # machine before the first call has completed, the second + # call returns an empty list. This is for infinite + # recursion protection; without this check, it'd be + # impossible to use things like metadata.query.in_group() + # in GroupLogic, since that requires building all + # metadata, which requires running + # GroupLogic.get_additional_groups() for all hosts, which + # requires building all metadata... + self._local.building = set() + if metadata.hostname in self._local.building: + return [] + self._local.building.add(metadata.hostname) + rv = [] + for el in self.config.get_xml_value(metadata).findall("Group"): + if el.get("category"): + rv.append(MetadataGroup(el.get("name"), + category=el.get("category"))) + else: + rv.append(el.get("name")) + self._local.building.discard(metadata.hostname) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index 4f2601f15..3c9b8a459 100644 --- a/src/lib/Bcfg2/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -37,6 +37,7 @@ class Guppy(Bcfg2.Server.Plugin.Plugin): experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable', 'Disable'] + __child_rmi__ = __rmi__[:] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 3b8361c76..f734c98d0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -16,7 +16,9 @@ import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked -from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 +# pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, any, wraps +# pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo try: @@ -40,6 +42,8 @@ if HAS_DJANGO: """ dict-like object to make it easier to access client bcfg2 versions from the database """ + create = False + def __getitem__(self, key): try: return MetadataClientModel.objects.get(hostname=key).version @@ -217,6 +221,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) + self.load_xml() def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" @@ -485,6 +490,7 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" @@ -493,6 +499,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) self.watch_clients = watch_clients @@ -526,21 +533,24 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} - # mappings of predicate -> MetadataGroup object + # mappings of groupname -> [predicates] self.group_membership = dict() self.negated_groups = dict() + # list of group names in document order + self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions(core, datastore) else: self.versions = dict() + self.uuid = {} self.session_cache = {} self.default = None self.pdirty = False self.password = core.setup['password'] self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients), + self.list_clients, self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -648,6 +658,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if attribs is None: attribs = dict() if self._use_db: + if attribs: + msg = "Metadata does not support setting client attributes " +\ + "with use_database enabled" + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: client = MetadataClientModel.objects.get(hostname=client_name) except MetadataClientModel.DoesNotExist: @@ -670,14 +685,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node is None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = 'Unexpected error finding %s "%s"' % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) @@ -747,7 +763,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) def remove_client(self, client_name): - """Remove a bundle.""" + """Remove a client.""" if self._use_db: try: client = MetadataClientModel.objects.get(hostname=client_name) @@ -828,51 +844,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() + def _get_condition(self, element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ + negate = element.get('negate', 'false').lower() == 'true' + pname = element.get("name") + if element.tag == 'Group': + return lambda c, g, _: negate != (pname in g) + elif element.tag == 'Client': + return lambda c, g, _: negate != (pname == c) + + def _get_category_condition(self, grpname): + """ get a predicate that returns False if a client is already + a member of a group in the given group's category, True + otherwise""" + return lambda client, _, categories: \ + bool(self._check_category(client, grpname, categories)) + + def _aggregate_conditions(self, conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ + return lambda client, groups, cats: \ + all(cond(client, groups, cats) for cond in conditions) + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ self.groups = {} - # these three functions must be separate functions in order to - # ensure that the scope is right for the closures they return - def get_condition(element): - """ Return a predicate that returns True if a client meets - the condition specified in the given Group or Client - element """ - negate = element.get('negate', 'false').lower() == 'true' - pname = element.get("name") - if element.tag == 'Group': - return lambda c, g, _: negate != (pname in g) - elif element.tag == 'Client': - return lambda c, g, _: negate != (pname == c) - - def get_category_condition(category, gname): - """ get a predicate that returns False if a client is - already a member of a group in the given category, True - otherwise """ - def in_cat(client, groups, categories): # pylint: disable=W0613 - """ return True if the client is already a member of a - group in the category given in the enclosing function, - False otherwise """ - if category in categories: - if (gname not in self.groups or - client not in self.groups[gname].warned): - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, gname, category, - client, categories[category])) - if gname in self.groups: - self.groups[gname].warned.append(client) - return False - return True - return in_cat - - def aggregate_conditions(conditions): - """ aggregate all conditions on a given group declaration - into a single predicate """ - return lambda client, groups, cats: \ - all(cond(client, groups, cats) for cond in conditions) - # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old @@ -898,6 +897,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() + self.ordered_groups = [] # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is @@ -908,37 +908,47 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): - if ((el.tag != 'Group' and el.tag != 'Client') or - el.getchildren()): + if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren(): continue conditions = [] for parent in el.iterancestors(): - cond = get_condition(parent) + cond = self._get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[aggregate_conditions(conditions)] = \ - self.groups[gname] + self.negated_groups.setdefault(gname, []) + self.negated_groups[gname].append( + self._aggregate_conditions(conditions)) else: if self.groups[gname].category: - conditions.append( - get_category_condition(self.groups[gname].category, - gname)) + conditions.append(self._get_category_condition(gname)) - self.group_membership[aggregate_conditions(conditions)] = \ - self.groups[gname] + if gname not in self.ordered_groups: + self.ordered_groups.append(gname) + self.group_membership.setdefault(gname, []) + self.group_membership[gname].append( + self._aggregate_conditions(conditions)) self.states['groups.xml'] = True + def expire_cache(self, key=None): + self.core.metadata_cache.expire(key) + def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.core.metadata_cache.expire() + self.expire_cache() + + # clear out the list of category suppressions that + # have been warned about, since this may change when + # clients.xml or groups.xml changes. + for group in self.groups.values(): + group.warned = [] event_handler(event) if False not in list(self.states.values()) and self.debug_flag: @@ -976,17 +986,21 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - profiles = [g for g in self.clientgroups[client] - if g in self.groups and self.groups[g].is_profile] - self.logger.info("Changing %s profile from %s to %s" % - (client, profiles, profile)) - self.update_client(client, dict(profile=profile)) - if client in self.clientgroups: - for prof in profiles: - self.clientgroups[client].remove(prof) - self.clientgroups[client].append(profile) + metadata = self.core.build_metadata(client) + if metadata.profile != profile: + self.logger.info("Changing %s profile from %s to %s" % + (client, metadata.profile, profile)) + self.update_client(client, dict(profile=profile)) + if client in self.clientgroups: + if metadata.profile in self.clientgroups[client]: + self.clientgroups[client].remove(metadata.profile) + self.clientgroups[client].append(profile) + else: + self.clientgroups[client] = [profile] else: - self.clientgroups[client] = [profile] + self.logger.debug( + "Ignoring %s request to change profile from %s to %s" + % (client, metadata.profile, profile)) else: self.logger.info("Creating new client: %s, profile %s" % (client, profile)) @@ -1002,8 +1016,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.add_client(client, dict(profile=profile)) self.clients.append(client) self.clientgroups[client] = [profile] - if not self._use_db: - self.clients_xml.write() + if not self._use_db: + self.clients_xml.write() def set_version(self, client, version): """Set version for provided client.""" @@ -1053,11 +1067,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataConsistencyError(err) return self.addresses[address][0] try: - cname = socket.gethostbyaddr(address)[0].lower() + cname = socket.getnameinfo(addresspair, + socket.NI_NAMEREQD)[0].lower() if cname in self.aliases: return self.aliases[cname] return cname - except socket.herror: + except (socket.gaierror, socket.herror): err = "Address resolution error for %s: %s" % (address, sys.exc_info()[1]) self.logger.error(err) @@ -1072,22 +1087,77 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() while numgroups != len(groups): numgroups = len(groups) - for predicate, group in self.group_membership.items(): - if group.name in groups: + newgroups = set() + removegroups = set() + for grpname in self.ordered_groups: + if grpname in groups: continue - if predicate(client, groups, categories): - groups.add(group.name) - if group.category: - categories[group.category] = group.name - for predicate, group in self.negated_groups.items(): - if group.name not in groups: + if any(p(client, groups, categories) + for p in self.group_membership[grpname]): + newgroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + categories[self.groups[grpname].category] = grpname + groups.update(newgroups) + for grpname, predicates in self.negated_groups.items(): + if grpname not in groups: continue - if predicate(client, groups, categories): - groups.remove(group.name) - if group.category: - del categories[group.category] + if any(p(client, groups, categories) for p in predicates): + removegroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + del categories[self.groups[grpname].category] + groups.difference_update(removegroups) return (groups, categories) + def _check_category(self, client, grpname, categories): + """ Determine if the given client is already a member of a + group in the same category as the named group. + + The return value is one of three possibilities: + + * If the client is already a member of a group in the same + category, then False is returned (i.e., the category check + failed); + * If the group is not in any categories, then True is returned; + * If the group is not a member of a group in the category, + then the name of the category is returned. This makes it + easy to add the category to the ClientMetadata object (or + other category list). + + If a pure boolean value is required, you can do + ``bool(self._check_category(...))``. + """ + if grpname not in self.groups: + return True + category = self.groups[grpname].category + if not category: + return True + if category in categories: + if client not in self.groups[grpname].warned: + self.logger.warning("%s: Group %s suppressed by category %s; " + "%s already a member of %s" % + (self.name, grpname, category, + client, categories[category])) + self.groups[grpname].warned.append(client) + return False + return category + + def _check_and_add_category(self, client, grpname, categories): + """ If the client is not a member of a group in the same + category as the named group, then the category is added to + ``categories``. + :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used + to determine if the category can be added. + + If the category check failed, returns False; otherwise, + returns True. """ + rv = self._check_category(client, grpname, categories) + if rv and rv is not True: + categories[rv] = grpname + return True + return rv + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): @@ -1109,39 +1179,37 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Handles setting categories and category suppression. Returns the new profile for the client (which might be unchanged). """ - groups.add(grpname) if grpname in self.groups: - group = self.groups[grpname] - category = group.category - if category: - if category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, grpname, category, - client, categories[category])) - return - categories[category] = grpname - if not profile and group.is_profile: + if not self._check_and_add_category(client, grpname, + categories): + return profile + groups.add(grpname) + if not profile and self.groups[grpname].is_profile: return grpname else: return profile + else: + groups.add(grpname) + return profile if client not in self.clients: pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] + self.debug_log("%s: Adding new client with profile %s" % + (self.name, pgroup)) elif self.default: pgroup = self.default + self.debug_log("%s: Adding new client with default profile %s" + % (self.name, pgroup)) if pgroup: self.set_profile(client, pgroup, (None, None), require_public=False) profile = _add_group(pgroup) else: - msg = "Cannot add new client %s; no default group set" % client - self.logger.error(msg) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError( + "Cannot add new client %s; no default group set" % client) for cgroup in self.clientgroups.get(client, []): if cgroup in groups: @@ -1150,6 +1218,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[cgroup] = MetadataGroup(cgroup) profile = _add_group(cgroup) + # we do this before setting the default because there may be + # groups set in <Client> tags in groups.xml that we want to + # set groups, categories = self._merge_groups(client, groups, categories=categories) @@ -1198,8 +1269,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) - all_groups.update([g.name for g in self.group_membership.values()]) - all_groups.update([g.name for g in self.negated_groups.values()]) + all_groups.update(self.group_membership.keys()) + all_groups.update(self.negated_groups.keys()) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups @@ -1212,7 +1283,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.clients): + for client in self.list_clients(): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1220,34 +1291,33 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.groups.issuperset(groups): + rv.append(client) + return rv def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.bundles.issuperset(bundles): + rv.append(client) + return rv def merge_additional_groups(self, imd, groups): for group in groups: if group in imd.groups: continue - if group in self.groups and self.groups[group].category: - category = self.groups[group].category - if self.groups[group].category in imd.categories: - self.logger.warning("%s: Group %s suppressed by category " - "%s; %s already a member of %s" % - (self.name, group, category, - imd.hostname, - imd.categories[category])) - continue - imd.categories[category] = group + if not self._check_and_add_category(imd.hostname, group, + imd.categories): + continue imd.groups.add(group) - self._merge_groups(imd.hostname, imd.groups, - categories=imd.categories) - + self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) @@ -1395,7 +1465,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, viz_str.extend(self._viz_groups(egroups, bundles, clientmeta)) if key: for category in categories: - viz_str.append('"%s" [label="%s", shape="record", ' + viz_str.append('"%s" [label="%s", shape="trapezium", ' 'style="filled", fillcolor="%s"];' % (category, category, categories[category])) return "\n".join("\t" + s for s in viz_str) @@ -1409,8 +1479,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] - for client in list(self.clients): - if include_client(client): + for client in list(self.list_clients()): + if not include_client(client): continue if client in self.clientgroups: grps = self.clientgroups[client] @@ -1438,9 +1508,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, the graph""" return not clientmeta or bundle in clientmeta.bundles - bundles = list(set(bund.get('name')) - for bund in self.groups_xml.xdata.findall('.//Bundle') - if include_bundle(bund.get('name'))) + bundles = \ + list(set(bund.get('name') + for bund in self.groups_xml.xdata.findall('.//Bundle') + if include_bundle(bund.get('name')))) bundles.sort() return ['"bundle-%s" [ label="%s", shape="septagon"];' % (bundle, bundle) @@ -1586,15 +1657,36 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "client") def duplicate_groups(self): - """ Check for groups that are defined more than once. We - count a group tag as a definition if it a) has profile or - public set; or b) has any children.""" - allgroups = [ - g - for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") - if g.get("profile") or g.get("public") or g.getchildren()] - self.duplicate_entries(allgroups, "group") + """ Check for groups that are defined more than once. There are two + ways this can happen: + + 1. The group is listed twice with contradictory options. + 2. The group is listed with no options *first*, and then with + options later. + + In this context, 'first' refers to the order in which groups + are parsed; see the loop condition below and + _handle_groups_xml_event above for details. """ + groups = dict() + duplicates = dict() + for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): + grpname = grp.get("name") + if grpname in duplicates: + duplicates[grpname].append(grp) + elif set(grp.attrib.keys()).difference(['negate', 'name']): + # group has options + if grpname in groups: + duplicates[grpname] = [grp, groups[grpname]] + else: + groups[grpname] = grp + else: # group has no options + groups[grpname] = grp + for grpname, grps in duplicates.items(): + self.LintError("duplicate-group", + "Group %s is defined multiple times:\n%s" % + (grpname, + "\n".join(self.RenderXML(g) for g in grps))) def duplicate_entries(self, allentries, etype): """ Generic duplicate entry finder. diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index 466665382..8f1d03586 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -24,9 +24,9 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, 'config.xml'), core.fam, should_monitor=True, create=self.name) - self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + self.Entries = { + 'Path': {'/etc/nagiosgen.status': self.createhostconfig, + '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 1ec3cbd60..0853ea993 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin try: import json -except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): import simplejson as json PROBECODE = """#!/bin/sh diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 1736becc7..71128d64c 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -9,6 +9,8 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """POSIXCompat is a goal validator plugin for POSIX entries.""" + create = False + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.GoalValidator.__init__(self) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index b25cb0fc4..39c51f351 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -614,6 +614,10 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.filter_unknown(unknown) return packages, unknown + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, + list.__repr__(self)) + def get_collection_class(source_type): """ Given a source type, determine the class of Collection object diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 332f0bbab..c47e18201 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -88,13 +88,12 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :type event: Bcfg2.Server.FileMonitor.Event :returns: None """ - Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if event and event.filename != self.name: for fpath in self.extras: if fpath == os.path.abspath(event.filename): self.parsed.add(fpath) break - + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if self.loaded: self.logger.info("Reloading Packages plugin") self.pkg_obj.Reload() @@ -111,10 +110,11 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] - for xsource in self.xdata.findall('.//Source'): - source = self.source_from_xml(xsource) - if source is not None: - self.entries.append(source) + if self.loaded: + for xsource in self.xdata.findall('.//Source'): + source = self.source_from_xml(xsource) + if source is not None: + self.entries.append(source) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index bb7caab0d..f038ec9c0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,13 +53,15 @@ The Yum Backend import os import re import sys +import time import copy import errno import socket import logging import lxml.etree -from subprocess import Popen, PIPE import Bcfg2.Server.Plugin +from lockfile import FileLock +from Bcfg2.Utils import Executor # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -85,7 +87,9 @@ try: import yum try: import json - except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 + except (ImportError, AttributeError): import simplejson as json HAS_YUM = True except ImportError: @@ -102,9 +106,6 @@ FL = '{http://linux.duke.edu/metadata/filelists}' PULPSERVER = None PULPCONFIG = None -#: The path to bcfg2-yum-helper -HELPER = None - def _setup_pulp(setup): """ Connect to a Pulp server and pass authentication credentials. @@ -263,6 +264,8 @@ class YumCollection(Collection): .. private-include: _add_gpg_instances, _get_pulp_consumer """ + _helper = None + #: Options that are included in the [packages:yum] section of the #: config but that should not be included in the temporary #: yum.conf we write out @@ -277,18 +280,25 @@ class YumCollection(Collection): debug=debug) self.keypath = os.path.join(self.cachepath, "keys") + #: A :class:`Bcfg2.Utils.Executor` object to use to run + #: external commands + self.cmd = Executor() + if self.use_yum: #: Define a unique cache file for this collection to use #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) #: The path to the server-side config file used when #: resolving packages with the Python yum libraries self.cfgfile = os.path.join(self.cachefile, "yum.conf") - self.write_config() + + if not os.path.exists(self.cachefile): + self.debug_log("Creating common cache %s" % self.cachefile) + os.mkdir(self.cachefile) + if not self.disableMetaData: + self.setup_data() else: self.cachefile = None @@ -309,7 +319,28 @@ class YumCollection(Collection): self.logger.error("Could not create Pulp consumer " "cert directory at %s: %s" % (certdir, err)) - self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) + self.__class__.pulp_cert_set = PulpCertificateSet(certdir, + self.fam) + + @property + def disableMetaData(self): + """ Report whether or not metadata processing is enabled. + This duplicates code in Packages/__init__.py, and can probably + be removed in Bcfg2 1.4 when we have a module-level setup + object. """ + if self.setup is None: + return True + try: + return not self.setup.cfp.getboolean("packages", "resolver") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return False + except ValueError: + # for historical reasons we also accept "enabled" and + # "disabled" + return self.setup.cfp.get( + "packages", + "metadata", + default="enabled").lower() == "disabled" @property def __package_groups__(self): @@ -323,20 +354,21 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ - global HELPER - if not HELPER: + if not self._helper: + # pylint: disable=W0212 try: - HELPER = self.setup.cfp.get("packages:yum", "helper") + self.__class__._helper = self.setup.cfp.get("packages:yum", + "helper") except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") - Popen(['bcfg2-yum-helper'], - stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() - HELPER = 'bcfg2-yum-helper' + self.cmd.run(['bcfg2-yum-helper']) + self.__class__._helper = 'bcfg2-yum-helper' except OSError: - HELPER = "/usr/sbin/bcfg2-yum-helper" - return HELPER + self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" + # pylint: enable=W0212 + return self._helper @property def use_yum(self): @@ -374,6 +406,7 @@ class YumCollection(Collection): # the rpmdb is so hopelessly intertwined with yum that we # have to totally reinvent the dependency resolver. mainopts = dict(cachedir='/', + persistdir='/', installroot=self.cachefile, keepcache="0", debuglevel="0", @@ -675,7 +708,10 @@ class YumCollection(Collection): gdicts.append(dict(group=group, type=ptype)) if self.use_yum: - return self.call_helper("get_groups", inputdata=gdicts) + try: + return self.call_helper("get_groups", inputdata=gdicts) + except ValueError: + return dict() else: pkgs = dict() for gdict in gdicts: @@ -837,13 +873,25 @@ class YumCollection(Collection): if not self.use_yum: return Collection.complete(self, packagelist) + lock = FileLock(os.path.join(self.cachefile, "lock")) + slept = 0 + while lock.is_locked(): + if slept > 30: + self.logger.warning("Packages: Timeout waiting for yum cache " + "to release its lock") + return set(), set() + self.logger.debug("Packages: Yum cache is locked, waiting...") + time.sleep(3) + slept += 3 + if packagelist: - result = \ - self.call_helper("complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) - if not result: - # some sort of error, reported by call_helper() + try: + result = self.call_helper( + "complete", + dict(packages=list(packagelist), + groups=list(self.get_relevant_groups()))) + except ValueError: + # error reported by call_helper() return set(), packagelist # json doesn't understand sets or tuples, so we get back a # lists of lists (packages) and a list of unicode strings @@ -874,38 +922,41 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - verbose = self.debug_flag or self.setup['verbose'] - if verbose: + if self.setup['verbose']: + cmd.append("-v") + if self.debug_flag: + if not self.setup['verbose']: + # ensure that running in debug gets -vv, even if + # verbose is not enabled + cmd.append("-v") cmd.append("-v") cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose) - try: - helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to execute %s: %s" % - (" ".join(cmd), err)) - return None + self.debug_log("Packages: running %s" % " ".join(cmd)) if inputdata: - idata = json.dumps(inputdata) - (stdout, stderr) = helper.communicate(idata) - else: - (stdout, stderr) = helper.communicate() - rv = helper.wait() - if rv: - self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, stderr)) + result = self.cmd.run(cmd, timeout=self.setup['client_timeout'], + inputdata=json.dumps(inputdata)) else: + result = self.cmd.run(cmd, timeout=self.setup['client_timeout']) + if not result.success: + self.logger.error("Packages: error running bcfg2-yum-helper: %s" % + result.error) + elif result.stderr: self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr, flag=verbose) + result.stderr) + try: - return json.loads(stdout) + return json.loads(result.stdout) except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: error reading bcfg2-yum-helper " - "output: %s" % err) - return None + if result.stdout: + err = sys.exc_info()[1] + self.logger.error("Packages: Error reading bcfg2-yum-helper " + "output: %s" % err) + self.logger.error("Packages: bcfg2-yum-helper output: %s" % + result.stdout) + else: + self.logger.error("Packages: No bcfg2-yum-helper output") + raise def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called @@ -917,8 +968,7 @@ class YumCollection(Collection): If using the yum Python libraries, this cleans up cached yum metadata, regenerates the server-side yum config (in order to catch any new sources that have been added to this server), - and then cleans up cached yum metadata again, in case the new - config has any preexisting cache. + then regenerates the yum cache. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -929,15 +979,22 @@ class YumCollection(Collection): return Collection.setup_data(self, force_update) if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config - self.call_helper("clean") + # clean up data from the old config + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass - os.unlink(self.cfgfile) + if os.path.exists(self.cfgfile): + os.unlink(self.cfgfile) self.write_config() - if force_update: - self.call_helper("clean") + try: + self.call_helper("makecache") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f82b8a392..479138ef1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,8 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ + MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -22,7 +23,54 @@ APT_CONFIG_DEFAULT = \ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +class OnDemandDict(MutableMapping): + """ This maps a set of keys to a set of value-getting functions; + the values are populated on-the-fly by the functions as the values + are needed (and not before). This is used by + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`; + see the docstring for that function for details on why. + + Unlike a dict, you should not specify values for for the righthand + side of this mapping, but functions that get values. E.g.: + + .. code-block:: python + + d = OnDemandDict(foo=load_foo, + bar=lambda: "bar"); + """ + + def __init__(self, **getters): + self._values = dict() + self._getters = dict(**getters) + + def __getitem__(self, key): + if key not in self._values: + self._values[key] = self._getters[key]() + return self._values[key] + + def __setitem__(self, key, getter): + self._getters[key] = getter + + def __delitem__(self, key): + del self._values[key] + del self._getters[key] + + def __len__(self): + return len(self._getters) + + def __iter__(self): + return iter(self._getters.keys()) + + def __repr__(self): + rv = dict(self._values) + for key in self._getters.keys(): + if key not in rv: + rv[key] = 'unknown' + return str(rv) + + class Packages(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -45,8 +93,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] + __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ + [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -110,8 +162,21 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() - # pylint: enable=C0301 + #: groupcache caches group lookups. It maps Collections (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of package groups, and thence to the packages + #: indicated by those groups. + self.groupcache = dict() + + #: pkgcache caches complete package sets. It maps Collections + #: (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of initial packages, and thence to the final + #: (complete) package selections resolved from the initial + #: packages + self.pkgcache = dict() + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def set_debug(self, debug): @@ -355,14 +420,24 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for el in to_remove: el.getparent().remove(el) - gpkgs = collection.get_groups(groups) - for pkgs in gpkgs.values(): + groups.sort() + # check for this set of groups in the group cache + gkey = hash(tuple(groups)) + if gkey not in self.groupcache[collection.cachekey]: + self.groupcache[collection.cachekey][gkey] = \ + collection.get_groups(groups) + for pkgs in self.groupcache[collection.cachekey][gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) - packages, unknown = collection.complete(base) + # check for this set of packages in the package cache + pkey = hash(tuple(base)) + if pkey not in self.pkgcache[collection.cachekey]: + self.pkgcache[collection.cachekey][pkey] = \ + collection.complete(base) + packages, unknown = self.pkgcache[collection.cachekey][pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -388,6 +463,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True + def expire_cache(self, _=None): + self.Reload() + def _load_config(self, force_update=False): """ Load the configuration data and setup sources @@ -415,9 +493,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.disableMetaData: collection.setup_data(force_update) - # clear Collection caches + # clear Collection and package caches self.clients = dict() self.collections = dict() + self.groupcache = dict() + self.pkgcache = dict() for source in self.sources.entries: cachefiles.add(source.cachefile) @@ -493,8 +573,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - return Collection(metadata, [], self.cachepath, self.data, - self.core.fam) + collection = Collection(metadata, [], self.cachepath, self.data, + self.core.fam) + ckey = collection.cachekey + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) + return collection if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -510,7 +594,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if len(sclasses) > 1: self.logger.warning("Packages: Multiple source types found for " "%s: %s" % - ",".join([s.__name__ for s in sclasses])) + (metadata.hostname, + ",".join([s.__name__ for s in sclasses]))) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % @@ -530,24 +615,47 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): """ Return additional data for the given client. This will be - a dict containing a single key, ``sources``, whose value is a - list of data returned from - :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, - namely, a list of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` - data. + an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` + containing two keys: + + * ``sources``, whose value is a list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data; and + * ``get_config``, whose value is the + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` + function, which can be used to get the Packages config for + other systems. + + This uses an OnDemandDict instead of just a normal dict + because loading a source collection can be a fairly + time-consuming process, particularly for the first time. As a + result, when all metadata objects are built at once (such as + after the server is restarted, or far more frequently if + Metadata caching is disabled), this function would be a major + bottleneck if we tried to build all collections at the same + time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ - collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - get_config=self.get_config) + def get_sources(): + """ getter for the 'sources' key of the OnDemandDict + returned by this function. This delays calling + get_collection() until it's absolutely necessary. """ + return self.get_collection(metadata).get_additional_data() + + return OnDemandDict( + sources=get_sources, + get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index f8baddb4b..48be1ac26 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -9,9 +9,11 @@ import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin +from Bcfg2.Compat import unicode # pylint: disable=W0622 try: from django.db import models + from django.core.exceptions import MultipleObjectsReturned HAS_DJANGO = True class ProbesDataModel(models.Model, @@ -32,8 +34,10 @@ except ImportError: try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -63,7 +67,10 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + if isinstance(data, unicode): + return str.__new__(cls, data.encode('utf-8')) + else: + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -180,14 +187,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: @@ -197,6 +206,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, err = sys.exc_info()[1] raise Bcfg2.Server.Plugin.PluginInitError(err) + self.allowed_cgroups = core.setup['probe_allowed_groups'] self.probedata = dict() self.cgroups = dict() self.load_data() @@ -222,9 +232,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=self.probedata[client][probe]) + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str( + self.probedata[client][probe]).decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: @@ -239,35 +255,50 @@ class Probes(Bcfg2.Server.Plugin.Probing, def _write_data_db(self, client): """ Write received probe data to the database """ for probe, data in self.probedata[client.hostname].items(): - pdata = \ - ProbesDataModel.objects.get_or_create(hostname=client.hostname, - probe=probe)[0] + try: + pdata = ProbesDataModel.objects.get_or_create( + hostname=client.hostname, + probe=probe)[0] + except MultipleObjectsReturned: + ProbesDataModel.objects.filter(hostname=client.hostname, + probe=probe).delete() + ProbesDataModel.objects.get_or_create( + hostname=client.hostname, + probe=probe) if pdata.data != data: pdata.data = data pdata.save() ProbesDataModel.objects.filter( hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() + probe__in=self.probedata[client.hostname]).delete() for group in self.cgroups[client.hostname]: try: - ProbesGroupsModel.objects.get(hostname=client.hostname, - group=group) - except ProbesGroupsModel.DoesNotExist: - grp = ProbesGroupsModel(hostname=client.hostname, - group=group) - grp.save() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) + except MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=client.hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) ProbesGroupsModel.objects.filter( hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() + group__in=self.cgroups[client.hostname]).delete() - def load_data(self): + def expire_cache(self, key=None): + self.load_data(client=key) + + def load_data(self, client=None): """ Load probe data from the appropriate backend (probed.xml or the database) """ if self._use_db: - return self._load_data_db() + return self._load_data_db(client=client) else: + # the XML backend doesn't support loading data for single + # clients, so it reloads all data return self._load_data_xml() def _load_data_xml(self): @@ -292,20 +323,36 @@ class Probes(Bcfg2.Server.Plugin.Probing, elif pdata.tag == 'Group': self.cgroups[client.get('name')].append(pdata.get('name')) - def _load_data_db(self): + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + + def _load_data_db(self, client=None): """ Load probe data from the database """ - self.probedata = {} - self.cgroups = {} - for pdata in ProbesDataModel.objects.all(): + if client is None: + self.probedata = {} + self.cgroups = {} + probedata = ProbesDataModel.objects.all() + groupdata = ProbesGroupsModel.objects.all() + else: + self.probedata.pop(client, None) + self.cgroups.pop(client, None) + probedata = ProbesDataModel.objects.filter(hostname=client) + groupdata = ProbesGroupsModel.objects.filter(hostname=client) + + for pdata in probedata: if pdata.hostname not in self.probedata: self.probedata[pdata.hostname] = ClientProbeDataSet( timestamp=time.mktime(pdata.timestamp.timetuple())) self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in ProbesGroupsModel.objects.all(): + for pgroup in groupdata: if pgroup.hostname not in self.cgroups: self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) + @Bcfg2.Server.Plugin.track_statistics() def GetProbes(self, meta): return self.probes.get_probe_data(meta) @@ -347,11 +394,22 @@ class Probes(Bcfg2.Server.Plugin.Probing, if line.split(':')[0] == 'group': newgroup = line.split(':')[1].strip() if newgroup not in cgroups: - cgroups.append(newgroup) + if self._group_allowed(newgroup): + cgroups.append(newgroup) + else: + self.logger.info( + "Disallowed group assignment %s from %s" % + (newgroup, client.hostname)) dlines.remove(line) dobj = ProbeData("\n".join(dlines)) cprobedata[data.get('name')] = dobj + def _group_allowed(self, group): + """ Determine if the named group can be set as a probe group + by checking the regexes listed in the [probes] groups_allowed + setting """ + return any(r.match(group) for r in self.allowed_cgroups) + def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) get_additional_groups.__doc__ = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index e97f66675..8c6cf799a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -17,8 +17,10 @@ except ImportError: try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True @@ -223,7 +225,7 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): if strict: raise PluginExecutionError(msg) else: - LOGGER.warning(msg) + LOGGER.info(msg) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ def _decrypt(self, element): diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 801e7006d..072f3f7e7 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -127,7 +127,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.warning("PuppetENC is incompatible with aggressive " "client metadata caching, try 'cautious' or " "'initial' instead") - self.core.cache.expire() + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def end_statistics(self, metadata): self.end_client_run(self, metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index d8b3104b7..2deea5f07 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -92,6 +92,7 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -125,6 +126,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -149,6 +151,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, HostKeyEntrySet(keypattern, self.data) self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk + def expire_cache(self, key=None): + self.__skn = False + def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index c3a2221f6..41e6bf8b5 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -6,7 +6,9 @@ import Bcfg2.Server.Plugin class ServiceCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """ Use old-style service modes for older clients """ - name = 'ServiceCompat' + + create = False + __author__ = 'bcfg-dev@mcs.anl.gov' mode_map = {('true', 'true'): 'default', ('interactive', 'true'): 'interactive_only', diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 51f44c52d..240fd7f89 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -59,9 +59,48 @@ class Svn(Bcfg2.Server.Plugin.Version): self.client.callback_conflict_resolver = \ self.get_conflict_resolver(choice) + try: + if self.core.setup.cfp.get( + "svn", + "always_trust").lower() == "true": + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.debug("Svn: Using subversion cache for SSL " + "certificate trust") + + try: + if (self.core.setup.cfp.get("svn", "user") and + self.core.setup.cfp.get("svn", "password")): + self.client.callback_get_login = \ + self.get_login + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.info("Svn: Using subversion cache for " + "password-based authetication") + self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) + # pylint: disable=W0613 + def get_login(self, realm, username, may_save): + """ PySvn callback to get credentials for HTTP basic authentication """ + self.logger.debug("Svn: Logging in with username: %s" % + self.core.setup.cfp.get("svn", "user")) + return True, \ + self.core.setup.cfp.get("svn", "user"), \ + self.core.setup.cfp.get("svn", "password"), \ + False + # pylint: enable=W0613 + + def ssl_server_trust_prompt(self, trust_dict): + """ PySvn callback to always trust SSL certificates from SVN server """ + self.logger.debug("Svn: Trusting SSL certificate from %s, " + "issued by %s for realm %s" % + (trust_dict['hostname'], + trust_dict['issuer_dname'], + trust_dict['realm'])) + return True, trust_dict['failures'], False + def get_conflict_resolver(self, choice): """ Get a PySvn conflict resolution callback """ def callback(conflict_description): diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index fcd73bae2..db7370f01 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -115,7 +115,7 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): def Run(self): for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper): + if self.HandlesFile(helper.name): self.check_helper(helper.name) def check_helper(self, helper): diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py index 73adaaaaf..c43c3cee7 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/models.py +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -9,11 +9,13 @@ except ImproperlyConfigured: print("Reports: unable to import django models: %s" % e) sys.exit(1) -from django.db import connection, transaction +from django.db import connection from django.db.models import Q from datetime import datetime, timedelta from time import strptime +from Bcfg2.Reporting.Compat import transaction + KIND_CHOICES = ( #These are the kinds of config elements ('Package', 'Package'), @@ -288,7 +290,7 @@ class Reason(models.Model): return rv @staticmethod - @transaction.commit_on_success + @transaction.atomic def prune_orphans(): '''Prune oprhaned rows... no good way to use the ORM''' cursor = connection.cursor() @@ -305,7 +307,7 @@ class Entries(models.Model): return self.name @staticmethod - @transaction.commit_on_success + @transaction.atomic def prune_orphans(): '''Prune oprhaned rows... no good way to use the ORM''' cursor = connection.cursor() @@ -334,7 +336,7 @@ class Performance(models.Model): return self.metric @staticmethod - @transaction.commit_on_success + @transaction.atomic def prune_orphans(): '''Prune oprhaned rows... no good way to use the ORM''' cursor = connection.cursor() diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 1f64111e7..7e2f5b09d 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -57,7 +57,7 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # the second attempt. LOGGER.error("Failed to load plugin %s: %s" % (plugin, err)) - continue + continue for sym in dir(mod): obj = getattr(mod, sym) if hasattr(obj, "__bases__") and models.Model in obj.__bases__: diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 1c2dceed2..ab1276178 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -81,9 +81,6 @@ class PackedDigitRange(object): # pylint: disable=E0012,R0924 def __str__(self): return "[%s]" % self.str - def __len__(self): - return sum(r[1] - r[0] + 1 for r in self.ranges) + len(self.ints) - def locked(fd): """ Acquire a lock on a file. @@ -219,7 +216,9 @@ class Executor(object): """ if isinstance(command, str): cmdstr = command - command = shlex.split(cmdstr) + + if not shell: + command = shlex.split(cmdstr) else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) @@ -245,9 +244,9 @@ class Executor(object): # py3k fixes if not isinstance(stdout, str): - stdout = stdout.decode('utf-8') + stdout = stdout.decode('utf-8') # pylint: disable=E1103 if not isinstance(stderr, str): - stderr = stderr.decode('utf-8') + stderr = stderr.decode('utf-8') # pylint: disable=E1103 for line in stdout.splitlines(): # pylint: disable=E1103 self.logger.debug('< %s' % line) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 9393830a8..82a3bdb2a 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -26,12 +26,16 @@ DATABASE_USER = None DATABASE_PASSWORD = None DATABASE_HOST = None DATABASE_PORT = None +DATABASE_OPTIONS = None +DATABASE_SCHEMA = None TIME_ZONE = None DEBUG = False TEMPLATE_DEBUG = DEBUG +ALLOWED_HOSTS = ['*'] + MEDIA_URL = '/site_media/' @@ -56,8 +60,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ - DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ - MEDIA_URL + DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DATABASE_SCHEMA, \ + DEBUG, TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): @@ -84,7 +88,9 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): USER=setup['db_user'], PASSWORD=setup['db_password'], HOST=setup['db_host'], - PORT=setup['db_port']) + PORT=setup['db_port'], + OPTIONS=setup['db_options'], + SCHEMA=setup['db_schema']) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: DATABASE_ENGINE = setup['db_engine'] @@ -93,6 +99,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] DATABASE_HOST = DATABASES['default']['HOST'] DATABASE_PORT = DATABASES['default']['PORT'] + DATABASE_OPTIONS = DATABASES['default']['OPTIONS'] + DATABASE_SCHEMA = DATABASES['default']['SCHEMA'] # dropping the version check. This was added in 1.1.2 TIME_ZONE = setup['time_zone'] diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 12fc584fe..35d4cfa0a 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.1" +__version__ = "1.3.3" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 |