diff options
Diffstat (limited to 'src/lib')
36 files changed, 449 insertions, 259 deletions
diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 5633764a8..1676ee717 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -91,7 +91,10 @@ class Client(object): try: script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) - script.write(probe.text) + if sys.hexversion >= 0x03000000: + script.write(probe.text) + else: + script.write(probe.text.encode('utf-8')) script.close() os.chmod(scriptname, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | @@ -105,7 +108,10 @@ class Client(object): self._probe_failure(name, "Return value %s" % rv) self.logger.info("Probe %s has result:" % name) self.logger.info(rv.stdout) - ret.text = rv.stdout + if sys.hexversion >= 0x03000000: + ret.text = rv.stdout + else: + ret.text = rv.stdout.decode('utf-8') finally: os.unlink(scriptname) except SystemExit: @@ -167,7 +173,7 @@ class Client(object): self.proxy.RecvProbeData( Bcfg2.Client.XML.tostring( probedata, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False).decode('utf-8')) except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to upload probe data: %s" % err) @@ -229,7 +235,7 @@ class Client(object): self.fatal_error("Failed to get decision list: %s" % err) try: - rawconfig = self.proxy.GetConfig().encode('UTF-8') + rawconfig = self.proxy.GetConfig().encode('utf-8') except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to download configuration from " @@ -247,7 +253,7 @@ class Client(object): self.logger.info("Starting Bcfg2 client run at %s" % times['start']) - rawconfig = self.get_config(times=times) + rawconfig = self.get_config(times=times).decode('utf-8') if self.setup['cache']: try: @@ -324,7 +330,7 @@ class Client(object): self.proxy.RecvStats( Bcfg2.Client.XML.tostring( feedback, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False).decode('utf-8')) except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.logger.error("Failed to upload configuration statistics: " diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index ada5320b8..850e58d9d 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -417,15 +417,18 @@ class Frame(object): bundle.get('name') not in self.setup['bundle']): # prune out unspecified bundles when running with -b continue + if bundle in mbundles: + self.logger.debug("Bundle %s was modified" % bundle) + func = "BundleUpdated" + else: + self.logger.debug("Bundle %s was not modified" % bundle) + func = "BundleNotUpdated" for tool in self.tools: try: - if bundle in mbundles: - tool.BundleUpdated(bundle, self.states) - else: - tool.BundleNotUpdated(bundle, self.states) + getattr(tool, func)(bundle, self.states) except: - self.logger.error("%s.BundleNotUpdated() call failed:" % - tool.name, exc_info=1) + self.logger.error("%s.%s() call failed:" % + (tool.name, func), exc_info=1) def Remove(self): """Remove extra entries.""" @@ -447,15 +450,16 @@ class Frame(object): self.logger.info('Incorrect entries: %d' % list(self.states.values()).count(False)) if phase == 'final' and list(self.states.values()).count(False): - for entry in self.states.keys(): + for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + + e.get('name')): if not self.states[entry]: etype = entry.get('type') if etype: self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) self.logger.info('Total managed entries: %d' % len(list(self.states.values()))) self.logger.info('Unmanaged entries: %d' % len(self.extra)) @@ -467,8 +471,8 @@ class Frame(object): self.logger.info("%s:%s:%s" % (entry.tag, etype, entry.get('name'))) else: - self.logger.info(" %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.info("%s:%s" % (entry.tag, + entry.get('name'))) if ((list(self.states.values()).count(False) == 0) and not self.extra): self.logger.info('All entries correct.') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 9b95d2234..168c35c98 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -34,13 +34,11 @@ class POSIXFile(POSIXTool): def _get_data(self, entry): """ Get a tuple of (<file data>, <is binary>) for the given entry """ - is_binary = False - if entry.get('encoding', 'ascii') == 'base64': - tempdata = b64decode(entry.text) - is_binary = True - - elif entry.get('empty', 'false') == 'true': + is_binary = entry.get('encoding', 'ascii') == 'base64' + if entry.get('empty', 'false') == 'true' or not entry.text: tempdata = '' + elif is_binary: + tempdata = b64decode(entry.text) else: tempdata = entry.text if isinstance(tempdata, unicode) and unicode != str: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 11f331ddb..16fe0acb5 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -275,7 +275,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if path is None: path = entry.get("name") context = entry.get("secontext") - if context is None: + if not context: # no context listed return True @@ -520,13 +520,19 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): "Current mtime is %s but should be %s" % (path, mtime, entry.get('mtime'))) - if HAS_SELINUX and entry.get("secontext"): + if HAS_SELINUX: + wanted_secontext = None if entry.get("secontext") == "__default__": - wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + try: + wanted_secontext = \ + selinux.matchpathcon(path, 0)[1].split(":")[2] + except OSError: + errors.append("%s has no default SELinux context" % + entry.get("name")) else: wanted_secontext = entry.get("secontext") - if attrib['current_secontext'] != wanted_secontext: + if (wanted_secontext and + attrib['current_secontext'] != wanted_secontext): errors.append("SELinux context for path %s is incorrect. " "Current context is %s but should be %s" % (path, attrib['current_secontext'], diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 552b27842..4b78581f7 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -12,6 +12,15 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'rc-update')] __req__ = {'Service': ['name', 'status']} + def get_enabled_svcs(self): + """ + Return a list of all enabled services. + """ + return [line.split()[0] + for line in self.cmd.run(['/bin/rc-status', + '-s']).stdout.splitlines() + if 'started' in line] + def VerifyService(self, entry, _): """ Verify Service status for entry. @@ -21,9 +30,12 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): if entry.get('status') == 'ignore': return True + # get a list of all started services + allsrv = self.get_enabled_svcs() + # check if service is enabled - result = self.cmd.run(["/sbin/rc-update", "show", "default"]) - is_enabled = entry.get("name") in result.stdout + result = self.cmd.run(["/sbin/rc-update", "show", "default"]).stdout + is_enabled = entry.get("name") in result # check if init script exists try: @@ -34,8 +46,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): return False # check if service is enabled - result = self.cmd.run(self.get_svc_command(entry, "status")) - is_running = "started" in result.stdout + is_running = entry.get('name') in allsrv if entry.get('status') == 'on' and not (is_enabled and is_running): entry.set('current_status', 'off') @@ -70,10 +81,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): def FindExtra(self): """Locate extra rc-update services.""" - allsrv = [line.split()[0] - for line in self.cmd.run(['/bin/rc-status', - '-s']).stdout.splitlines() - if 'started' in line] + allsrv = self.get_enabled_svcs() self.logger.debug('Found active services:') self.logger.debug(allsrv) specified = [srv.get('name') for srv in self.getSupportedEntries()] diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 0041ce61a..0b4aba60d 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -204,7 +204,16 @@ class SELinuxEntryHandler(object): type, if the records object supports the customized() method """ if hasattr(self.records, "customized") and self.custom_re: - return dict([(k, self.all_records[k]) for k in self.custom_keys]) + rv = dict() + for key in self.custom_keys: + if key in self.all_records: + rv[key] = self.all_records[key] + else: + self.logger.warning("SELinux %s %s customized, but no " + "record found. This may indicate an " + "error in your SELinux policy." % + (self.etype, key)) + return rv else: # ValueError is really a pretty dumb exception to raise, # but that's what the seobject customized() method raises @@ -491,7 +500,8 @@ class SELinuxSeportHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding and modifying entries """ (port, proto) = entry.get("name").split("/") - return (port, proto, '', entry.get("selinuxtype")) + return (port, proto, entry.get("mlsrange", ""), + entry.get("selinuxtype")) def _deleteargs(self, entry): return tuple(entry.get("name").split("/")) @@ -564,7 +574,7 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler): """ argument list for adding, modifying, and deleting entries """ return (entry.get("name"), entry.get("selinuxtype"), self.filetypeargs[entry.get("filetype", "all")], - '', '') + entry.get("mlsrange", ""), '') def primarykey(self, entry): return ":".join([entry.tag, entry.get("name"), @@ -599,7 +609,7 @@ class SELinuxSenodeHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ (addr, netmask) = entry.get("name").split("/") - return (addr, netmask, entry.get("proto"), "", + return (addr, netmask, entry.get("proto"), entry.get("mlsrange", ""), entry.get("selinuxtype")) @@ -611,7 +621,8 @@ class SELinuxSeloginHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), entry.get("selinuxuser"), "") + return (entry.get("name"), entry.get("selinuxuser"), + entry.get("mlsrange", "")) class SELinuxSeuserHandler(SELinuxEntryHandler): @@ -651,15 +662,16 @@ class SELinuxSeuserHandler(SELinuxEntryHandler): # prefix. see the comment in Install() above for more # details. rv = [entry.get("name"), - entry.get("roles", "").replace(" ", ",").split(",")] + entry.get("roles", "").replace(" ", ",").split(","), + '', entry.get("mlsrange", "")] if self.needs_prefix: - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) else: key = self._key(entry) if key in self.all_records: attrs = self._key2attrs(key) if attrs['prefix'] != entry.get("prefix"): - rv.extend(['', '', entry.get("prefix")]) + rv.append(entry.get("prefix")) return tuple(rv) @@ -671,7 +683,8 @@ class SELinuxSeinterfaceHandler(SELinuxEntryHandler): def _defaultargs(self, entry): """ argument list for adding, modifying, and deleting entries """ - return (entry.get("name"), '', entry.get("selinuxtype")) + return (entry.get("name"), entry.get("mlsrange", ""), + entry.get("selinuxtype")) class SELinuxSepermissiveHandler(SELinuxEntryHandler): diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index c9fae7fc7..c30c0a13a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -131,10 +131,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): def __init__(self, logger, setup, config): self.yumbase = self._loadYumBase(setup=setup, logger=logger) Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) - self.ignores = [entry.get('name') for struct in config \ - for entry in struct \ - if entry.tag == 'Path' and \ - entry.get('type') == 'ignore'] + self.ignores = [] + for struct in config: + self.ignores.extend([entry.get('name') + for entry in struct + if (entry.tag == 'Path' and + entry.get('type') == 'ignore')]) self.instance_status = {} self.extra_instances = [] self.modlists = {} @@ -293,8 +295,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): group. """ missing = Bcfg2.Client.Tools.PkgTool.missing_attrs(self, entry) - if entry.get('name', None) == None and \ - entry.get('group', None) == None: + if (entry.get('name', None) is None and + entry.get('group', None) is None): missing += ['name', 'group'] return missing @@ -422,10 +424,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if entry.get('group'): self.logger.debug("Verifying packages for group %s" % - entry.get('group')) + entry.get('group')) else: self.logger.debug("Verifying package instances for %s" % - entry.get('name')) + entry.get('name')) self.verify_cache = dict() # Used for checking multilib packages self.modlists[entry] = modlist @@ -434,10 +436,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = False qtext_versions = [] virt_pkg = False - pkg_checks = self.pkg_checks and \ - entry.get('pkg_checks', 'true').lower() == 'true' - pkg_verify = self.pkg_verify and \ - entry.get('pkg_verify', 'true').lower() == 'true' + pkg_checks = (self.pkg_checks and + entry.get('pkg_checks', 'true').lower() == 'true') + pkg_verify = (self.pkg_verify and + entry.get('pkg_verify', 'true').lower() == 'true') yum_group = False if entry.get('name') == 'gpg-pubkey': @@ -455,15 +457,13 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if d] group_type = entry.get('choose', 'default') if group_type in ['default', 'optional', 'all']: - group_packages += [p - for p, d in - group.default_packages.items() - if d] + group_packages += [ + p for p, d in group.default_packages.items() + if d] if group_type in ['optional', 'all']: - group_packages += [p - for p, d in - group.optional_packages.items() - if d] + group_packages += [ + p for p, d in group.optional_packages.items() + if d] if len(group_packages) == 0: self.logger.error("No packages found for group %s" % entry.get("group")) @@ -489,7 +489,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: all_pkg_objs = \ self.yumbase.rpmdb.searchNevra(name=entry.get('name')) - if len(all_pkg_objs) == 0 and yum_group != True: + if len(all_pkg_objs) == 0 and yum_group is not True: # Some sort of virtual capability? Try to resolve it all_pkg_objs = self.yumbase.rpmdb.searchProvides(entry.get('name')) if len(all_pkg_objs) > 0: @@ -567,9 +567,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): pkg_objs = [po for po in all_pkg_objs] else: pkg_objs = [po for po in all_pkg_objs - if po.checkPrco('provides', - (nevra["name"], 'EQ', - tuple(vlist)))] + if po.checkPrco('provides', + (nevra["name"], 'EQ', + tuple(vlist)))] elif entry.get('name') == 'gpg-pubkey': if 'version' not in nevra: self.logger.warning("Skipping verify: gpg-pubkey without " @@ -622,7 +622,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if self.setup.get('quick', False): # Passed -q on the command line continue - if not (pkg_verify and \ + if not (pkg_verify and inst.get('pkg_verify', 'true').lower() == 'true'): continue @@ -648,8 +648,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Now take out the Yum specific objects / modlists / unproblems ignores = [ig.get('name') for ig in entry.findall('Ignore')] + \ - [ig.get('name') for ig in inst.findall('Ignore')] + \ - self.ignores + [ig.get('name') for ig in inst.findall('Ignore')] + \ + self.ignores for fname, probs in list(vrfy_result.items()): if fname in modlist: self.logger.debug(" %s in modlist, skipping" % fname) @@ -737,8 +737,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): for pkg in pkg_objs: self.logger.debug(" Extra Instance Found: %s" % str(pkg)) Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', - epoch=pkg.epoch, name=pkg.name, version=pkg.version, - release=pkg.release, arch=pkg.arch) + epoch=pkg.epoch, name=pkg.name, + version=pkg.version, + release=pkg.release, arch=pkg.arch) if pkg_objs == []: return None @@ -782,7 +783,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): ver = yum.misc.keyIdToRPMVer(gpg['keyid']) rel = yum.misc.keyIdToRPMVer(gpg['timestamp']) if not (ver == inst.get('version') and rel == inst.get('release')): - self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s"\ + self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s" % (key_file, inst.get('version'), inst.get('release'))) return False @@ -791,20 +792,21 @@ class YUM(Bcfg2.Client.Tools.PkgTool): gpg['timestamp']) == 0: result = tset.pgpImportPubkey(yum.misc.procgpgkey(rawkey)) else: - self.logger.debug("gpg-pubkey-%s-%s already installed"\ - % (inst.get('version'), - inst.get('release'))) + self.logger.debug("gpg-pubkey-%s-%s already installed" % + (inst.get('version'), inst.get('release'))) return True if result != 0: - self.logger.debug("Unable to install %s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), - nevra2string(inst))) + self.logger.debug( + "Unable to install %s-%s" % + (self.instance_status[inst].get('pkg').get('name'), + nevra2string(inst))) return False else: - self.logger.debug("Installed %s-%s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), - inst.get('version'), inst.get('release'))) + self.logger.debug( + "Installed %s-%s-%s" % + (self.instance_status[inst].get('pkg').get('name'), + inst.get('version'), inst.get('release'))) return True def _runYumTransaction(self): @@ -898,7 +900,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if self.extra_instances is not None and len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or \ + if (self.setup.get('remove') == 'all' or self.setup.get('remove') == 'packages'): self.Remove(self.extra_instances) else: @@ -913,7 +915,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Figure out which instances of the packages actually need something # doing to them and place in the appropriate work 'queue'. for pkg in packages: - insts = [pinst for pinst in pkg \ + insts = [pinst for pinst in pkg if pinst.tag in ['Instance', 'Package']] if insts: for inst in insts: @@ -1006,10 +1008,11 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if not self.setup['kevlar']: for pkg_entry in [p for p in packages if self.canVerify(p)]: - self.logger.debug("Reverifying Failed Package %s" \ - % (pkg_entry.get('name'))) - states[pkg_entry] = self.VerifyPackage(pkg_entry, - self.modlists.get(pkg_entry, [])) + self.logger.debug("Reverifying Failed Package %s" % + pkg_entry.get('name')) + states[pkg_entry] = \ + self.VerifyPackage(pkg_entry, + self.modlists.get(pkg_entry, [])) for entry in [ent for ent in packages if states[ent]]: self.modified.append(entry) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 8c8c4fd94..e40ef750b 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -3,9 +3,7 @@ import os import sys import select -from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 - -__all__ = [m[1] for m in walk_packages(path=__path__)] +from Bcfg2.Compat import input # pylint: disable=W0622 def prompt(msg): diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py index 2b4ba6237..b4674d72f 100755 --- a/src/lib/Bcfg2/Encryption.py +++ b/src/lib/Bcfg2/Encryption.py @@ -27,7 +27,7 @@ ALGORITHM = "aes_256_cbc" #: Default initialization vector. For best security, you should use a #: unique IV for each message. :func:`ssl_encrypt` does this in an #: automated fashion. -IV = '\0' * 16 +IV = r'\0' * 16 #: The config file section encryption options and passphrases are #: stored in @@ -116,9 +116,11 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): # base64-decode the data data = b64decode(data) salt = data[8:16] + # pylint: disable=E1101 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101 key = hashes[0] + hashes[1] iv = hashes[2] @@ -144,9 +146,11 @@ def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): if salt is None: salt = Rand.rand_bytes(8) + # pylint: disable=E1101 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101 key = hashes[0] + hashes[1] iv = hashes[2] diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 66e987b91..c7604c5c4 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -401,7 +401,8 @@ CFILE = \ Option('Specify configuration file', default=DEFAULT_CONFIG_LOCATION, cmd='-C', - odesc='<conffile>') + odesc='<conffile>', + env="BCFG2_CONFIG") LOCKFILE = \ Option('Specify lockfile', default='/var/lock/bcfg2.run', @@ -534,6 +535,11 @@ SERVER_FAM_IGNORE = \ 'SCCS', '.svn', '4913', '.gitignore'], cf=('server', 'ignore_files'), cook=list_split) +SERVER_FAM_BLOCK = \ + Option('FAM blocks on startup until all events are processed', + default=False, + cook=get_bool, + cf=('server', 'fam_blocking')) SERVER_LISTEN_ALL = \ Option('Listen on all interfaces', default=False, @@ -1082,6 +1088,15 @@ VERBOSE = \ cmd='-v', cook=get_bool, cf=('logging', 'verbose')) +LOG_PERFORMANCE = \ + Option("Periodically log performance statistics", + default=False, + cf=('logging', 'performance')) +PERFLOG_INTERVAL = \ + Option("Performance statistics logging interval in seconds", + default=300.0, + cook=get_timeout, + cf=('logging', 'performance_interval')) # Plugin-specific options CFG_VALIDATION = \ @@ -1156,6 +1171,7 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, password=SERVER_PASSWORD, filemonitor=SERVER_FILEMONITOR, ignore=SERVER_FAM_IGNORE, + fam_blocking=SERVER_FAM_BLOCK, location=SERVER_LOCATION, key=SERVER_KEY, cert=SERVER_CERT, @@ -1164,7 +1180,9 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, web_configfile=WEB_CFILE, backend=SERVER_BACKEND, vcs_root=SERVER_VCS_ROOT, - authentication=SERVER_AUTHENTICATION) + authentication=SERVER_AUTHENTICATION, + perflog=LOG_PERFORMANCE, + perflog_interval=PERFLOG_INTERVAL) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, @@ -1269,6 +1287,11 @@ TEST_COMMON_OPTIONS = dict(noseopts=TEST_NOSEOPTS, xunit=TEST_XUNIT, validate=CFG_VALIDATION) +INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, + max_copies=PARANOID_MAX_COPIES) +INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) +INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) + class OptionParser(OptionSet): """ diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index b2b9fcc2e..9246c0f87 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -24,6 +24,7 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus version = sys.version_info[:2] has_py26 = version >= (2, 6) +has_py32 = version >= (3, 2) __all__ = ["ComponentProxy", "RetryMethod", @@ -173,8 +174,12 @@ class SSLHTTPConnection(httplib.HTTPConnection): """ if not has_py26: httplib.HTTPConnection.__init__(self, host, port, strict) - else: + elif not has_py32: httplib.HTTPConnection.__init__(self, host, port, strict, timeout) + else: + # the strict parameter is deprecated. + # HTTP 0.9-style "Simple Responses" are not supported anymore. + httplib.HTTPConnection.__init__(self, host, port, timeout=timeout) self.key = key self.cert = cert self.ca = ca diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index 0a0f032e5..c7d5c512a 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -87,7 +87,7 @@ class LocalFilesystem(TransportBase): # using a tmpfile to hopefully avoid the file monitor from grabbing too # soon - saved = open(tmp_file, 'w') + saved = open(tmp_file, 'wb') try: saved.write(payload) except IOError: @@ -123,7 +123,7 @@ class LocalFilesystem(TransportBase): self.debug_log("Handling event %s" % event.filename) payload = os.path.join(self.work_path, event.filename) try: - payloadfd = open(payload, "r") + payloadfd = open(payload, "rb") interaction = cPickle.load(payloadfd) payloadfd.close() os.unlink(payload) diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py index 5c51dad1e..73bdd0b3a 100644 --- a/src/lib/Bcfg2/Reporting/Transport/__init__.py +++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py @@ -2,11 +2,11 @@ Public transport routines """ -import traceback - +import sys from Bcfg2.Reporting.Transport.base import TransportError, \ TransportImportError + def load_transport(transport_name, setup): """ Try to load the transport. Raise TransportImportError on failure @@ -18,13 +18,14 @@ def load_transport(transport_name, setup): try: mod = __import__(transport_name) except: - raise TransportImportError("Unavailable") + raise TransportImportError("Error importing transport %s: %s" % + (transport_name, sys.exc_info()[1])) try: - cls = getattr(mod, transport_name) - return cls(setup) + return getattr(mod, transport_name)(setup) except: - raise TransportImportError("Transport unavailable: %s" % - traceback.format_exc().splitlines()[-1]) + raise TransportImportError("Error instantiating transport %s: %s" % + (transport_name, sys.exc_info()[1])) + def load_transport_from_config(setup): """Load the transport in the config... eventually""" @@ -32,4 +33,3 @@ def load_transport_from_config(setup): return load_transport(setup['reporting_transport'], setup) except KeyError: raise TransportImportError('Transport missing in config') - diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 4be509f53..e63c180a8 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -139,9 +139,11 @@ class Interaction(models.Model): posixgroups = models.ManyToManyField("POSIXGroupEntry") failures = models.ManyToManyField("FailureEntry") - entry_types = ('actions', 'packages', 'paths', 'services', 'sebooleans', - 'seports', 'sefcontexts', 'senodes', 'selogins', 'seusers', - 'seinterfaces', 'sepermissives', 'semodules', 'posixusers', + entry_types = ('actions', 'failures', 'packages', + 'paths', 'services', 'sebooleans', + 'seports', 'sefcontexts', 'senodes', + 'selogins', 'seusers', 'seinterfaces', + 'sepermissives', 'semodules', 'posixusers', 'posixgroups') # Formerly InteractionMetadata diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 13c0563ec..93e42305c 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -3,6 +3,7 @@ import getopt import lxml.etree import sys import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource class Minestruct(Bcfg2.Server.Admin.StructureMode): @@ -39,7 +40,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: extra = set() - for source in self.bcore.pull_sources: + for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) except: diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 9f1b3d138..8001425df 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -6,6 +6,7 @@ import sys import getopt import select import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource, Generator from Bcfg2.Compat import input # pylint: disable=W0622 @@ -62,13 +63,14 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): given client/entry from statistics. """ new_entry = {'type': etype, 'name': ename} - for plugin in self.bcore.pull_sources: + pull_sources = self.bcore.plugins_by_type(PullSource) + for plugin in pull_sources: try: (owner, group, mode, contents) = \ plugin.GetCurrentEntry(client, etype, ename) break except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == self.bcore.pull_sources[-1]: + if plugin == pull_sources[-1]: print("Pull Source failure; could not fetch current state") raise SystemExit(1) @@ -121,8 +123,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): meta = self.bcore.build_metadata(client) # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.generators if - ename in gen.Entries.get(etype, {})] + glist = [gen for gen in self.bcore.plugins_by_type(Generator) + if ename in gen.Entries.get(etype, {})] if len(glist) != 1: self.errExit("Got wrong numbers of matching generators for entry:" "%s" % ([g.name for g in glist])) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 4d7453840..c3302f1d0 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -9,12 +9,12 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.SSLServer import XMLRPCServer -from lockfile import LockFailed +from lockfile import LockFailed, LockTimeout # pylint: disable=E0611 try: - from daemon.pidfile import PIDLockFile + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 @@ -33,7 +33,8 @@ class Core(BaseCore): gid=self.setup['daemon_gid'], umask=int(self.setup['umask'], 8)) if self.setup['daemon']: - daemon_args['pidfile'] = PIDLockFile(self.setup['daemon']) + daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], + acquire_timeout=5) #: The :class:`daemon.DaemonContext` used to drop #: privileges, write the PID file (with :class:`PidFile`), #: and daemonize this core. @@ -89,6 +90,11 @@ class Core(BaseCore): err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False + except LockTimeout: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire lock " + "on %s" % (self.name, self.setup['daemon'])) + return False def _run(self): """ Create :attr:`server` to start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index deb9065a5..ab8cda3da 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,8 +19,9 @@ from Bcfg2.Cache import Cache import Bcfg2.Statistics from itertools import chain from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError, \ - track_statistics +from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 +from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 +from Bcfg2.Server.Plugin import track_statistics try: import psyco @@ -96,6 +97,7 @@ class BaseCore(object): .. automethod:: _block .. ----- .. automethod:: _file_monitor_thread + .. automethod:: _perflog_thread """ #: The Bcfg2 repository directory self.datastore = setup['repo'] @@ -174,6 +176,9 @@ class BaseCore(object): #: the first one loaded wins. self.plugin_blacklist = {} + #: The Metadata plugin + self.metadata = None + #: Revision of the Bcfg2 specification. This will be sent to #: the client in the configuration, and can be set by a #: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin. @@ -235,71 +240,6 @@ class BaseCore(object): self.logger.error("Failed to set ownership of database " "at %s: %s" % (db_settings['NAME'], err)) - if '' in setup['plugins']: - setup['plugins'].remove('') - - for plugin in setup['plugins']: - if not plugin in self.plugins: - self.init_plugin(plugin) - # Remove blacklisted plugins - for plugin, blacklist in list(self.plugin_blacklist.items()): - if len(blacklist) > 0: - self.logger.error("The following plugins conflict with %s;" - "Unloading %s" % (plugin, blacklist)) - for plug in blacklist: - del self.plugins[plug] - - # Log experimental plugins - expl = [plug for plug in list(self.plugins.values()) - if plug.experimental] - if expl: - self.logger.info("Loading experimental plugin(s): %s" % - (" ".join([x.name for x in expl]))) - self.logger.info("NOTE: Interfaces subject to change") - - # Log deprecated plugins - depr = [plug for plug in list(self.plugins.values()) - if plug.deprecated] - if depr: - self.logger.info("Loading deprecated plugin(s): %s" % - (" ".join([x.name for x in depr]))) - - # Find the metadata plugin and set self.metadata - mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) - if len(mlist) >= 1: - #: The Metadata plugin - self.metadata = mlist[0] - if len(mlist) > 1: - self.logger.error("Multiple Metadata plugins loaded; " - "using %s" % self.metadata) - else: - self.logger.error("No Metadata plugin loaded; " - "failed to instantiate Core") - raise CoreInitError("No Metadata Plugin") - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics` - self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource` - #: interface - self.pull_sources = \ - self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) - - #: The list of - #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins - self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Structure` - #: generation - self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface - self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) - #: The CA that signed the server cert self.ca = setup['ca'] @@ -317,6 +257,12 @@ class BaseCore(object): threading.Thread(name="%sFAMThread" % setup['filemonitor'], target=self._file_monitor_thread) + self.perflog_thread = None + if self.setup['perflog']: + self.perflog_thread = \ + threading.Thread(name="PerformanceLoggingThread", + target=self._perflog_thread) + #: A :func:`threading.Lock` for use by #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() @@ -325,10 +271,6 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() - if self.debug_flag: - # enable debugging on everything else. - self.plugins[plugin].set_debug(self.debug_flag) - def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -349,11 +291,23 @@ class BaseCore(object): if isinstance(plugin, base_cls)], key=lambda p: (p.sort_order, p.name)) + def _perflog_thread(self): + """ The thread that periodically logs performance statistics + to syslog. """ + self.logger.debug("Performance logging thread starting") + while not self.terminate.isSet(): + self.terminate.wait(self.setup['perflog_interval']) + for name, stats in self.get_statistics(None).items(): + self.logger.info("Performance statistics: " + "%s min=%.06f, max=%.06f, average=%.06f, " + "count=%d" % ((name, ) + stats)) + def _file_monitor_thread(self): """ The thread that runs the :class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also queries :class:`Bcfg2.Server.Plugin.interfaces.Version` plugins for the current revision of the Bcfg2 repo. """ + self.logger.debug("File monitor thread starting") famfd = self.fam.fileno() terminate = self.terminate while not terminate.isSet(): @@ -372,7 +326,7 @@ class BaseCore(object): def _update_vcs_revision(self): """ Update the revision of the current configuration on-disk from the VCS plugin """ - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): + for plugin in self.plugins_by_type(Version): try: newrev = plugin.get_revision() if newrev != self.revision: @@ -384,6 +338,58 @@ class BaseCore(object): (plugin.name, sys.exc_info()[1])) self.revision = '-1' + def load_plugins(self): + """ Load all plugins, setting + :attr:`Bcfg2.Server.Core.BaseCore.plugins` and + :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects. + This does not start plugin threads; that is done later, in + :func:`Bcfg2.Server.Core.BaseCore.run` """ + while '' in self.setup['plugins']: + self.setup['plugins'].remove('') + + for plugin in self.setup['plugins']: + if not plugin in self.plugins: + self.init_plugin(plugin) + + # Remove blacklisted plugins + for plugin, blacklist in list(self.plugin_blacklist.items()): + if len(blacklist) > 0: + self.logger.error("The following plugins conflict with %s;" + "Unloading %s" % (plugin, blacklist)) + for plug in blacklist: + del self.plugins[plug] + + # Log experimental plugins + expl = [plug for plug in list(self.plugins.values()) + if plug.experimental] + if expl: + self.logger.info("Loading experimental plugin(s): %s" % + (" ".join([x.name for x in expl]))) + self.logger.info("NOTE: Interfaces subject to change") + + # Log deprecated plugins + depr = [plug for plug in list(self.plugins.values()) + if plug.deprecated] + if depr: + self.logger.info("Loading deprecated plugin(s): %s" % + (" ".join([x.name for x in depr]))) + + # Find the metadata plugin and set self.metadata + mlist = self.plugins_by_type(Metadata) + if len(mlist) >= 1: + self.metadata = mlist[0] + if len(mlist) > 1: + self.logger.error("Multiple Metadata plugins loaded; using %s" + % self.metadata) + else: + self.logger.error("No Metadata plugin loaded; " + "failed to instantiate Core") + raise CoreInitError("No Metadata Plugin") + + if self.debug_flag: + # enable debugging on plugins + self.plugins[plugin].set_debug(self.debug_flag) + def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. @@ -468,8 +474,7 @@ class BaseCore(object): metadata.hostname)) start = time.time() try: - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.ClientRunHooks): + for plugin in self.plugins_by_type(ClientRunHooks): try: getattr(plugin, hook)(metadata) except AttributeError: @@ -500,11 +505,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating structures for %s" % metadata.hostname) - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): + for plugin in self.plugins_by_type(StructureValidator): try: plugin.validate_structures(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s structure validation failed: %s" % (plugin.name, err)) @@ -527,10 +531,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating goals for %s" % metadata.hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): + for plugin in self.plugins_by_type(GoalValidator): try: plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s goal validation failed: %s" % (plugin.name, err.message)) @@ -548,8 +552,9 @@ class BaseCore(object): :returns: list of :class:`lxml.etree._Element` objects """ self.logger.debug("Getting structures for %s" % metadata.hostname) - structures = list(chain(*[struct.BuildStructures(metadata) - for struct in self.structures])) + structures = list( + chain(*[struct.BuildStructures(metadata) + for struct in self.plugins_by_type(Structure)])) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: @@ -634,8 +639,9 @@ class BaseCore(object): self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) - glist = [gen for gen in self.generators if - entry.get('name') in gen.Entries.get(entry.tag, {})] + generators = self.plugins_by_type(Generator) + glist = [gen for gen in generators + if entry.get('name') in gen.Entries.get(entry.tag, {})] if len(glist) == 1: return glist[0].Entries[entry.tag][entry.get('name')](entry, metadata) @@ -643,8 +649,8 @@ class BaseCore(object): generators = ", ".join([gen.name for gen in glist]) self.logger.error("%s %s served by multiple generators: %s" % (entry.tag, entry.get('name'), generators)) - g2list = [gen for gen in self.generators if - gen.HandlesEntry(entry, metadata)] + g2list = [gen for gen in generators + if gen.HandlesEntry(entry, metadata)] try: if len(g2list) == 1: return g2list[0].HandleEntry(entry, metadata) @@ -671,7 +677,7 @@ class BaseCore(object): revision=self.revision) try: meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.logger.error("Metadata consistency error for client %s" % client) return lxml.etree.Element("error", type='metadata error') @@ -718,7 +724,8 @@ class BaseCore(object): :type event: Bcfg2.Server.FileMonitor.Event """ if event.filename != self.cfile: - print("Got event for unknown file: %s" % event.filename) + self.logger.error("Got event for unknown file: %s" % + event.filename) return if event.code2str() == 'deleted': return @@ -755,16 +762,25 @@ class BaseCore(object): return False try: + self.load_plugins() + self.fam.start() self.fam_thread.start() self.fam.AddMonitor(self.cfile, self) + if self.perflog_thread is not None: + self.perflog_thread.start() - for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded): + for plug in self.plugins_by_type(Threaded): plug.start_threads() except: self.shutdown() raise + if self.setup['fam_blocking']: + time.sleep(1) + while self.fam.pending() != 0: + time.sleep(1) + self.set_debug(None, self.debug_flag) self._block() @@ -796,7 +812,7 @@ class BaseCore(object): """ self.logger.debug("Getting decision list for %s" % metadata.hostname) result = [] - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): + for plugin in self.plugins_by_type(Decision): try: result.extend(plugin.GetDecisions(metadata, mode)) except: @@ -815,7 +831,7 @@ class BaseCore(object): """ if not hasattr(self, 'metadata'): # some threads start before metadata is even loaded - raise Bcfg2.Server.Plugin.MetadataRuntimeError + raise MetadataRuntimeError("Metadata not loaded yet") if self.metadata_cache_mode == 'initial': # the Metadata plugin handles loading the cached data if # we're only caching the initial metadata object @@ -825,10 +841,11 @@ class BaseCore(object): if not imd: self.logger.debug("Building metadata for %s" % client_name) imd = self.metadata.get_initial_metadata(client_name) - for conn in self.connectors: + connectors = self.plugins_by_type(Connector) + for conn in connectors: grps = conn.get_additional_groups(imd) self.metadata.merge_additional_groups(imd, grps) - for conn in self.connectors: + for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) imd.query.by_name = self.build_metadata @@ -849,7 +866,7 @@ class BaseCore(object): meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': - for plugin in self.statistics: + for plugin in self.plugins_by_type(Statistics): try: plugin.process_statistics(meta, statistics) except: @@ -891,11 +908,11 @@ class BaseCore(object): meta = self.build_metadata(client) else: meta = None - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: err = sys.exc_info()[1] self.critical_error("Client metadata resolution error for %s: %s" % (address[0], err)) - except Bcfg2.Server.Plugin.MetadataRuntimeError: + except MetadataRuntimeError: err = sys.exc_info()[1] self.critical_error('Metadata system runtime failure for %s: %s' % (address[0], err)) @@ -989,8 +1006,7 @@ class BaseCore(object): version)) try: self.metadata.set_version(client, version) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to set version for %s: %s" % (client, err)) @@ -1010,7 +1026,7 @@ class BaseCore(object): client, metadata = self.resolve_client(address, cleanup_cache=True) self.logger.debug("Getting probes for %s" % client) try: - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for plugin in self.plugins_by_type(Probing): for probe in plugin.GetProbes(metadata): resp.append(probe) return lxml.etree.tostring(resp, @@ -1080,8 +1096,7 @@ class BaseCore(object): self.logger.debug("%s sets its profile to %s" % (client, profile)) try: self.metadata.set_profile(client, profile, address) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to assert profile for %s: %s" % (client, err)) @@ -1103,7 +1118,7 @@ class BaseCore(object): config = self.BuildConfiguration(client) return lxml.etree.tostring(config, xml_declaration=False).decode('UTF-8') - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.critical_error("Metadata consistency failure for %s" % client) @exposed diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 37bc230d1..ae7c75804 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -40,7 +40,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", - "SSLCA/**/key.xml": "sslca-key.xsd" + "SSLCA/**/key.xml": "sslca-key.xsd", + "GroupLogic/groups.xml": "grouplogic.xsd" } self.filelists = {} @@ -83,17 +84,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): else: self.LintError("properties-schema-not-found", "No schema found for %s" % filename) + # ensure that it at least parses + self.parse(filename) - def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ - if schema is None: - # if no schema object was provided, instantiate one - schema = self._load_schema(schemafile) - if not schema: - return False + def parse(self, filename): + """ Parse an XML file, raising the appropriate LintErrors if + it can't be parsed or read. Return the + lxml.etree._ElementTree parsed from the file. """ try: - datafile = lxml.etree.parse(filename) + return lxml.etree.parse(filename) except SyntaxError: lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) self.LintError("xml-failed-to-parse", @@ -106,6 +105,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def validate(self, filename, schemafile, schema=None): + """validate a file against the given lxml.etree.Schema. + return True on success, False on failure """ + if schema is None: + # if no schema object was provided, instantiate one + schema = self._load_schema(schemafile) + if not schema: + return False + datafile = self.parse(filename) if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 2b878d7e2..b14968d77 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -530,8 +530,8 @@ class XMLFileBacked(FileBacked): #: XInclude. self.extra_monitors = [] - if ((create or (self.create is not None and self.create)) - and not os.path.exists(self.name)): + if ((create is not None or self.create not in [None, False]) and + not os.path.exists(self.name)): toptag = create or self.create self.logger.warning("%s does not exist, creating" % self.name) if hasattr(toptag, "getroottree"): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 581a997d8..c7b62f352 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -69,7 +69,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): the given client metadata, and may be obtained by doing ``self.XMLMatch(metadata)`` :type spec: lxml.etree._Element - :returns: None + :returns: string - The filename of the private key """ if spec is None: spec = self.XMLMatch(metadata) @@ -140,7 +140,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if spec is None: spec = self.XMLMatch(metadata) category = spec.get("category", self.category) - print("category=%s" % category) if category is None: per_host_default = "true" else: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 926172e57..ffe93c25b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -599,6 +599,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, else: try: if not isinstance(data, unicode): + if not isinstance(data, str): + data = data.decode('utf-8') data = u_str(data, self.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index c8362db41..44971aba7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -44,7 +44,7 @@ class Git(Version): else: cmd = ["git", "--git-dir", self.vcs_path, "--work-tree", self.vcs_root, "rev-parse", "HEAD"] - self.debug_log("Git: Running cmd") + self.debug_log("Git: Running %s" % cmd) proc = Popen(cmd, stdout=PIPE, stderr=PIPE) rv, err = proc.communicate() if proc.wait(): diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py new file mode 100644 index 000000000..810b273af --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -0,0 +1,47 @@ +""" GroupLogic is a connector plugin that lets you use an XML Genshi +template to dynamically set additional groups for clients. """ + +import os +import lxml.etree +import Bcfg2.Server.Plugin +try: + from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile +except ImportError: + # BundleTemplateFile missing means that genshi is missing. we + # import genshi to get the _real_ error + import genshi # pylint: disable=W0611 + + +class GroupLogicConfig(BundleTemplateFile): + """ Representation of the GroupLogic groups.xml file """ + create = lxml.etree.Element("GroupLogic", + nsmap=dict(py="http://genshi.edgewall.org/")) + + def __init__(self, name, fam): + BundleTemplateFile.__init__(self, name, + Bcfg2.Server.Plugin.Specificity(), None) + self.fam = fam + self.should_monitor = True + self.fam.AddMonitor(self.name, self) + + def _match(self, item, metadata): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return BundleTemplateFile._match(self, item, metadata) + + +class GroupLogic(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + """ GroupLogic is a connector plugin that lets you use an XML + Genshi template to dynamically set additional groups for + clients. """ + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), + core.fam) + + def get_additional_groups(self, metadata): + return [el.get("name") + for el in self.config.get_xml_value(metadata).findall("Group")] diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index bdf3b87fe..71e81c1fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - def set_profile(self, client, profile, addresspair): + def set_profile(self, client, profile, # pylint: disable=W0221 + addresspair, require_public=True): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) @@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] - if not group.is_public: + if require_public and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) @@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = self.default if pgroup: - self.set_profile(client, pgroup, (None, None)) + self.set_profile(client, pgroup, (None, None), + require_public=False) profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 4eefd0722..bc2928fa6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -93,6 +93,8 @@ class AptSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') words = str(line.strip()).split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index b4d481459..7ba374dd3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -315,7 +315,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: OSError - If the saved data cannot be read :raises: cPickle.UnpicklingError - If the saved data is corrupt """ - data = open(self.cachefile) + data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, self.essentialpkgs) = cPickle.load(data) @@ -615,7 +615,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'w').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4f163a1ab..efbca28cd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -18,7 +18,8 @@ from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" #: The default path for generated apt configs -APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2" +APT_CONFIG_DEFAULT = \ + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" class Packages(Bcfg2.Server.Plugin.Plugin, diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index a8001d9af..309b96475 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -63,7 +63,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + return str.__new__(cls, data.encode('utf-8')) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -153,7 +153,20 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe = lxml.etree.Element('probe') probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) - probe.text = entry.data + if (metadata.version_info and + metadata.version_info > (1, 3, 1, '', 0)): + try: + probe.text = entry.data.decode('utf-8') + except AttributeError: + probe.text = entry.data + else: + try: + probe.text = entry.data + except: # pylint: disable=W0702 + self.logger.error("Client unable to handle unicode " + "probes. Skipping %s" % + probe.get('name')) + continue match = self.bangline.match(entry.data.split('\n')[0]) if match: probe.set('interpreter', match.group('interpreter')) @@ -209,8 +222,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement(ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str( + self.probedata[client][probe]).decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 3bd6fd14f..3354763d4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -96,7 +96,7 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): client.hostname, cdata, lxml.etree.tostring( stats, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False)) self.debug_log("%s: Queued statistics data for %s" % (self.__class__.__name__, client.hostname)) return diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index fc07a90e9..5aa7c4d9e 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -172,7 +172,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for name in names[cmeta.hostname]: newnames.add(name.split('.')[0]) try: - newips.add(self.get_ipcache_entry(name)[0]) + newips.update(self.get_ipcache_entry(name)[0]) except: # pylint: disable=W0702 continue names[cmeta.hostname].update(newnames) @@ -288,7 +288,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: # need to add entry try: - ipaddr = socket.gethostbyname(client) + ipaddr = set([addr[0] for (_, _, _, _, addr) in socket.getaddrinfo(client, None)]) self.ipcache[client] = (ipaddr, client) return (ipaddr, client) except socket.gaierror: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 7d00201da..f111ffc60 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -68,7 +68,7 @@ class SSLCACertSpec(SSLCAXMLSpec): def get_spec(self, metadata): rv = SSLCAXMLSpec.get_spec(self, metadata) rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "SubjectAltName"] + if e.tag == "subjectAltName"] return rv diff --git a/src/lib/Bcfg2/Statistics.py b/src/lib/Bcfg2/Statistics.py index a869b03cd..3825941af 100644 --- a/src/lib/Bcfg2/Statistics.py +++ b/src/lib/Bcfg2/Statistics.py @@ -28,10 +28,10 @@ class Statistic(object): :param value: The value to add to this statistic :type value: int or float """ - self.min = min(self.min, value) - self.max = max(self.max, value) - self.ave = (((self.ave * (self.count - 1)) + value) / self.count) + self.min = min(self.min, float(value)) + self.max = max(self.max, float(value)) self.count += 1 + self.ave = (((self.ave * (self.count - 1)) + value) / self.count) def get_value(self): """ Get a tuple of all the stats tracked on this named item. @@ -46,6 +46,11 @@ class Statistic(object): """ return (self.name, (self.min, self.max, self.ave, self.count)) + def __repr__(self): + return "%s(%s, (min=%s, avg=%s, max=%s, count=%s))" % ( + self.__class__.__name__, + self.name, self.min, self.ave, self.max, self.count) + class Statistics(object): """ A collection of named :class:`Statistic` objects. """ diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 39cf5255e..581445bf4 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -108,10 +108,16 @@ class ExecutorResult(object): def __init__(self, stdout, stderr, retval): #: The output of the command - self.stdout = stdout + if isinstance(stdout, str): + self.stdout = stdout + else: + self.stdout = stdout.decode('utf-8') #: The error produced by the command - self.stderr = stderr + if isinstance(stdout, str): + self.stderr = stderr + else: + self.stderr = stderr.decode('utf-8') #: The return value of the command. self.retval = retval @@ -234,6 +240,13 @@ class Executor(object): for line in inputdata.splitlines(): self.logger.debug('> %s' % line) (stdout, stderr) = proc.communicate(input=inputdata) + + # py3k fixes + if not isinstance(stdout, str): + stdout = stdout.decode('utf-8') + if not isinstance(stderr, str): + stderr = stderr.decode('utf-8') + for line in stdout.splitlines(): # pylint: disable=E1103 self.logger.debug('< %s' % line) for line in stderr.splitlines(): # pylint: disable=E1103 diff --git a/src/lib/Bcfg2/__init__.py b/src/lib/Bcfg2/__init__.py index 41743d100..74a871f2a 100644 --- a/src/lib/Bcfg2/__init__.py +++ b/src/lib/Bcfg2/__init__.py @@ -1,4 +1 @@ """Base modules definition.""" - -from Bcfg2.Compat import walk_packages -__all__ = [m[1] for m in walk_packages(path=__path__)] diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 7d405f868..9393830a8 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -54,11 +54,11 @@ DEFAULT_CONFIG = _default_config() def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ - # pylint: disable=W0603 + # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ MEDIA_URL - # pylint: enable=W0603 + # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): print("%s does not exist, using %s for database configuration" % |