From 8938b4025400d150db4b6fb32181cd36c294e07c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 4 May 2011 13:35:38 -0500 Subject: SSHbase: PY3K string join method fix Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/SSHbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index ce08235eb..a8bf596de 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -73,7 +73,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([value.data for key, value in \ + self.__skn = "\n".join([str(value.data) for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() -- cgit v1.2.3-1-g7c22 From 071ef1a1fbe6368b1abb81855e1ab95e316e6911 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 08:35:54 -0500 Subject: TGenshi: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TGenshi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index 83b60c958..bc5e00400 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -3,7 +3,11 @@ __revision__ = '$Revision$' import binascii import logging +import sys import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TGenshi') @@ -76,9 +80,6 @@ class TemplateFile: def bind_entry(self, entry, metadata): """Build literal file information.""" fname = entry.get('realname', entry.get('name')) - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if entry.tag == 'Path': entry.set('type', 'file') try: -- cgit v1.2.3-1-g7c22 From c7f6cb353df45bb3662b78a49815a6b1a54a7f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 13 Apr 2011 15:16:06 +0200 Subject: Cfg: Fix the output encoding of Genshi templates Encode the configuration files generated from Genshi templates according to the encoding setting from Options.py instead of unconditionally using UTF-8. --- src/lib/Server/Plugins/Cfg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 41cf6c9c1..a97cbd550 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -134,9 +134,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): metadata=metadata, path=basefile.name).filter(removecomment) try: - data = stream.render('text', strip_whitespace=False) + data = stream.render('text', encoding=self.encoding, + strip_whitespace=False) except TypeError: - data = stream.render('text') + data = stream.render('text', encoding=self.encoding) if data == '': entry.set('empty', 'true') except Exception: -- cgit v1.2.3-1-g7c22 From af8bf14ee0079649d4583fb0850ec25b74242344 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 12:46:57 -0500 Subject: TCheetah: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TCheetah.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py index 151cc6543..49be88881 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Server/Plugins/TCheetah.py @@ -6,6 +6,9 @@ import logging import sys import traceback import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TCheetah') @@ -53,9 +56,6 @@ class TemplateFile: if entry.tag == 'Path': entry.set('type', 'file') try: - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if type(self.template) == unicode: entry.text = self.template else: -- cgit v1.2.3-1-g7c22 From 88249702412125c36179f5ca9f65b45aa0dc42a5 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:53:00 -0400 Subject: Add missing import sys. --- src/lib/Server/Plugins/SSHbase.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index a8bf596de..cf0998aaa 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -5,6 +5,7 @@ import binascii import os import socket import shutil +import sys import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From e810fa812c23e87fc43908f8f72c4c6d751df625 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:47:46 +0200 Subject: add missing sys import --- src/lib/Server/Plugins/Ldap.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index b904dbe02..7d6d0b609 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,4 +1,5 @@ import imp +import sys import time import ldap import Bcfg2.Options -- cgit v1.2.3-1-g7c22 From 31cfa9f3a7b9f4ffd5fa32e22042681015489149 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:54:42 +0200 Subject: log more useful error message if python-ldap is not installed --- src/lib/Server/Plugins/Ldap.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index 7d6d0b609..06ecaed7b 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,10 +1,18 @@ import imp +import logging import sys import time -import ldap import Bcfg2.Options import Bcfg2.Server.Plugin +logger = logging.getLogger('Bcfg2.Plugins.Ldap') + +try: + import ldap +except: + logger.error("Unable to load ldap module. Is python-ldap installed?") + raise ImportError + # time in seconds between retries after failed LDAP connection RETRY_DELAY = 5 # how many times to try reaching the LDAP server if a connection is broken -- cgit v1.2.3-1-g7c22 From 0326069de94cdc8300a8b917ae7c440f02793d6e Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:32:44 +0200 Subject: add another missing sys import --- src/lib/Server/Plugins/SGenshi.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py index 04942c2bd..efd981956 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Server/Plugins/SGenshi.py @@ -5,6 +5,7 @@ import genshi.input import genshi.template import lxml.etree import logging +import sys import Bcfg2.Server.Plugin import Bcfg2.Server.Plugins.TGenshi -- cgit v1.2.3-1-g7c22 From 92d4cfc6e8e2225b3b8d00578d83397313983c0b Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:53:44 +0200 Subject: add yet another missing sys import --- src/lib/Server/Plugins/Bundler.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 1a8e7348b..01ad3c78b 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import copy import lxml.etree import re +import sys import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From fa70c550a90c369ac48862d976e3d4294181be89 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 09:37:50 -0400 Subject: Improved Svn2 error handling -- ClientError may not have a message attribute. --- src/lib/Server/Plugins/Svn2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py index 35f555294..b8a8e6b7e 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Server/Plugins/Svn2.py @@ -75,9 +75,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: @@ -95,9 +95,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: -- cgit v1.2.3-1-g7c22 From 34e5e12fd58ec4b0015abdf1a04a4f684c9194ba Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 10:30:33 -0400 Subject: Added FileProbes plugin. --- src/lib/Server/Plugins/FileProbes.py | 177 +++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 src/lib/Server/Plugins/FileProbes.py (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py new file mode 100644 index 000000000..4df5a7f4a --- /dev/null +++ b/src/lib/Server/Plugins/FileProbes.py @@ -0,0 +1,177 @@ +""" This module allows you to probe a client for a file, which is then +added to the specification. On subsequent runs, the file will be +replaced on the client if it is missing; if it has changed on the +client, it can either be updated in the specification or replaced on +the client """ +__revision__ = '$Revision: 1465 $' + +import os +import errno +import binascii +import lxml.etree +import Bcfg2.Options +import Bcfg2.Server.Plugin + +probecode = """#!/usr/bin/env python + +import os +import pwd +import grp +import binascii +import lxml.etree + +path = "%s" + +if not os.path.exists(path): + print "%%s does not exist" %% path + raise SystemExit(1) + +stat = os.stat(path) +data = lxml.etree.Element("ProbedFileData", + name=path, + owner=pwd.getpwuid(stat[4])[0], + group=grp.getgrgid(stat[5])[0], + perms=oct(stat[0] & 07777)) +data.text = binascii.b2a_base64(open(path).read()) +print lxml.etree.tostring(data) +""" + +class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, + Bcfg2.Server.Plugin.StructFile): + """ Config file handler for FileProbes """ + def __init__(self, filename, fam): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename) + + +class FileProbes(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Probing): + """ This module allows you to probe a client for a file, which is then + added to the specification. On subsequent runs, the file will be + replaced on the client if it is missing; if it has changed on the + client, it can either be updated in the specification or replaced on + the client """ + + name = 'FileProbes' + experimental = True + __version__ = '$Id$' + __author__ = 'stpierreca@ornl.gov' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Probing.__init__(self) + self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'), + core.fam) + self.entries = dict() + self.probes = dict() + + def GetProbes(self, metadata): + """Return a set of probes for execution on client.""" + if metadata.hostname not in self.probes: + cfg = self.core.plugins['Cfg'] + self.entries[metadata.hostname] = dict() + self.probes[metadata.hostname] = [] + for entry in self.config.Match(metadata): + path = entry.get("name") + # do not probe for files that are already in Cfg and + # for which update is false; we can't possibly do + # anything with the data we get from such a probe + try: + if (cfg.entries[path].get_pertinent_entries(metadata) and + entry.get('update', 'false').lower() == "false"): + continue + except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError): + pass + self.entries[metadata.hostname][path] = entry + probe = lxml.etree.Element('probe', name=path, + source=self.name, + interpreter="/usr/bin/env python") + probe.text = probecode % path + self.probes[metadata.hostname].append(probe) + self.logger.debug("Adding file probe for %s to %s" % + (path, metadata.hostname)) + return self.probes[metadata.hostname] + + def ReceiveData(self, metadata, datalist): + """Receive data from probe.""" + self.logger.debug("Receiving file probe data from %s" % + metadata.hostname) + + for data in datalist: + if data.text is None: + self.logger.error("Got null response to %s file probe from %s" % + (data.get('name'), metadata.hostname)) + else: + self.logger.debug("%s:fileprobe:%s:%s" % + (metadata.hostname, + data.get("name"), + data.text)) + try: + filedata = lxml.etree.XML(data.text) + self.write_file(filedata, metadata) + except lxml.etree.XMLSyntaxError: + # if we didn't get XML back from the probe, assume + # it's an error message + self.logger.error(data.text) + + def write_file(self, data, metadata): + """Write the probed file data to the bcfg2 specification.""" + filename = data.get("name") + contents = binascii.a2b_base64(data.text) + entry = self.entries[metadata.hostname][filename] + cfg = self.core.plugins['Cfg'] + specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) + # we can't use os.path.join() for this because specific + # already has a leading /, which confuses os.path.join() + fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific)) + if filename not in cfg.entries.keys(): + self.logger.info("Writing new probed file %s" % fileloc) + try: + os.makedirs(os.path.dirname(fileloc)) + except OSError, err: + if err.errno == errno.EEXIST: + pass + else: + raise + open(fileloc, 'wb').write(contents) + + infoxml = os.path.join("%s%s" % (cfg.data, filename), + "info.xml") + self.write_infoxml(infoxml, entry, data) + else: + try: + cfgentry = \ + cfg.entries[filename].get_pertinent_entries(metadata)[0] + except Bcfg2.Server.Plugin.PluginExecutionError: + self.logger.info("Writing new probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + return + + if cfgentry.data == contents: + self.logger.debug("Existing %s contents match probed contents" % + filename) + elif (entry.get('update', 'false').lower() == "true"): + self.logger.info("Writing updated probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + else: + self.logger.info("Skipping updated probed file %s" % fileloc) + + def write_infoxml(self, infoxml, entry, data): + """ write an info.xml for the file """ + self.logger.info("Writing info.xml at %s for %s" % + (infoxml, data.get("name"))) + info = \ + lxml.etree.Element("Info", + owner=data.get("owner", + Bcfg2.Options.MDATA_OWNER.value), + group=data.get("group", + Bcfg2.Options.MDATA_GROUP.value), + perms=data.get("perms", + Bcfg2.Options.MDATA_PERMS.value), + encoding=entry.get("encoding", + Bcfg2.Options.ENCODING.value)) + + root = lxml.etree.Element("FileInfo") + root.append(info) + open(infoxml, "w").write(lxml.etree.tostring(root, + pretty_print=True)) -- cgit v1.2.3-1-g7c22 From a408e7cbe36beacc2aefe291ac3f5caec36ddf35 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Wed, 11 May 2011 11:37:18 -0500 Subject: Cfg: Fix PluginExecutionError Replaces PluginExecutionError with Bcfg2.Server.Plugin.PluginExecutionError. Reported by emias. --- src/lib/Server/Plugins/Cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index a97cbd550..7fc35cec1 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -188,7 +188,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): name = self.build_filename(specific) if name.endswith(".genshi"): logger.error("Cfg: Unable to pull data for genshi types") - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) if log: logger.info("Wrote file %s" % name) -- cgit v1.2.3-1-g7c22 From 34d9c0652dbbaaf28fbc311ef168e29fc58155f7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 13 May 2011 13:10:00 -0500 Subject: Cfg: Fix bcfg2-admin pull behavior for genshi templates (#1010) Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 7fc35cec1..832f7ab41 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -5,6 +5,7 @@ import binascii import logging import lxml import os +import os.path import re import sys import tempfile @@ -186,7 +187,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): def write_update(self, specific, new_entry, log): if 'text' in new_entry: name = self.build_filename(specific) - if name.endswith(".genshi"): + if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) -- cgit v1.2.3-1-g7c22 From 99495c559da8bb432274e94248f98d9b34c2d205 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 31 May 2011 10:36:15 -0500 Subject: Cfg: Fix traceback for non-ascii files Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 832f7ab41..5e3cca847 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -162,6 +162,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) logger.error("Please verify you are using the proper encoding.") raise Bcfg2.Server.Plugin.PluginExecutionError + except ValueError: + e = sys.exc_info()[1] + logger.error("Error in specification for %s" % entry.get('name')) + logger.error("%s" % e) + logger.error("You need to specify base64 encoding for %s." % + entry.get('name')) + raise Bcfg2.Server.Plugin.PluginExecutionError if entry.text in ['', None]: entry.set('empty', 'true') -- cgit v1.2.3-1-g7c22 From 368b34bcdd509d24bcc82ccbdbd056455bba7c37 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 3 Jun 2011 19:10:18 -0500 Subject: Base: Deprecate Base in favor of Bundler Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Base.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 5e7d89727..5de57a87c 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = Bcfg2.Server.Plugin.StructFile + deprecated = True """Base creates independent clauses based on client metadata.""" def __init__(self, core, datastore): -- cgit v1.2.3-1-g7c22 From b2e9d11b0dca1af6e0ce9a9f21558b35b35f5777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 16:32:53 +0200 Subject: Accept non-ASCII diffs Currently, client reports don't include diffs of files which aren't US-ASCII encoded. The client transmits such files as Base64 blobs. As we'd like to change that, this commit teaches the server to properly handle non-ASCII diffs. --- src/lib/Server/Plugins/Cfg.py | 7 ++++++- src/lib/Server/Plugins/DBStats.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 5e3cca847..c08e8c4b6 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -197,7 +197,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(new_entry['text']) + try: + etext = new_entry['text'].encode(self.encoding) + except: + logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) + raise Bcfg2.Server.Plugin.PluginExecutionError + open(name, 'w').write(etext) if log: logger.info("Wrote file %s" % name) badattr = [attr for attr in ['owner', 'group', 'perms'] diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py index 5ef1920e1..103fb7353 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Server/Plugins/DBStats.py @@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, try: Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, container, + self.core.encoding, 0, logger, True, -- cgit v1.2.3-1-g7c22 From b1ab3e2bab9f07c13daf5dcfd4a9502eb84dcf0d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 14 Jun 2011 15:23:31 -0500 Subject: PY3K: Finish server-side code fixes Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 5 +++-- src/lib/Server/Plugins/Metadata.py | 2 +- src/lib/Server/Plugins/Probes.py | 5 +++-- src/lib/Server/Plugins/SSHbase.py | 21 +++++++++++++-------- 4 files changed, 20 insertions(+), 13 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c08e8c4b6..4a0fd2dfe 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import binascii import logging import lxml +import operator import os import os.path import re @@ -25,7 +26,7 @@ logger = logging.getLogger('Bcfg2.Plugins.Cfg') def u_str(string, encoding): if sys.hexversion >= 0x03000000: - return str(string, encoding) + return string.encode(encoding) else: return unicode(string, encoding) @@ -105,7 +106,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] - matching.sort(self.sort_by_specific) + matching.sort(key=operator.attrgetter('specific')) non_delta = [matching.index(m) for m in matching if not m.specific.delta] if not non_delta: diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ca6e43851..6570f2912 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (address[0])) return False # populate the session cache - if user != 'root': + if user.decode('utf-8') != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index ea2e79ccc..3599f2af1 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -1,4 +1,5 @@ import lxml.etree +import operator import re import Bcfg2.Server.Plugin @@ -27,7 +28,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret = [] build = dict() candidates = self.get_matching(metadata) - candidates.sort(lambda x, y: cmp(x.specific, y.specific)) + candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: rem = specific_probe_matcher.match(entry.name) if not rem: @@ -90,7 +91,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data) + datafile.write(data.decode('utf-8')) def load_data(self): try: diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cf0998aaa..4a33c0cb0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", - "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"] + "ssh_host_rsa_key.pub.H_%s", + "ssh_host_key.pub.H_%s"] hostkeys = ["ssh_host_dsa_key.H_%s", - "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"] - keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key', - 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub', - 'ssh_host_key.pub'] + "ssh_host_rsa_key.H_%s", + "ssh_host_key.H_%s"] + keypatterns = ["ssh_host_dsa_key", + "ssh_host_rsa_key", + "ssh_host_key", + "ssh_host_dsa_key.pub", + "ssh_host_rsa_key.pub", + "ssh_host_key.pub"] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([str(value.data) for key, value in \ + self.__skn = "\n".join([value.data.decode() for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() @@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname) continue self.__skn += "%s %s" % (','.join(names[hostname]), - self.entries[pubkey].data) + self.entries[pubkey].data.decode()) return self.__skn def set_skn(self, value): @@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, hostkeys.sort() for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - self.entries[hostkey].data) + self.entries[hostkey].data.decode()) permdata = {'owner': 'root', 'group': 'root', 'type': 'file', -- cgit v1.2.3-1-g7c22 From a1bc75c200fb9237ecc0dd5ccd5b8bf94214cf4b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 09:23:19 -0500 Subject: PY3K: Add compatibility comments Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 1 + src/lib/Server/Plugins/Snapshots.py | 1 + 2 files changed, 2 insertions(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 4a0fd2dfe..c93b76488 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -24,6 +24,7 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') +# py3k compatibility def u_str(string, encoding): if sys.hexversion >= 0x03000000: return string.encode(encoding) diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index 8b6bad574..aeb3b9f74 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -28,6 +28,7 @@ datafields = { } +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string -- cgit v1.2.3-1-g7c22 From ac578f15ca785d9b1fcd0d42bfa3ffd017985e78 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 11:32:29 -0400 Subject: make Bcfg2 automatically handle JSON, XML, and YAML output from probes --- src/lib/Server/Plugins/Probes.py | 93 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 3599f2af1..4a7171297 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -2,12 +2,96 @@ import lxml.etree import operator import re +try: + import json + has_json = True +except ImportError: + has_json = False + +try: + import syck + has_syck = True +except ImportError: + has_syck = False + +try: + import yaml + has_yaml = True +except ImportError: + has_yaml = False + import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P\S+)(.(?P[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P\S+)") +class ProbeData (object): + """ a ProbeData object emulates a str object, but also has .xdata + and .json properties to provide convenient ways to use ProbeData + objects as XML or JSON data """ + def __init__(self, data): + self.data = data + self._xdata = None + self._json = None + self._yaml = None + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __getattr__(self, name): + """ make ProbeData act like a str object """ + return getattr(self.data, name) + + def __complex__(self): + return complex(self.data) + + def __int__(self): + return int(self.data) + + def __long__(self): + return long(self.data) + + def __float__(self): + return float(self.data) + + @property + def xdata(self): + if self._xdata is None: + try: + self._xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + pass + return self._xdata + + @property + def json(self): + if self._json is None and has_json: + try: + self._json = json.loads(self.data) + except ValueError: + pass + return self._json + + @property + def yaml(self): + if self._yaml is None: + if has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml.YAMLError: + pass + elif has_syck: + try: + self._yaml = syck.load(self.data) + except syck.error: + pass + return self._yaml + + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") @@ -106,7 +190,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') + self.probedata[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get('value')) elif (pdata.tag == 'Group'): self.cgroups[client.get('name')].append(pdata.get('name')) @@ -142,11 +227,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, if newgroup not in self.cgroups[client.hostname]: self.cgroups[client.hostname].append(newgroup) dlines.remove(line) - dtext = "\n".join(dlines) + dobj = ProbeData("\n".join(dlines)) try: - self.probedata[client.hostname].update({data.get('name'): dtext}) + self.probedata[client.hostname].update({data.get('name'): dobj}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): dtext} + self.probedata[client.hostname] = {data.get('name'): dobj} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) -- cgit v1.2.3-1-g7c22 From 5f93d780fc2dcd6ba35179acd61c05754d1e4fbc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 12:21:17 -0400 Subject: made StructFile.Match() work with Group/Client tags inside other tags --- src/lib/Server/Plugins/Properties.py | 45 ++++++++---------------------------- 1 file changed, 9 insertions(+), 36 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index dea797a10..95565f2e4 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -7,42 +7,15 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" def Index(self): - """Build internal data structures.""" - if type(self.data) is not lxml.etree._Element: - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % - self.name) - - self.fragments = {} - work = {lambda x: True: self.data.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + """Build local data structures.""" + try: + xdata = XML(self.data) + except XMLSyntaxError: + self.label = None + self.entries = [] + return + self.label = xdata.attrib[self.__identifier__] + self.entries = xdata.getchildren() class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): -- cgit v1.2.3-1-g7c22 From bc1a6b8d0a46e37a108a752a7b6f54e637ff804d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 13:22:19 -0400 Subject: bug fixes --- src/lib/Server/Plugins/Properties.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index 95565f2e4..54c5def57 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,17 +6,7 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - def Index(self): - """Build local data structures.""" - try: - xdata = XML(self.data) - except XMLSyntaxError: - self.label = None - self.entries = [] - return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() - + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile -- cgit v1.2.3-1-g7c22 From dfee52f047c4d50b9b7120b4310c11397d1a0620 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 15:04:04 -0400 Subject: cleaned up some ProbeData stuff, make null probe storage more consistent --- src/lib/Server/Plugins/Probes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 4a7171297..b07c4dfd3 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -165,7 +165,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, cx = lxml.etree.SubElement(top, 'Client', name=client) for probe in sorted(probed): lxml.etree.SubElement(cx, 'Probe', name=probe, - value=self.probedata[client][probe]) + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) data = lxml.etree.tostring(top, encoding='UTF-8', @@ -214,9 +214,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.logger.error("Got null response to probe %s from %s" % \ (data.get('name'), client.hostname)) try: - self.probedata[client.hostname].update({data.get('name'): ''}) + self.probedata[client.hostname].update({data.get('name'): + ProbeData('')}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): ''} + self.probedata[client.hostname] = \ + {data.get('name'): ProbeData('')} return dlines = data.text.split('\n') self.logger.debug("%s:probe:%s:%s" % (client.hostname, -- cgit v1.2.3-1-g7c22 From 3ec2fc27deefc647127c9e729221826a90fd7a96 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 17 Jun 2011 08:03:26 -0400 Subject: Improved handling of JSON data from probes --- src/lib/Server/Plugins/Probes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index b07c4dfd3..ec0f294dd 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -6,19 +6,22 @@ try: import json has_json = True except ImportError: - has_json = False + try: + import simplejson as json + has_json = True + except ImportError: + has_json = False try: import syck has_syck = True except ImportError: has_syck = False - -try: - import yaml - has_yaml = True -except ImportError: - has_yaml = False + try: + import yaml + has_yaml = True + except ImportError: + has_yaml = False import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 19586bc42aa90543cf27e33ec32bd7df222138e8 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 20 Jun 2011 12:07:16 -0500 Subject: Cfg: Add support for perms='inherit' (Ticket #642) This feature allows you to use the on-disk permissions of the file in the Cfg repository rather than specifying them using the traditional means in info.xml. Note that this only works for the octal permissions of the file on disk since the owner/group may not exist on the destination machine. Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c93b76488..23ba0a745 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -8,6 +8,7 @@ import operator import os import os.path import re +import stat import sys import tempfile @@ -97,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) self.specific = CfgMatcher(path.split('/')[-1]) + path = path def sort_by_specific(self, one, other): return cmp(one.specific, other.specific) @@ -121,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): self.bind_info_to_entry(entry, metadata) used = self.get_pertinent_entries(metadata) basefile = used.pop(0) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = "%s/%s" % (self.path, entry.get('name')) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): -- cgit v1.2.3-1-g7c22