From 8938b4025400d150db4b6fb32181cd36c294e07c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 4 May 2011 13:35:38 -0500 Subject: SSHbase: PY3K string join method fix Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/SSHbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index ce08235eb..a8bf596de 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -73,7 +73,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([value.data for key, value in \ + self.__skn = "\n".join([str(value.data) for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() -- cgit v1.2.3-1-g7c22 From 071ef1a1fbe6368b1abb81855e1ab95e316e6911 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 08:35:54 -0500 Subject: TGenshi: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TGenshi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index 83b60c958..bc5e00400 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -3,7 +3,11 @@ __revision__ = '$Revision$' import binascii import logging +import sys import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TGenshi') @@ -76,9 +80,6 @@ class TemplateFile: def bind_entry(self, entry, metadata): """Build literal file information.""" fname = entry.get('realname', entry.get('name')) - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if entry.tag == 'Path': entry.set('type', 'file') try: -- cgit v1.2.3-1-g7c22 From c7f6cb353df45bb3662b78a49815a6b1a54a7f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 13 Apr 2011 15:16:06 +0200 Subject: Cfg: Fix the output encoding of Genshi templates Encode the configuration files generated from Genshi templates according to the encoding setting from Options.py instead of unconditionally using UTF-8. --- src/lib/Server/Plugins/Cfg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 41cf6c9c1..a97cbd550 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -134,9 +134,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): metadata=metadata, path=basefile.name).filter(removecomment) try: - data = stream.render('text', strip_whitespace=False) + data = stream.render('text', encoding=self.encoding, + strip_whitespace=False) except TypeError: - data = stream.render('text') + data = stream.render('text', encoding=self.encoding) if data == '': entry.set('empty', 'true') except Exception: -- cgit v1.2.3-1-g7c22 From af8bf14ee0079649d4583fb0850ec25b74242344 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 12:46:57 -0500 Subject: TCheetah: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TCheetah.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py index 151cc6543..49be88881 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Server/Plugins/TCheetah.py @@ -6,6 +6,9 @@ import logging import sys import traceback import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TCheetah') @@ -53,9 +56,6 @@ class TemplateFile: if entry.tag == 'Path': entry.set('type', 'file') try: - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if type(self.template) == unicode: entry.text = self.template else: -- cgit v1.2.3-1-g7c22 From 417f502abe96ab7d784cbc872aeb65add9a509ba Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:43:43 -0400 Subject: Add missing import. --- src/lib/Server/Admin/Init.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index e69412a5e..f450e5303 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -4,6 +4,7 @@ import random import socket import stat import string +import sys import subprocess import Bcfg2.Server.Admin import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 4838c1477108fd61b2f65f79f8e3d31776cd1ba6 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:48:49 -0400 Subject: Pass keypath to create_conf function. --- src/lib/Server/Admin/Init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index f450e5303..fff8bcd1c 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -142,7 +142,7 @@ def create_key(hostname, keypath, certpath, country, state, location): os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 -def create_conf(confpath, confdata): +def create_conf(confpath, confdata, keypath): # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): # py3k compatibility @@ -402,7 +402,7 @@ class Init(Bcfg2.Server.Admin.Mode): self.server_uri) # Create the configuration file and SSL key - create_conf(self.configfile, confdata) + create_conf(self.configfile, confdata, keypath) kpath = keypath + '/bcfg2.key' cpath = keypath + '/bcfg2.crt' create_key(self.shostname, kpath, cpath, self.country, -- cgit v1.2.3-1-g7c22 From 88249702412125c36179f5ca9f65b45aa0dc42a5 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:53:00 -0400 Subject: Add missing import sys. --- src/lib/Server/Plugins/SSHbase.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index a8bf596de..cf0998aaa 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -5,6 +5,7 @@ import binascii import os import socket import shutil +import sys import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From e810fa812c23e87fc43908f8f72c4c6d751df625 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:47:46 +0200 Subject: add missing sys import --- src/lib/Server/Plugins/Ldap.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index b904dbe02..7d6d0b609 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,4 +1,5 @@ import imp +import sys import time import ldap import Bcfg2.Options -- cgit v1.2.3-1-g7c22 From 31cfa9f3a7b9f4ffd5fa32e22042681015489149 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:54:42 +0200 Subject: log more useful error message if python-ldap is not installed --- src/lib/Server/Plugins/Ldap.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index 7d6d0b609..06ecaed7b 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,10 +1,18 @@ import imp +import logging import sys import time -import ldap import Bcfg2.Options import Bcfg2.Server.Plugin +logger = logging.getLogger('Bcfg2.Plugins.Ldap') + +try: + import ldap +except: + logger.error("Unable to load ldap module. Is python-ldap installed?") + raise ImportError + # time in seconds between retries after failed LDAP connection RETRY_DELAY = 5 # how many times to try reaching the LDAP server if a connection is broken -- cgit v1.2.3-1-g7c22 From 9ec715f1e25f5274d6472536ce18a899358d3e66 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 25 Apr 2011 21:33:46 -0500 Subject: DBStats: Stop duplicating data in reports_reason --- src/lib/Server/Reports/importscript.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 1781e2fac..b6a3c2599 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -133,12 +133,9 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): try: rr = None - if not quick: - try: - rr = Reason.objects.filter(**kargs)[0] - except IndexError: - pass - if not rr: + try: + rr = Reason.objects.filter(**kargs)[0] + except IndexError: rr = Reason(**kargs) rr.save() if vlevel > 0: -- cgit v1.2.3-1-g7c22 From 0326069de94cdc8300a8b917ae7c440f02793d6e Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:32:44 +0200 Subject: add another missing sys import --- src/lib/Server/Plugins/SGenshi.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py index 04942c2bd..efd981956 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Server/Plugins/SGenshi.py @@ -5,6 +5,7 @@ import genshi.input import genshi.template import lxml.etree import logging +import sys import Bcfg2.Server.Plugin import Bcfg2.Server.Plugins.TGenshi -- cgit v1.2.3-1-g7c22 From 92d4cfc6e8e2225b3b8d00578d83397313983c0b Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:53:44 +0200 Subject: add yet another missing sys import --- src/lib/Server/Plugins/Bundler.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 1a8e7348b..01ad3c78b 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import copy import lxml.etree import re +import sys import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From fa70c550a90c369ac48862d976e3d4294181be89 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 09:37:50 -0400 Subject: Improved Svn2 error handling -- ClientError may not have a message attribute. --- src/lib/Server/Plugins/Svn2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py index 35f555294..b8a8e6b7e 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Server/Plugins/Svn2.py @@ -75,9 +75,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: @@ -95,9 +95,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: -- cgit v1.2.3-1-g7c22 From 564a0ae955286194094ea5e6eda0afb7f755ca91 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 10:12:45 -0400 Subject: Fixed error messages from info.xml bcfg2-lint check --- src/lib/Server/Lint/InfoXML.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 7725ad748..c88e54e95 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -13,12 +13,13 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if self.HandlesFile(infoxml_fname): if (hasattr(entryset, "infoxml") and entryset.infoxml is not None): - self.check_infoxml(entryset.infoxml.pnode.data) + self.check_infoxml(infoxml_fname, + entryset.infoxml.pnode.data) else: self.LintError("no-infoxml", "No info.xml found for %s" % filename) - def check_infoxml(self, xdata): + def check_infoxml(self, fname, xdata): for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: @@ -28,8 +29,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if missing: self.LintError("required-infoxml-attrs-missing", "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), infoxml_fname, - self.RenderXML(info))) + (",".join(missing), fname, self.RenderXML(info))) if ((Bcfg2.Options.MDATA_PARANOID.value and info.get("paranoid") is not None and @@ -39,5 +39,5 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): info.get("paranoid").lower() != "true"))): self.LintError("paranoid-false", "Paranoid must be true in %s:%s" % - (infoxml_fname, self.RenderXML(info))) + (fname, self.RenderXML(info))) -- cgit v1.2.3-1-g7c22 From 34e5e12fd58ec4b0015abdf1a04a4f684c9194ba Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 10:30:33 -0400 Subject: Added FileProbes plugin. --- src/lib/Server/Lint/Validate.py | 4 +- src/lib/Server/Plugins/FileProbes.py | 177 +++++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 src/lib/Server/Plugins/FileProbes.py (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c87c55ee9..834608378 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -24,7 +24,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "%s/Decisions/*.xml":"%s/decisions.xsd", "%s/Packages/config.xml":"%s/packages.xsd", "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd"} + "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", + "%s/FileProbes/config.xml":"%s/fileprobes.xsd", + } self.filelists = {} self.get_filelists() diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py new file mode 100644 index 000000000..4df5a7f4a --- /dev/null +++ b/src/lib/Server/Plugins/FileProbes.py @@ -0,0 +1,177 @@ +""" This module allows you to probe a client for a file, which is then +added to the specification. On subsequent runs, the file will be +replaced on the client if it is missing; if it has changed on the +client, it can either be updated in the specification or replaced on +the client """ +__revision__ = '$Revision: 1465 $' + +import os +import errno +import binascii +import lxml.etree +import Bcfg2.Options +import Bcfg2.Server.Plugin + +probecode = """#!/usr/bin/env python + +import os +import pwd +import grp +import binascii +import lxml.etree + +path = "%s" + +if not os.path.exists(path): + print "%%s does not exist" %% path + raise SystemExit(1) + +stat = os.stat(path) +data = lxml.etree.Element("ProbedFileData", + name=path, + owner=pwd.getpwuid(stat[4])[0], + group=grp.getgrgid(stat[5])[0], + perms=oct(stat[0] & 07777)) +data.text = binascii.b2a_base64(open(path).read()) +print lxml.etree.tostring(data) +""" + +class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, + Bcfg2.Server.Plugin.StructFile): + """ Config file handler for FileProbes """ + def __init__(self, filename, fam): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename) + + +class FileProbes(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Probing): + """ This module allows you to probe a client for a file, which is then + added to the specification. On subsequent runs, the file will be + replaced on the client if it is missing; if it has changed on the + client, it can either be updated in the specification or replaced on + the client """ + + name = 'FileProbes' + experimental = True + __version__ = '$Id$' + __author__ = 'stpierreca@ornl.gov' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Probing.__init__(self) + self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'), + core.fam) + self.entries = dict() + self.probes = dict() + + def GetProbes(self, metadata): + """Return a set of probes for execution on client.""" + if metadata.hostname not in self.probes: + cfg = self.core.plugins['Cfg'] + self.entries[metadata.hostname] = dict() + self.probes[metadata.hostname] = [] + for entry in self.config.Match(metadata): + path = entry.get("name") + # do not probe for files that are already in Cfg and + # for which update is false; we can't possibly do + # anything with the data we get from such a probe + try: + if (cfg.entries[path].get_pertinent_entries(metadata) and + entry.get('update', 'false').lower() == "false"): + continue + except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError): + pass + self.entries[metadata.hostname][path] = entry + probe = lxml.etree.Element('probe', name=path, + source=self.name, + interpreter="/usr/bin/env python") + probe.text = probecode % path + self.probes[metadata.hostname].append(probe) + self.logger.debug("Adding file probe for %s to %s" % + (path, metadata.hostname)) + return self.probes[metadata.hostname] + + def ReceiveData(self, metadata, datalist): + """Receive data from probe.""" + self.logger.debug("Receiving file probe data from %s" % + metadata.hostname) + + for data in datalist: + if data.text is None: + self.logger.error("Got null response to %s file probe from %s" % + (data.get('name'), metadata.hostname)) + else: + self.logger.debug("%s:fileprobe:%s:%s" % + (metadata.hostname, + data.get("name"), + data.text)) + try: + filedata = lxml.etree.XML(data.text) + self.write_file(filedata, metadata) + except lxml.etree.XMLSyntaxError: + # if we didn't get XML back from the probe, assume + # it's an error message + self.logger.error(data.text) + + def write_file(self, data, metadata): + """Write the probed file data to the bcfg2 specification.""" + filename = data.get("name") + contents = binascii.a2b_base64(data.text) + entry = self.entries[metadata.hostname][filename] + cfg = self.core.plugins['Cfg'] + specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) + # we can't use os.path.join() for this because specific + # already has a leading /, which confuses os.path.join() + fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific)) + if filename not in cfg.entries.keys(): + self.logger.info("Writing new probed file %s" % fileloc) + try: + os.makedirs(os.path.dirname(fileloc)) + except OSError, err: + if err.errno == errno.EEXIST: + pass + else: + raise + open(fileloc, 'wb').write(contents) + + infoxml = os.path.join("%s%s" % (cfg.data, filename), + "info.xml") + self.write_infoxml(infoxml, entry, data) + else: + try: + cfgentry = \ + cfg.entries[filename].get_pertinent_entries(metadata)[0] + except Bcfg2.Server.Plugin.PluginExecutionError: + self.logger.info("Writing new probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + return + + if cfgentry.data == contents: + self.logger.debug("Existing %s contents match probed contents" % + filename) + elif (entry.get('update', 'false').lower() == "true"): + self.logger.info("Writing updated probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + else: + self.logger.info("Skipping updated probed file %s" % fileloc) + + def write_infoxml(self, infoxml, entry, data): + """ write an info.xml for the file """ + self.logger.info("Writing info.xml at %s for %s" % + (infoxml, data.get("name"))) + info = \ + lxml.etree.Element("Info", + owner=data.get("owner", + Bcfg2.Options.MDATA_OWNER.value), + group=data.get("group", + Bcfg2.Options.MDATA_GROUP.value), + perms=data.get("perms", + Bcfg2.Options.MDATA_PERMS.value), + encoding=entry.get("encoding", + Bcfg2.Options.ENCODING.value)) + + root = lxml.etree.Element("FileInfo") + root.append(info) + open(infoxml, "w").write(lxml.etree.tostring(root, + pretty_print=True)) -- cgit v1.2.3-1-g7c22 From a408e7cbe36beacc2aefe291ac3f5caec36ddf35 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Wed, 11 May 2011 11:37:18 -0500 Subject: Cfg: Fix PluginExecutionError Replaces PluginExecutionError with Bcfg2.Server.Plugin.PluginExecutionError. Reported by emias. --- src/lib/Server/Plugins/Cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index a97cbd550..7fc35cec1 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -188,7 +188,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): name = self.build_filename(specific) if name.endswith(".genshi"): logger.error("Cfg: Unable to pull data for genshi types") - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) if log: logger.info("Wrote file %s" % name) -- cgit v1.2.3-1-g7c22 From 32659c415a2c438eaa2dbf160d118465439da6dd Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 11 May 2011 15:24:26 -0500 Subject: Plugin: Fix nasty list comprehension bug It appears as though Python 2 kept around variables assigned within a list comprehensions which caused this to go unnoticed. Signed-off-by: Sol Jerome --- src/lib/Server/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index cd2b63656..3b331b300 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -644,9 +644,9 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): def BindEntry(self, entry, metadata): """Check package lists of package entries.""" - [src.Cache(metadata) for src in list(self.entries.values())] name = entry.get('name') - if not src.cache: + if False in [src.Cache(metadata) for src in + list(self.entries.values())]: self.logger.error("Called before data loaded") raise PluginExecutionError matching = [src for src in list(self.entries.values()) -- cgit v1.2.3-1-g7c22 From 34d9c0652dbbaaf28fbc311ef168e29fc58155f7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 13 May 2011 13:10:00 -0500 Subject: Cfg: Fix bcfg2-admin pull behavior for genshi templates (#1010) Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 7fc35cec1..832f7ab41 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -5,6 +5,7 @@ import binascii import logging import lxml import os +import os.path import re import sys import tempfile @@ -186,7 +187,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): def write_update(self, specific, new_entry, log): if 'text' in new_entry: name = self.build_filename(specific) - if name.endswith(".genshi"): + if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) -- cgit v1.2.3-1-g7c22 From b7234fe39d1f25eb55320fed8491781aef1ed9fe Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 17 May 2011 15:46:57 -0400 Subject: added bcfg2-lint MergeFiles plugin to suggest config files and probes that are very similar and could be merged added text wrapping to bcfg2-lint error handling --- src/lib/Server/Lint/MergeFiles.py | 71 +++++++++++++++++++++++++++++++++++++++ src/lib/Server/Lint/__init__.py | 43 ++++++++++++++++++------ 2 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 src/lib/Server/Lint/MergeFiles.py (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py new file mode 100644 index 000000000..1e177acff --- /dev/null +++ b/src/lib/Server/Lint/MergeFiles.py @@ -0,0 +1,71 @@ +import os +from copy import deepcopy +from difflib import SequenceMatcher +import Bcfg2.Options +import Bcfg2.Server.Lint + +class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): + """ find Probes or Cfg files with multiple similar files that + might be merged into one """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + if 'Probes' in self.core.plugins: + self.check_probes() + + def check_cfg(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for mset in self.get_similar(entryset.entries): + self.LintError("merge-cfg", + "The following files are similar: %s. " + "Consider merging them into a single Genshi " + "template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + + def check_probes(self): + probes = self.core.plugins['Probes'].probes.entries + for mset in self.get_similar(probes): + self.LintError("merge-cfg", + "The following probes are similar: %s. " + "Consider merging them into a single probe." % + ", ".join([p for p in mset])) + + def get_similar(self, entries): + if "threshold" in self.config: + # accept threshold either as a percent (e.g., "threshold=75") or + # as a ratio (e.g., "threshold=.75") + threshold = float(self.config['threshold']) + if threshold > 1: + threshold /= 100 + else: + threshold = 0.75 + rv = [] + elist = entries.items() + while elist: + result = self._find_similar(elist.pop(0), deepcopy(elist), + threshold) + if len(result) > 1: + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in result] + rv.append(result) + return rv + + def _find_similar(self, ftuple, others, threshold): + fname, fdata = ftuple + rv = [fname] + while others: + cname, cdata = others.pop(0) + sm = SequenceMatcher(None, fdata.data, cdata.data) + # perform progressively more expensive comparisons + if (sm.real_quick_ratio() > threshold and + sm.quick_ratio() > threshold and + sm.ratio() > threshold): + rv.extend(self._find_similar((cname, cdata), deepcopy(others), + threshold)) + return rv + + diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 3b89d1f9e..013cbf2ba 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -4,6 +4,7 @@ __all__ = ['Bundles', 'Comments', 'Duplicates', 'InfoXML', + 'MergeFiles', 'Pkgmgr', 'RequiredAttrs', 'Validate'] @@ -11,6 +12,7 @@ __all__ = ['Bundles', import logging import os.path from copy import copy +import textwrap import lxml.etree import Bcfg2.Logger @@ -84,7 +86,9 @@ class ErrorHandler (object): "properties-schema-not-found":"warning", "xml-failed-to-parse":"error", "xml-failed-to-read":"error", - "xml-failed-to-verify":"error",} + "xml-failed-to-verify":"error", + "merge-cfg":"warning", + "merge-probes":"warning",} def __init__(self, config=None): self.errors = 0 @@ -92,6 +96,9 @@ class ErrorHandler (object): self.logger = logging.getLogger('bcfg2-lint') + self._wrapper = textwrap.TextWrapper(initial_indent = " ", + subsequent_indent = " ") + self._handlers = {} if config is not None: for err, action in config.items(): @@ -116,26 +123,42 @@ class ErrorHandler (object): self._handlers[err](msg) self.logger.debug(" (%s)" % err) else: - self.logger.info("Unknown error %s" % err) + # assume that it's an error, but complain + self.error(msg) + self.logger.warning("Unknown error %s" % err) def error(self, msg): """ log an error condition """ self.errors += 1 - lines = msg.splitlines() - self.logger.error("ERROR: %s" % lines.pop()) - [self.logger.error(" %s" % l) for l in lines] + self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): """ log a warning condition """ self.warnings += 1 - lines = msg.splitlines() - self.logger.warning("WARNING: %s" % lines.pop()) - [self.logger.warning(" %s" % l) for l in lines] + self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): """ log a silent/debug condition """ - lines = msg.splitlines() - [self.logger.debug("%s" % l) for l in lines] + self._log(msg, self.logger.debug) + + def _log(self, msg, logfunc, prefix=""): + # a message may itself consist of multiple lines. wrap() will + # elide them all into a single paragraph, which we don't want. + # so we split the message into its paragraphs and wrap each + # paragraph individually. this means, unfortunately, that we + # lose textwrap's built-in initial indent functionality, + # because we want to only treat the very first line of the + # first paragraph specially. so we do some silliness. + rawlines = msg.splitlines() + firstline = True + for rawline in rawlines: + lines = self._wrapper.wrap(rawline) + for line in lines: + if firstline: + logfunc("%s%s" % (prefix, line.lstrip())) + firstline = False + else: + logfunc(line) class ServerlessPlugin (Plugin): -- cgit v1.2.3-1-g7c22 From 065724b38235cb8ba4aa493f43a3f4f426ac1abd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 18 May 2011 10:44:50 -0400 Subject: fixed several major bugs in bcfg2-lint Validate plugin --- src/lib/Server/Lint/Validate.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index 834608378..c0a400dd6 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -14,7 +14,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filesets = {"metadata:groups":"%s/metadata.xsd", "metadata:clients":"%s/clients.xsd", "info":"%s/info.xsd", - "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd", + "%s/Bundler/*.xml":"%s/bundle.xsd", + "%s/Bundler/*.genshi":"%s/bundle.xsd", "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", "%s/Base/*.xml":"%s/base.xsd", "%s/Rules/*.xml":"%s/rules.xsd", @@ -33,14 +34,16 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): - self.schemadir = self.config['schema'] + schemadir = self.config['schema'] - for schemaname, path in self.filesets.items(): + for path, schemaname in self.filesets.items(): try: filelist = self.filelists[path] except KeyError: filelist = [] - + + print "validating %s" % path + print "filelist = %s" % filelist if filelist: # avoid loading schemas for empty file lists try: @@ -48,8 +51,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): schemadir)) except: self.LintError("schema-failed-to-parse", - "Failed to process schema %s", - schemaname % schemadir) + "Failed to process schema %s" % + (schemaname % schemadir)) continue for filename in filelist: self.validate(filename, schemaname % schemadir, @@ -132,6 +135,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == 'info.xml']) else: + print "globbing for %s" % (path % self.config['repo']) self.filelists[path] = listfiles(path) self.filelists['props'] = listfiles("%s/Properties/*.xml") -- cgit v1.2.3-1-g7c22 From 37c5d256bda2ffe037680c9d5bfc5d986a58f7ef Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 May 2011 09:12:59 -0400 Subject: removed debugging output from Validate plugin --- src/lib/Server/Lint/Validate.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c0a400dd6..c9b5688e1 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -42,8 +42,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): except KeyError: filelist = [] - print "validating %s" % path - print "filelist = %s" % filelist if filelist: # avoid loading schemas for empty file lists try: @@ -135,7 +133,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == 'info.xml']) else: - print "globbing for %s" % (path % self.config['repo']) self.filelists[path] = listfiles(path) self.filelists['props'] = listfiles("%s/Properties/*.xml") -- cgit v1.2.3-1-g7c22 From 99495c559da8bb432274e94248f98d9b34c2d205 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 31 May 2011 10:36:15 -0500 Subject: Cfg: Fix traceback for non-ascii files Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 832f7ab41..5e3cca847 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -162,6 +162,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) logger.error("Please verify you are using the proper encoding.") raise Bcfg2.Server.Plugin.PluginExecutionError + except ValueError: + e = sys.exc_info()[1] + logger.error("Error in specification for %s" % entry.get('name')) + logger.error("%s" % e) + logger.error("You need to specify base64 encoding for %s." % + entry.get('name')) + raise Bcfg2.Server.Plugin.PluginExecutionError if entry.text in ['', None]: entry.set('empty', 'true') -- cgit v1.2.3-1-g7c22 From 3f13297b48c511de37c2a3233780d07d089c9993 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 1 Jun 2011 09:50:25 -0400 Subject: added -t option to set client timeout --- src/lib/Server/Admin/Perf.py | 4 +++- src/lib/Server/Admin/Xcmd.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py index af1c83072..d03b37d57 100644 --- a/src/lib/Server/Admin/Perf.py +++ b/src/lib/Server/Admin/Perf.py @@ -22,6 +22,7 @@ class Perf(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'server': Bcfg2.Options.SERVER_LOCATION, 'user': Bcfg2.Options.CLIENT_USER, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -30,7 +31,8 @@ class Perf(Bcfg2.Server.Admin.Mode): setup['password'], key=setup['key'], cert=setup['certificate'], - ca=setup['ca']) + ca=setup['ca'], + timeout=setup['timeout']) data = proxy.get_statistics() for key, value in list(data.items()): data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]]) diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py index fd5794f88..2cb085346 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Server/Admin/Xcmd.py @@ -20,7 +20,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'key': Bcfg2.Options.SERVER_KEY, 'certificate': Bcfg2.Options.CLIENT_CERT, - 'ca': Bcfg2.Options.CLIENT_CA + 'ca': Bcfg2.Options.CLIENT_CA, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -31,7 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): key=setup['key'], cert=setup['certificate'], ca=setup['ca'], - timeout=180) + timeout=setup['timeout']) if len(setup['args']) == 0: print("Usage: xcmd ") return -- cgit v1.2.3-1-g7c22 From 368b34bcdd509d24bcc82ccbdbd056455bba7c37 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 3 Jun 2011 19:10:18 -0500 Subject: Base: Deprecate Base in favor of Bundler Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Base.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 5e7d89727..5de57a87c 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = Bcfg2.Server.Plugin.StructFile + deprecated = True """Base creates independent clauses based on client metadata.""" def __init__(self, core, datastore): -- cgit v1.2.3-1-g7c22 From 70d480f2a77902347601596a7ab8f606a4b4290d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sun, 5 Jun 2011 14:00:56 -0500 Subject: Validate: Unsuppress errors for invalid schema paths (#1007) Signed-off-by: Sol Jerome --- src/lib/Server/Lint/Validate.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c9b5688e1..a561232b6 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -1,10 +1,12 @@ +import fnmatch import glob import lxml.etree import os -import fnmatch +from subprocess import Popen, PIPE, STDOUT +import sys + import Bcfg2.Options import Bcfg2.Server.Lint -from subprocess import Popen, PIPE, STDOUT class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that the repo validates """ @@ -47,6 +49,10 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname % schemadir)) + except IOError: + e = sys.exc_info()[1] + self.LintError("input/output error", e.message) + continue except: self.LintError("schema-failed-to-parse", "Failed to process schema %s" % -- cgit v1.2.3-1-g7c22 From b2e9d11b0dca1af6e0ce9a9f21558b35b35f5777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 16:32:53 +0200 Subject: Accept non-ASCII diffs Currently, client reports don't include diffs of files which aren't US-ASCII encoded. The client transmits such files as Base64 blobs. As we'd like to change that, this commit teaches the server to properly handle non-ASCII diffs. --- src/lib/Server/Admin/Reports.py | 6 ++++++ src/lib/Server/Admin/__init__.py | 5 +++-- src/lib/Server/Plugins/Cfg.py | 7 ++++++- src/lib/Server/Plugins/DBStats.py | 1 + src/lib/Server/Reports/importscript.py | 18 +++++++++++++++--- 5 files changed, 31 insertions(+), 6 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 942477a49..c9f3d3f58 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode): except (IOError, XMLSyntaxError): self.errExit("StatReports: Failed to parse %s" % (stats_file)) + try: + encoding = self.cfp.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientspath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: load_stats(clientsdata, statsdata, + encoding, verb, self.log, quick=quick, diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index 8915492a3..b34d7108c 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -113,7 +113,8 @@ class MetadataCore(Mode): def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None): Mode.__init__(self, configfile) options = {'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'configfile': Bcfg2.Options.CFILE} + 'configfile': Bcfg2.Options.CFILE, + 'encoding': Bcfg2.Options.ENCODING} setup = Bcfg2.Options.OptionParser(options) setup.hm = usage setup.parse(sys.argv[1:]) @@ -126,7 +127,7 @@ class MetadataCore(Mode): try: self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), setup['plugins'], - 'foo', 'UTF-8') + 'foo', setup['encoding']) except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed because %s" % msg) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 5e3cca847..c08e8c4b6 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -197,7 +197,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(new_entry['text']) + try: + etext = new_entry['text'].encode(self.encoding) + except: + logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) + raise Bcfg2.Server.Plugin.PluginExecutionError + open(name, 'w').write(etext) if log: logger.info("Wrote file %s" % name) badattr = [attr for attr in ['owner', 'group', 'perms'] diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py index 5ef1920e1..103fb7353 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Server/Plugins/DBStats.py @@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, try: Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, container, + self.core.encoding, 0, logger, True, diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index b6a3c2599..68774cec6 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -38,7 +38,7 @@ import platform from Bcfg2.Bcfg2Py3k import ConfigParser -def build_reason_kwargs(r_ent): +def build_reason_kwargs(r_ent, encoding, logger): binary_file = False if r_ent.get('current_bfile', False): binary_file = True @@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent): rc_diff = r_ent.get('current_diff') else: rc_diff = '' + if not binary_file: + try: + rc_diff = rc_diff.decode(encoding) + except: + logger.error("Reason isn't %s encoded, cannot decode it" % encoding) + rc_diff = '' return dict(owner=r_ent.get('owner', default=""), current_owner=r_ent.get('current_owner', default=""), group=r_ent.get('group', default=""), @@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent): is_binary=binary_file) -def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): +def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): clients = {} [clients.__setitem__(c.name, c) \ for c in Client.objects.all()] @@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): for (xpath, type) in pattern: for x in statistics.findall(xpath): counter_fields[type] = counter_fields[type] + 1 - kargs = build_reason_kwargs(x) + kargs = build_reason_kwargs(x, encoding, logger) try: rr = None @@ -270,6 +276,11 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s" % (statpath)) raise SystemExit(1) + try: + encoding = cf.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -288,6 +299,7 @@ if __name__ == '__main__': update_database() load_stats(clientsdata, statsdata, + encoding, verb, logger, quick=q, -- cgit v1.2.3-1-g7c22 From 129aa6d584765fba5b0ff686d5d8da62b5ae405c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 6 Jun 2011 12:36:56 -0500 Subject: Validate: Create and use new error type for missing schemas Signed-off-by: Sol Jerome --- src/lib/Server/Lint/Validate.py | 2 +- src/lib/Server/Lint/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index a561232b6..8a8406e73 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -51,7 +51,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): schemadir)) except IOError: e = sys.exc_info()[1] - self.LintError("input/output error", e.message) + self.LintError("input-output-error", e.message) continue except: self.LintError("schema-failed-to-parse", diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 013cbf2ba..63cb2463b 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -88,7 +88,8 @@ class ErrorHandler (object): "xml-failed-to-read":"error", "xml-failed-to-verify":"error", "merge-cfg":"warning", - "merge-probes":"warning",} + "merge-probes":"warning", + "input-output-error": "error"} def __init__(self, config=None): self.errors = 0 -- cgit v1.2.3-1-g7c22 From 0b074d1d8dff851081a892e6a6755c36f51c189f Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 10:34:13 -0500 Subject: Init: Remove Base from default plugin list Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 1 - 1 file changed, 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index fff8bcd1c..a9170530e 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,7 +77,6 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', - 'Base', 'Bundler', 'Bzr', 'Cfg', -- cgit v1.2.3-1-g7c22 From c1dfee4c7ed9eb1d8ca62d95182552cc847ec52e Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 11:24:42 -0500 Subject: Revert "Init: Remove Base from default plugin list" This reverts commit 611ce16c8cef81a6fc754c46dcb5cbe618b20b67. --- src/lib/Server/Admin/Init.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index a9170530e..fff8bcd1c 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,6 +77,7 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', + 'Base', 'Bundler', 'Bzr', 'Cfg', -- cgit v1.2.3-1-g7c22 From 2c9a76cdfcce02f8d89129405f1f477753c47d3c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 11:25:13 -0500 Subject: Init: Remove Base from default plugins list Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index fff8bcd1c..eab030cf8 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -103,8 +103,7 @@ plugin_list = ['Account', 'TGenshi'] # Default list of plugins to use -default_plugins = ['Base', - 'Bundler', +default_plugins = ['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', -- cgit v1.2.3-1-g7c22 From b1ab3e2bab9f07c13daf5dcfd4a9502eb84dcf0d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 14 Jun 2011 15:23:31 -0500 Subject: PY3K: Finish server-side code fixes Signed-off-by: Sol Jerome --- src/lib/Server/Core.py | 4 ++-- src/lib/Server/Plugin.py | 3 +++ src/lib/Server/Plugins/Cfg.py | 5 +++-- src/lib/Server/Plugins/Metadata.py | 2 +- src/lib/Server/Plugins/Probes.py | 5 +++-- src/lib/Server/Plugins/SSHbase.py | 21 +++++++++++++-------- 6 files changed, 25 insertions(+), 15 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 8f9d3e746..5adfa5381 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -384,7 +384,7 @@ class Core(Component): # clear dynamic groups self.metadata.cgroups[meta.hostname] = [] try: - xpdata = lxml.etree.XML(probedata) + xpdata = lxml.etree.XML(probedata.encode('utf-8')) except: self.logger.error("Failed to parse probe data from client %s" % \ (address[0])) @@ -433,7 +433,7 @@ class Core(Component): @exposed def RecvStats(self, address, stats): """Act on statistics upload.""" - sdata = lxml.etree.XML(stats) + sdata = lxml.etree.XML(stats.encode('utf-8')) client = self.metadata.resolve_client(address) self.process_statistics(client, sdata) return "" diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3b331b300..3a36baf8e 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -693,6 +693,9 @@ class Specificity: self.prio = prio self.delta = delta + def __lt__(self, other): + return self.__cmp__(other) < 0 + def matches(self, metadata): return self.all or \ self.hostname == metadata.hostname or \ diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c08e8c4b6..4a0fd2dfe 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import binascii import logging import lxml +import operator import os import os.path import re @@ -25,7 +26,7 @@ logger = logging.getLogger('Bcfg2.Plugins.Cfg') def u_str(string, encoding): if sys.hexversion >= 0x03000000: - return str(string, encoding) + return string.encode(encoding) else: return unicode(string, encoding) @@ -105,7 +106,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] - matching.sort(self.sort_by_specific) + matching.sort(key=operator.attrgetter('specific')) non_delta = [matching.index(m) for m in matching if not m.specific.delta] if not non_delta: diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ca6e43851..6570f2912 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (address[0])) return False # populate the session cache - if user != 'root': + if user.decode('utf-8') != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index ea2e79ccc..3599f2af1 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -1,4 +1,5 @@ import lxml.etree +import operator import re import Bcfg2.Server.Plugin @@ -27,7 +28,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret = [] build = dict() candidates = self.get_matching(metadata) - candidates.sort(lambda x, y: cmp(x.specific, y.specific)) + candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: rem = specific_probe_matcher.match(entry.name) if not rem: @@ -90,7 +91,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data) + datafile.write(data.decode('utf-8')) def load_data(self): try: diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cf0998aaa..4a33c0cb0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", - "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"] + "ssh_host_rsa_key.pub.H_%s", + "ssh_host_key.pub.H_%s"] hostkeys = ["ssh_host_dsa_key.H_%s", - "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"] - keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key', - 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub', - 'ssh_host_key.pub'] + "ssh_host_rsa_key.H_%s", + "ssh_host_key.H_%s"] + keypatterns = ["ssh_host_dsa_key", + "ssh_host_rsa_key", + "ssh_host_key", + "ssh_host_dsa_key.pub", + "ssh_host_rsa_key.pub", + "ssh_host_key.pub"] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([str(value.data) for key, value in \ + self.__skn = "\n".join([value.data.decode() for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() @@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname) continue self.__skn += "%s %s" % (','.join(names[hostname]), - self.entries[pubkey].data) + self.entries[pubkey].data.decode()) return self.__skn def set_skn(self, value): @@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, hostkeys.sort() for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - self.entries[hostkey].data) + self.entries[hostkey].data.decode()) permdata = {'owner': 'root', 'group': 'root', 'type': 'file', -- cgit v1.2.3-1-g7c22 From a1bc75c200fb9237ecc0dd5ccd5b8bf94214cf4b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 09:23:19 -0500 Subject: PY3K: Add compatibility comments Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 1 + src/lib/Server/Plugins/Snapshots.py | 1 + src/lib/Server/Reports/reports/templatetags/syntax_coloring.py | 1 + src/lib/Server/Snapshots/model.py | 1 + 4 files changed, 4 insertions(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 4a0fd2dfe..c93b76488 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -24,6 +24,7 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') +# py3k compatibility def u_str(string, encoding): if sys.hexversion >= 0x03000000: return string.encode(encoding) diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index 8b6bad574..aeb3b9f74 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -28,6 +28,7 @@ datafields = { } +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py index 291528e2e..2e30125f9 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py @@ -15,6 +15,7 @@ try: except: colorize = False +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 2aa35f1ec..f30c38a05 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string -- cgit v1.2.3-1-g7c22 From ac578f15ca785d9b1fcd0d42bfa3ffd017985e78 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 11:32:29 -0400 Subject: make Bcfg2 automatically handle JSON, XML, and YAML output from probes --- src/lib/Server/Plugins/Probes.py | 93 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 3599f2af1..4a7171297 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -2,12 +2,96 @@ import lxml.etree import operator import re +try: + import json + has_json = True +except ImportError: + has_json = False + +try: + import syck + has_syck = True +except ImportError: + has_syck = False + +try: + import yaml + has_yaml = True +except ImportError: + has_yaml = False + import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P\S+)(.(?P[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P\S+)") +class ProbeData (object): + """ a ProbeData object emulates a str object, but also has .xdata + and .json properties to provide convenient ways to use ProbeData + objects as XML or JSON data """ + def __init__(self, data): + self.data = data + self._xdata = None + self._json = None + self._yaml = None + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __getattr__(self, name): + """ make ProbeData act like a str object """ + return getattr(self.data, name) + + def __complex__(self): + return complex(self.data) + + def __int__(self): + return int(self.data) + + def __long__(self): + return long(self.data) + + def __float__(self): + return float(self.data) + + @property + def xdata(self): + if self._xdata is None: + try: + self._xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + pass + return self._xdata + + @property + def json(self): + if self._json is None and has_json: + try: + self._json = json.loads(self.data) + except ValueError: + pass + return self._json + + @property + def yaml(self): + if self._yaml is None: + if has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml.YAMLError: + pass + elif has_syck: + try: + self._yaml = syck.load(self.data) + except syck.error: + pass + return self._yaml + + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") @@ -106,7 +190,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') + self.probedata[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get('value')) elif (pdata.tag == 'Group'): self.cgroups[client.get('name')].append(pdata.get('name')) @@ -142,11 +227,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, if newgroup not in self.cgroups[client.hostname]: self.cgroups[client.hostname].append(newgroup) dlines.remove(line) - dtext = "\n".join(dlines) + dobj = ProbeData("\n".join(dlines)) try: - self.probedata[client.hostname].update({data.get('name'): dtext}) + self.probedata[client.hostname].update({data.get('name'): dobj}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): dtext} + self.probedata[client.hostname] = {data.get('name'): dobj} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) -- cgit v1.2.3-1-g7c22 From 5f93d780fc2dcd6ba35179acd61c05754d1e4fbc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 12:21:17 -0400 Subject: made StructFile.Match() work with Group/Client tags inside other tags --- src/lib/Server/Plugin.py | 80 +++++++++++++++++------------------- src/lib/Server/Plugins/Properties.py | 45 ++++---------------- 2 files changed, 46 insertions(+), 79 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3a36baf8e..aff9af12b 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -457,51 +457,45 @@ class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" def __init__(self, name): XMLFileBacked.__init__(self, name) - self.fragments = {} - - def Index(self): - """Build internal data structures.""" - try: - xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - logger.error("Failed to parse file %s" % self.name) - return - self.fragments = {} - work = {lambda x: True: xdata.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + self.matches = {} + + def _match(self, item, metadata): + """ recursive helper for Match() """ + if isinstance(item, lxml.etree._Comment): + return [] + elif item.tag == 'Group': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') not in metadata.groups) or + item.get('name') in metadata.groups): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + elif item.tag == 'Client': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') != metadata.hostname) or + item.get('name') == metadata.hostname): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + else: + rv = copy.deepcopy(item) + for child in rv.iterchildren(): + rv.remove(child) + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return [rv] + def Match(self, metadata): """Return matching fragments of independent.""" - matching = [frag for (pred, frag) in list(self.fragments.items()) - if pred(metadata)] - if matching: - return reduce(lambda x, y: x + y, matching) - logger.error("File %s got null match" % (self.name)) - return [] + rv = [] + if metadata.hostname not in self.matches: + for child in self.entries(): + rv.extend(self._match(child, metadata)) + if not rv: + logger.error("File %s got null match" % (self.name)) + return rv class INode: diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index dea797a10..95565f2e4 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -7,42 +7,15 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" def Index(self): - """Build internal data structures.""" - if type(self.data) is not lxml.etree._Element: - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % - self.name) - - self.fragments = {} - work = {lambda x: True: self.data.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + """Build local data structures.""" + try: + xdata = XML(self.data) + except XMLSyntaxError: + self.label = None + self.entries = [] + return + self.label = xdata.attrib[self.__identifier__] + self.entries = xdata.getchildren() class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): -- cgit v1.2.3-1-g7c22 From bc1a6b8d0a46e37a108a752a7b6f54e637ff804d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 13:22:19 -0400 Subject: bug fixes --- src/lib/Server/Plugin.py | 20 ++++++++++++-------- src/lib/Server/Plugins/Properties.py | 12 +----------- 2 files changed, 13 insertions(+), 19 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index aff9af12b..e535802c7 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked): def Index(self): """Build local data structures.""" try: - xdata = XML(self.data) + self.xdata = XML(self.data) except XMLSyntaxError: logger.error("Failed to parse %s" % (self.name)) return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() + self.entries = self.xdata.getchildren() + if self.__identifier__ is not None: + self.label = self.xdata.attrib[self.__identifier__] def __iter__(self): return iter(self.entries) @@ -455,6 +456,8 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" + __identifier__ = None + def __init__(self, name): XMLFileBacked.__init__(self, name) self.matches = {} @@ -489,13 +492,14 @@ class StructFile(XMLFileBacked): def Match(self, metadata): """Return matching fragments of independent.""" - rv = [] if metadata.hostname not in self.matches: - for child in self.entries(): + rv = [] + for child in self.entries: rv.extend(self._match(child, metadata)) - if not rv: - logger.error("File %s got null match" % (self.name)) - return rv + if not rv: + logger.error("File %s got null match" % (self.name)) + self.matches[metadata.hostname] = rv + return self.matches[metadata.hostname] class INode: diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index 95565f2e4..54c5def57 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,17 +6,7 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - def Index(self): - """Build local data structures.""" - try: - xdata = XML(self.data) - except XMLSyntaxError: - self.label = None - self.entries = [] - return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() - + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile -- cgit v1.2.3-1-g7c22 From dfee52f047c4d50b9b7120b4310c11397d1a0620 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 15:04:04 -0400 Subject: cleaned up some ProbeData stuff, make null probe storage more consistent --- src/lib/Server/Plugins/Probes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 4a7171297..b07c4dfd3 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -165,7 +165,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, cx = lxml.etree.SubElement(top, 'Client', name=client) for probe in sorted(probed): lxml.etree.SubElement(cx, 'Probe', name=probe, - value=self.probedata[client][probe]) + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) data = lxml.etree.tostring(top, encoding='UTF-8', @@ -214,9 +214,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.logger.error("Got null response to probe %s from %s" % \ (data.get('name'), client.hostname)) try: - self.probedata[client.hostname].update({data.get('name'): ''}) + self.probedata[client.hostname].update({data.get('name'): + ProbeData('')}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): ''} + self.probedata[client.hostname] = \ + {data.get('name'): ProbeData('')} return dlines = data.text.split('\n') self.logger.debug("%s:probe:%s:%s" % (client.hostname, -- cgit v1.2.3-1-g7c22 From 707a17cf8bcf0e91a2c74f671f0d3fffff3c294e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 15:26:53 -0400 Subject: fixed bugs with handling of negate in new StructFile.Match() --- src/lib/Server/Plugin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index e535802c7..740de247a 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -470,7 +470,8 @@ class StructFile(XMLFileBacked): rv = [] if ((item.get('negate', 'false').lower() == 'true' and item.get('name') not in metadata.groups) or - item.get('name') in metadata.groups): + (item.get('negate', 'false').lower() == 'false' and + item.get('name') in metadata.groups)): for child in item.iterchildren(): rv.extend(self._match(child, metadata)) return rv @@ -478,7 +479,8 @@ class StructFile(XMLFileBacked): rv = [] if ((item.get('negate', 'false').lower() == 'true' and item.get('name') != metadata.hostname) or - item.get('name') == metadata.hostname): + (item.get('negate', 'false').lower() == 'false' and + item.get('name') == metadata.hostname)): for child in item.iterchildren(): rv.extend(self._match(child, metadata)) return rv -- cgit v1.2.3-1-g7c22 From 644d34552c4e2bc6a1da1daef5e3ac734b06b205 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 14:49:52 -0500 Subject: Hostbase: More PY3K fixes Signed-off-by: Sol Jerome --- src/lib/Server/Hostbase/backends.py | 6 ++++-- src/lib/Server/Hostbase/ldapauth.py | 12 ++++++++---- src/lib/Server/Hostbase/settings.py | 5 +++-- 3 files changed, 15 insertions(+), 8 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py index aa822409c..bf774f695 100644 --- a/src/lib/Server/Hostbase/backends.py +++ b/src/lib/Server/Hostbase/backends.py @@ -57,12 +57,14 @@ class NISBackend(object): return user - except NISAUTHError, e: + except NISAUTHError: + e = sys.exc_info()[1] return None def get_user(self, user_id): try: return User.objects.get(pk=user_id) - except User.DoesNotExist, e: + except User.DoesNotExist: + e = sys.exc_info()[1] return None diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py index 21b462c86..f3db26f67 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Server/Hostbase/ldapauth.py @@ -69,7 +69,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -86,7 +87,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -108,7 +110,8 @@ class ldapauth(object): raw_obj = result_data[0][1] distinguishedName = raw_obj['distinguishedName'] return ('success', distinguishedName[0],) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -134,7 +137,8 @@ class ldapauth(object): self.is_superuser = False return - except KeyError, e: + except KeyError: + e = sys.exc_info()[1] raise LDAPAUTHError("Portions of the LDAP User profile not present") def member_of(self): diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py index c44c7bf16..4e641f13c 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Server/Hostbase/settings.py @@ -1,9 +1,10 @@ -from ConfigParser import ConfigParser, NoSectionError, NoOptionError import os.path +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) -c = ConfigParser() +c = ConfigParser.ConfigParser() #This needs to be configurable one day somehow c.read(['./bcfg2.conf']) -- cgit v1.2.3-1-g7c22 From edb09c53992ef3aca8bfe440df66717290b1771f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 16 Jun 2011 07:33:18 -0400 Subject: change error to debug --- src/lib/Server/Plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 740de247a..30c4f9686 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -498,8 +498,7 @@ class StructFile(XMLFileBacked): rv = [] for child in self.entries: rv.extend(self._match(child, metadata)) - if not rv: - logger.error("File %s got null match" % (self.name)) + logger.debug("File %s got %d match(es)" % (self.name, len(rv))) self.matches[metadata.hostname] = rv return self.matches[metadata.hostname] -- cgit v1.2.3-1-g7c22 From 18368164103d047cf742f0679b4a8cb9ab5cb9c2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 16 Jun 2011 12:41:30 -0400 Subject: updated Lint.Comments for new properties XML handling --- src/lib/Server/Lint/Comments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8e86cc564..09443d4c0 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): props = self.core.plugins['Properties'] for propfile, pdata in props.store.entries.items(): if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.data, 'properties') + self.check_xml(pdata.name, pdata.xdata, 'properties') def check_metadata(self): """ check metadata files for required headers """ -- cgit v1.2.3-1-g7c22 From 3ec2fc27deefc647127c9e729221826a90fd7a96 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 17 Jun 2011 08:03:26 -0400 Subject: Improved handling of JSON data from probes --- src/lib/Server/Plugins/Probes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index b07c4dfd3..ec0f294dd 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -6,19 +6,22 @@ try: import json has_json = True except ImportError: - has_json = False + try: + import simplejson as json + has_json = True + except ImportError: + has_json = False try: import syck has_syck = True except ImportError: has_syck = False - -try: - import yaml - has_yaml = True -except ImportError: - has_yaml = False + try: + import yaml + has_yaml = True + except ImportError: + has_yaml = False import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 4f762722925113c56582a10dd4abead6cd84facc Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 18 Jun 2011 22:41:29 -0500 Subject: Version bump to 1.2.0pre3 Signed-off-by: Sol Jerome --- src/lib/Server/Reports/reports/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html index 6ef4c9aff..ec9a17468 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Server/Reports/reports/templates/base.html @@ -87,7 +87,7 @@
-- cgit v1.2.3-1-g7c22 From 19586bc42aa90543cf27e33ec32bd7df222138e8 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 20 Jun 2011 12:07:16 -0500 Subject: Cfg: Add support for perms='inherit' (Ticket #642) This feature allows you to use the on-disk permissions of the file in the Cfg repository rather than specifying them using the traditional means in info.xml. Note that this only works for the octal permissions of the file on disk since the owner/group may not exist on the destination machine. Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c93b76488..23ba0a745 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -8,6 +8,7 @@ import operator import os import os.path import re +import stat import sys import tempfile @@ -97,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) self.specific = CfgMatcher(path.split('/')[-1]) + path = path def sort_by_specific(self, one, other): return cmp(one.specific, other.specific) @@ -121,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): self.bind_info_to_entry(entry, metadata) used = self.get_pertinent_entries(metadata) basefile = used.pop(0) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = "%s/%s" % (self.path, entry.get('name')) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): -- cgit v1.2.3-1-g7c22 From 772382d68d6d9014f7f590b3dbd68a87dc918e32 Mon Sep 17 00:00:00 2001 From: Kristian Kostecky Date: Tue, 21 Jun 2011 16:36:35 -0400 Subject: Added auth attrib to list of valid attribs that the Client object can add/update --- src/lib/Server/Admin/Client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py index 3af25b15a..81fc4a1b1 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Server/Admin/Client.py @@ -27,7 +27,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val @@ -41,7 +42,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val -- cgit v1.2.3-1-g7c22