From 8938b4025400d150db4b6fb32181cd36c294e07c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 4 May 2011 13:35:38 -0500 Subject: SSHbase: PY3K string join method fix Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/SSHbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index ce08235eb..a8bf596de 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -73,7 +73,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([value.data for key, value in \ + self.__skn = "\n".join([str(value.data) for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() -- cgit v1.2.3-1-g7c22 From 071ef1a1fbe6368b1abb81855e1ab95e316e6911 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 08:35:54 -0500 Subject: TGenshi: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TGenshi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index 83b60c958..bc5e00400 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -3,7 +3,11 @@ __revision__ = '$Revision$' import binascii import logging +import sys import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TGenshi') @@ -76,9 +80,6 @@ class TemplateFile: def bind_entry(self, entry, metadata): """Build literal file information.""" fname = entry.get('realname', entry.get('name')) - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if entry.tag == 'Path': entry.set('type', 'file') try: -- cgit v1.2.3-1-g7c22 From c7f6cb353df45bb3662b78a49815a6b1a54a7f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 13 Apr 2011 15:16:06 +0200 Subject: Cfg: Fix the output encoding of Genshi templates Encode the configuration files generated from Genshi templates according to the encoding setting from Options.py instead of unconditionally using UTF-8. --- src/lib/Server/Plugins/Cfg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 41cf6c9c1..a97cbd550 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -134,9 +134,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): metadata=metadata, path=basefile.name).filter(removecomment) try: - data = stream.render('text', strip_whitespace=False) + data = stream.render('text', encoding=self.encoding, + strip_whitespace=False) except TypeError: - data = stream.render('text') + data = stream.render('text', encoding=self.encoding) if data == '': entry.set('empty', 'true') except Exception: -- cgit v1.2.3-1-g7c22 From 5952ca5ebf0908b29e2fb09091249dc6ce67540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Sun, 1 May 2011 23:50:09 +0200 Subject: Fix bcfg2-reports --badentry and --extraentry The reporting system schema has been changed. This change lead to exceptions such as | AttributeError: 'Entries_interactions' object has no attribute 'name' when running bcfg2-reports with the --badentry or --extraentry options. --- src/sbin/bcfg2-reports | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 20288fc5e..33a291395 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -233,7 +233,7 @@ else: for c_inst in c_list: baditems = c_inst.current_interaction.bad() for item in baditems: - if item.name == badentry[1] and item.kind == badentry[0]: + if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: result.append(c_inst) if c_inst in entrydict: entrydict.get(c_inst).append(badentry[1]) @@ -244,7 +244,7 @@ else: for c_inst in c_list: baditems = c_inst.current_interaction.bad() for item in baditems: - if item.name == badentry[1] and item.kind == badentry[0]: + if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: result.append(c_inst) break elif extraentry != "": @@ -255,7 +255,7 @@ else: for c_inst in c_list: extraitems = c_inst.current_interaction.extra() for item in extraitems: - if item.name == extraentry[1] and item.kind == extraentry[0]: + if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]: result.append(c_inst) if c_inst in entrydict: entrydict.get(c_inst).append(extraentry[1]) @@ -266,7 +266,7 @@ else: for c_inst in c_list: extraitems = c_inst.current_interaction.extra() for item in extraitems: - if item.name == extraentry[1] and item.kind == extraentry[0]: + if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]: result.append(c_inst) break -- cgit v1.2.3-1-g7c22 From af8bf14ee0079649d4583fb0850ec25b74242344 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 12:46:57 -0500 Subject: TCheetah: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TCheetah.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py index 151cc6543..49be88881 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Server/Plugins/TCheetah.py @@ -6,6 +6,9 @@ import logging import sys import traceback import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TCheetah') @@ -53,9 +56,6 @@ class TemplateFile: if entry.tag == 'Path': entry.set('type', 'file') try: - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if type(self.template) == unicode: entry.text = self.template else: -- cgit v1.2.3-1-g7c22 From 417f502abe96ab7d784cbc872aeb65add9a509ba Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:43:43 -0400 Subject: Add missing import. --- src/lib/Server/Admin/Init.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index e69412a5e..f450e5303 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -4,6 +4,7 @@ import random import socket import stat import string +import sys import subprocess import Bcfg2.Server.Admin import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 4838c1477108fd61b2f65f79f8e3d31776cd1ba6 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:48:49 -0400 Subject: Pass keypath to create_conf function. --- src/lib/Server/Admin/Init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index f450e5303..fff8bcd1c 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -142,7 +142,7 @@ def create_key(hostname, keypath, certpath, country, state, location): os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 -def create_conf(confpath, confdata): +def create_conf(confpath, confdata, keypath): # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): # py3k compatibility @@ -402,7 +402,7 @@ class Init(Bcfg2.Server.Admin.Mode): self.server_uri) # Create the configuration file and SSL key - create_conf(self.configfile, confdata) + create_conf(self.configfile, confdata, keypath) kpath = keypath + '/bcfg2.key' cpath = keypath + '/bcfg2.crt' create_key(self.shostname, kpath, cpath, self.country, -- cgit v1.2.3-1-g7c22 From 88249702412125c36179f5ca9f65b45aa0dc42a5 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:53:00 -0400 Subject: Add missing import sys. --- src/lib/Server/Plugins/SSHbase.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index a8bf596de..cf0998aaa 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -5,6 +5,7 @@ import binascii import os import socket import shutil +import sys import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From e810fa812c23e87fc43908f8f72c4c6d751df625 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:47:46 +0200 Subject: add missing sys import --- src/lib/Server/Plugins/Ldap.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index b904dbe02..7d6d0b609 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,4 +1,5 @@ import imp +import sys import time import ldap import Bcfg2.Options -- cgit v1.2.3-1-g7c22 From 31cfa9f3a7b9f4ffd5fa32e22042681015489149 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:54:42 +0200 Subject: log more useful error message if python-ldap is not installed --- src/lib/Server/Plugins/Ldap.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index 7d6d0b609..06ecaed7b 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,10 +1,18 @@ import imp +import logging import sys import time -import ldap import Bcfg2.Options import Bcfg2.Server.Plugin +logger = logging.getLogger('Bcfg2.Plugins.Ldap') + +try: + import ldap +except: + logger.error("Unable to load ldap module. Is python-ldap installed?") + raise ImportError + # time in seconds between retries after failed LDAP connection RETRY_DELAY = 5 # how many times to try reaching the LDAP server if a connection is broken -- cgit v1.2.3-1-g7c22 From 9ec715f1e25f5274d6472536ce18a899358d3e66 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 25 Apr 2011 21:33:46 -0500 Subject: DBStats: Stop duplicating data in reports_reason --- src/lib/Server/Reports/importscript.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 1781e2fac..b6a3c2599 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -133,12 +133,9 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): try: rr = None - if not quick: - try: - rr = Reason.objects.filter(**kargs)[0] - except IndexError: - pass - if not rr: + try: + rr = Reason.objects.filter(**kargs)[0] + except IndexError: rr = Reason(**kargs) rr.save() if vlevel > 0: -- cgit v1.2.3-1-g7c22 From 0326069de94cdc8300a8b917ae7c440f02793d6e Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:32:44 +0200 Subject: add another missing sys import --- src/lib/Server/Plugins/SGenshi.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py index 04942c2bd..efd981956 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Server/Plugins/SGenshi.py @@ -5,6 +5,7 @@ import genshi.input import genshi.template import lxml.etree import logging +import sys import Bcfg2.Server.Plugin import Bcfg2.Server.Plugins.TGenshi -- cgit v1.2.3-1-g7c22 From 84152d61ad34b0c9e1ebae7c4e00e100b41d211f Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 9 May 2011 09:46:47 -0500 Subject: Upstart: Fix typo reported by justintime on IRC Signed-off-by: Sol Jerome --- src/lib/Client/Tools/Upstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Client/Tools/Upstart.py b/src/lib/Client/Tools/Upstart.py index a9d4b166b..41a585c23 100644 --- a/src/lib/Client/Tools/Upstart.py +++ b/src/lib/Client/Tools/Upstart.py @@ -74,7 +74,7 @@ class Upstart(Bcfg2.Client.Tools.SvcTool): if entry.get('mode', 'default') == 'manual': self.logger.info("Service %s mode set to manual. Skipping " "installation." % (entry.get('name'))) - return Fasle + return False if entry.get('status') == 'on': pstatus = self.cmd.run(self.get_svc_command(entry, 'start'))[0] elif entry.get('status') == 'off': -- cgit v1.2.3-1-g7c22 From 92d4cfc6e8e2225b3b8d00578d83397313983c0b Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:53:44 +0200 Subject: add yet another missing sys import --- src/lib/Server/Plugins/Bundler.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 1a8e7348b..01ad3c78b 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import copy import lxml.etree import re +import sys import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 382d96a8146a888c6edca7a20ee6cb348b202c4b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 9 May 2011 11:54:07 -0500 Subject: Logger: Use bytes for PY3K Signed-off-by: Sol Jerome --- src/lib/Logger.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Logger.py b/src/lib/Logger.py index b49a7069f..0a72e038c 100644 --- a/src/lib/Logger.py +++ b/src/lib/Logger.py @@ -12,6 +12,8 @@ import sys import termios # Compatibility import from Bcfg2.Bcfg2Py3k import fprint +if sys.hexversion >= 0x03000000: + str = bytes logging.raiseExceptions = 0 @@ -118,7 +120,7 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): def emit(self, record): """Chunk and deliver records.""" record.name = self.procname - if len(record.msg) > 250: + if str(record.msg) > 250: msgs = [] error = record.exc_info record.exc_info = None -- cgit v1.2.3-1-g7c22 From ffce8cbd14a0ddc2cd266d80d63b36a786c08cb0 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 9 May 2011 15:20:11 -0500 Subject: Logger: Statement always was True in python 2 Signed-off-by: Sol Jerome --- src/lib/Logger.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/lib/Logger.py b/src/lib/Logger.py index 0a72e038c..8920ba747 100644 --- a/src/lib/Logger.py +++ b/src/lib/Logger.py @@ -12,8 +12,6 @@ import sys import termios # Compatibility import from Bcfg2.Bcfg2Py3k import fprint -if sys.hexversion >= 0x03000000: - str = bytes logging.raiseExceptions = 0 @@ -120,19 +118,16 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): def emit(self, record): """Chunk and deliver records.""" record.name = self.procname - if str(record.msg) > 250: - msgs = [] - error = record.exc_info - record.exc_info = None - msgdata = record.msg - while msgdata: - newrec = copy.deepcopy(record) - newrec.msg = msgdata[:250] - msgs.append(newrec) - msgdata = msgdata[250:] - msgs[0].exc_info = error - else: - msgs = [record] + msgs = [] + error = record.exc_info + record.exc_info = None + msgdata = record.msg + while msgdata: + newrec = copy.deepcopy(record) + newrec.msg = msgdata[:250] + msgs.append(newrec) + msgdata = msgdata[250:] + msgs[0].exc_info = error for newrec in msgs: msg = self.log_format_string % (self.encodePriority(self.facility, newrec.levelname.lower()), -- cgit v1.2.3-1-g7c22 From e6383bdb7d19729d340565fb091a8b67831a5835 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 10 May 2011 08:55:53 -0500 Subject: Logger: Fix non-string logging Signed-off-by: Sol Jerome --- src/lib/Logger.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/lib/Logger.py b/src/lib/Logger.py index 8920ba747..9fe81f47e 100644 --- a/src/lib/Logger.py +++ b/src/lib/Logger.py @@ -118,16 +118,19 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): def emit(self, record): """Chunk and deliver records.""" record.name = self.procname - msgs = [] - error = record.exc_info - record.exc_info = None - msgdata = record.msg - while msgdata: - newrec = copy.deepcopy(record) - newrec.msg = msgdata[:250] - msgs.append(newrec) - msgdata = msgdata[250:] - msgs[0].exc_info = error + if isinstance(record.msg, str): + msgs = [] + error = record.exc_info + record.exc_info = None + msgdata = record.msg + while msgdata: + newrec = copy.deepcopy(record) + newrec.msg = msgdata[:250] + msgs.append(newrec) + msgdata = msgdata[250:] + msgs[0].exc_info = error + else: + msgs = [record] for newrec in msgs: msg = self.log_format_string % (self.encodePriority(self.facility, newrec.levelname.lower()), -- cgit v1.2.3-1-g7c22 From aa8c53f8cf7d2e8bcf63176c45c44b19cbba2bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 11 May 2011 14:51:29 +0200 Subject: bcfg2: Convert specification from Unicode to UTF-8 The client receives the configuration specification as a Unicode string and then hands it over to the XML() function, which expects a UTF-8 encoded string. Therefore, the configuration specification is now converted to UTF-8. Resolves ticket #1009. --- src/sbin/bcfg2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 7f7d8f5c6..996d773ff 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -243,7 +243,7 @@ class Client: raise SystemExit(1) try: - rawconfig = proxy.GetConfig() + rawconfig = proxy.GetConfig().encode('UTF-8') except xmlrpclib.Fault: self.logger.error("Failed to download configuration from Bcfg2") raise SystemExit(2) -- cgit v1.2.3-1-g7c22 From 716168325c7221b33e3100b8a903bc5d075f6cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 11 May 2011 15:33:14 +0200 Subject: Fix a typo in an SSL error message --- src/lib/Proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index 8a1ad683e..ed33316af 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -181,7 +181,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): other_side_required = ssl.CERT_NONE self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") if self.cert and not self.key: - self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") self.cert = None if self.key and not self.cert: self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") @@ -226,7 +226,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): # authentication to the server ctx.load_cert(self.cert, self.key) elif self.cert: - self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") elif self.key: self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") -- cgit v1.2.3-1-g7c22 From fa70c550a90c369ac48862d976e3d4294181be89 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 09:37:50 -0400 Subject: Improved Svn2 error handling -- ClientError may not have a message attribute. --- src/lib/Server/Plugins/Svn2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py index 35f555294..b8a8e6b7e 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Server/Plugins/Svn2.py @@ -75,9 +75,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: @@ -95,9 +95,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: -- cgit v1.2.3-1-g7c22 From 564a0ae955286194094ea5e6eda0afb7f755ca91 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 10:12:45 -0400 Subject: Fixed error messages from info.xml bcfg2-lint check --- src/lib/Server/Lint/InfoXML.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 7725ad748..c88e54e95 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -13,12 +13,13 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if self.HandlesFile(infoxml_fname): if (hasattr(entryset, "infoxml") and entryset.infoxml is not None): - self.check_infoxml(entryset.infoxml.pnode.data) + self.check_infoxml(infoxml_fname, + entryset.infoxml.pnode.data) else: self.LintError("no-infoxml", "No info.xml found for %s" % filename) - def check_infoxml(self, xdata): + def check_infoxml(self, fname, xdata): for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: @@ -28,8 +29,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if missing: self.LintError("required-infoxml-attrs-missing", "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), infoxml_fname, - self.RenderXML(info))) + (",".join(missing), fname, self.RenderXML(info))) if ((Bcfg2.Options.MDATA_PARANOID.value and info.get("paranoid") is not None and @@ -39,5 +39,5 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): info.get("paranoid").lower() != "true"))): self.LintError("paranoid-false", "Paranoid must be true in %s:%s" % - (infoxml_fname, self.RenderXML(info))) + (fname, self.RenderXML(info))) -- cgit v1.2.3-1-g7c22 From 34e5e12fd58ec4b0015abdf1a04a4f684c9194ba Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 10:30:33 -0400 Subject: Added FileProbes plugin. --- src/lib/Server/Lint/Validate.py | 4 +- src/lib/Server/Plugins/FileProbes.py | 177 +++++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 src/lib/Server/Plugins/FileProbes.py (limited to 'src') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c87c55ee9..834608378 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -24,7 +24,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "%s/Decisions/*.xml":"%s/decisions.xsd", "%s/Packages/config.xml":"%s/packages.xsd", "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd"} + "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", + "%s/FileProbes/config.xml":"%s/fileprobes.xsd", + } self.filelists = {} self.get_filelists() diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py new file mode 100644 index 000000000..4df5a7f4a --- /dev/null +++ b/src/lib/Server/Plugins/FileProbes.py @@ -0,0 +1,177 @@ +""" This module allows you to probe a client for a file, which is then +added to the specification. On subsequent runs, the file will be +replaced on the client if it is missing; if it has changed on the +client, it can either be updated in the specification or replaced on +the client """ +__revision__ = '$Revision: 1465 $' + +import os +import errno +import binascii +import lxml.etree +import Bcfg2.Options +import Bcfg2.Server.Plugin + +probecode = """#!/usr/bin/env python + +import os +import pwd +import grp +import binascii +import lxml.etree + +path = "%s" + +if not os.path.exists(path): + print "%%s does not exist" %% path + raise SystemExit(1) + +stat = os.stat(path) +data = lxml.etree.Element("ProbedFileData", + name=path, + owner=pwd.getpwuid(stat[4])[0], + group=grp.getgrgid(stat[5])[0], + perms=oct(stat[0] & 07777)) +data.text = binascii.b2a_base64(open(path).read()) +print lxml.etree.tostring(data) +""" + +class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, + Bcfg2.Server.Plugin.StructFile): + """ Config file handler for FileProbes """ + def __init__(self, filename, fam): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename) + + +class FileProbes(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Probing): + """ This module allows you to probe a client for a file, which is then + added to the specification. On subsequent runs, the file will be + replaced on the client if it is missing; if it has changed on the + client, it can either be updated in the specification or replaced on + the client """ + + name = 'FileProbes' + experimental = True + __version__ = '$Id$' + __author__ = 'stpierreca@ornl.gov' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Probing.__init__(self) + self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'), + core.fam) + self.entries = dict() + self.probes = dict() + + def GetProbes(self, metadata): + """Return a set of probes for execution on client.""" + if metadata.hostname not in self.probes: + cfg = self.core.plugins['Cfg'] + self.entries[metadata.hostname] = dict() + self.probes[metadata.hostname] = [] + for entry in self.config.Match(metadata): + path = entry.get("name") + # do not probe for files that are already in Cfg and + # for which update is false; we can't possibly do + # anything with the data we get from such a probe + try: + if (cfg.entries[path].get_pertinent_entries(metadata) and + entry.get('update', 'false').lower() == "false"): + continue + except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError): + pass + self.entries[metadata.hostname][path] = entry + probe = lxml.etree.Element('probe', name=path, + source=self.name, + interpreter="/usr/bin/env python") + probe.text = probecode % path + self.probes[metadata.hostname].append(probe) + self.logger.debug("Adding file probe for %s to %s" % + (path, metadata.hostname)) + return self.probes[metadata.hostname] + + def ReceiveData(self, metadata, datalist): + """Receive data from probe.""" + self.logger.debug("Receiving file probe data from %s" % + metadata.hostname) + + for data in datalist: + if data.text is None: + self.logger.error("Got null response to %s file probe from %s" % + (data.get('name'), metadata.hostname)) + else: + self.logger.debug("%s:fileprobe:%s:%s" % + (metadata.hostname, + data.get("name"), + data.text)) + try: + filedata = lxml.etree.XML(data.text) + self.write_file(filedata, metadata) + except lxml.etree.XMLSyntaxError: + # if we didn't get XML back from the probe, assume + # it's an error message + self.logger.error(data.text) + + def write_file(self, data, metadata): + """Write the probed file data to the bcfg2 specification.""" + filename = data.get("name") + contents = binascii.a2b_base64(data.text) + entry = self.entries[metadata.hostname][filename] + cfg = self.core.plugins['Cfg'] + specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) + # we can't use os.path.join() for this because specific + # already has a leading /, which confuses os.path.join() + fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific)) + if filename not in cfg.entries.keys(): + self.logger.info("Writing new probed file %s" % fileloc) + try: + os.makedirs(os.path.dirname(fileloc)) + except OSError, err: + if err.errno == errno.EEXIST: + pass + else: + raise + open(fileloc, 'wb').write(contents) + + infoxml = os.path.join("%s%s" % (cfg.data, filename), + "info.xml") + self.write_infoxml(infoxml, entry, data) + else: + try: + cfgentry = \ + cfg.entries[filename].get_pertinent_entries(metadata)[0] + except Bcfg2.Server.Plugin.PluginExecutionError: + self.logger.info("Writing new probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + return + + if cfgentry.data == contents: + self.logger.debug("Existing %s contents match probed contents" % + filename) + elif (entry.get('update', 'false').lower() == "true"): + self.logger.info("Writing updated probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + else: + self.logger.info("Skipping updated probed file %s" % fileloc) + + def write_infoxml(self, infoxml, entry, data): + """ write an info.xml for the file """ + self.logger.info("Writing info.xml at %s for %s" % + (infoxml, data.get("name"))) + info = \ + lxml.etree.Element("Info", + owner=data.get("owner", + Bcfg2.Options.MDATA_OWNER.value), + group=data.get("group", + Bcfg2.Options.MDATA_GROUP.value), + perms=data.get("perms", + Bcfg2.Options.MDATA_PERMS.value), + encoding=entry.get("encoding", + Bcfg2.Options.ENCODING.value)) + + root = lxml.etree.Element("FileInfo") + root.append(info) + open(infoxml, "w").write(lxml.etree.tostring(root, + pretty_print=True)) -- cgit v1.2.3-1-g7c22 From a408e7cbe36beacc2aefe291ac3f5caec36ddf35 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Wed, 11 May 2011 11:37:18 -0500 Subject: Cfg: Fix PluginExecutionError Replaces PluginExecutionError with Bcfg2.Server.Plugin.PluginExecutionError. Reported by emias. --- src/lib/Server/Plugins/Cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index a97cbd550..7fc35cec1 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -188,7 +188,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): name = self.build_filename(specific) if name.endswith(".genshi"): logger.error("Cfg: Unable to pull data for genshi types") - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) if log: logger.info("Wrote file %s" % name) -- cgit v1.2.3-1-g7c22 From 32659c415a2c438eaa2dbf160d118465439da6dd Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 11 May 2011 15:24:26 -0500 Subject: Plugin: Fix nasty list comprehension bug It appears as though Python 2 kept around variables assigned within a list comprehensions which caused this to go unnoticed. Signed-off-by: Sol Jerome --- src/lib/Server/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index cd2b63656..3b331b300 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -644,9 +644,9 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): def BindEntry(self, entry, metadata): """Check package lists of package entries.""" - [src.Cache(metadata) for src in list(self.entries.values())] name = entry.get('name') - if not src.cache: + if False in [src.Cache(metadata) for src in + list(self.entries.values())]: self.logger.error("Called before data loaded") raise PluginExecutionError matching = [src for src in list(self.entries.values()) -- cgit v1.2.3-1-g7c22 From 34d9c0652dbbaaf28fbc311ef168e29fc58155f7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 13 May 2011 13:10:00 -0500 Subject: Cfg: Fix bcfg2-admin pull behavior for genshi templates (#1010) Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 7fc35cec1..832f7ab41 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -5,6 +5,7 @@ import binascii import logging import lxml import os +import os.path import re import sys import tempfile @@ -186,7 +187,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): def write_update(self, specific, new_entry, log): if 'text' in new_entry: name = self.build_filename(specific) - if name.endswith(".genshi"): + if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) -- cgit v1.2.3-1-g7c22 From 010bc893324ddcf8be5a3b00d01bbeb9786a120b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 16 May 2011 12:44:34 -0400 Subject: bcfg2-info help returns help without starting a server instance --- src/sbin/bcfg2-info | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 161fee441..fc36b2602 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -487,7 +487,10 @@ if __name__ == '__main__': }) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) - if setup['profile'] and have_profile: + if setup['args'][0] == 'help': + print(USAGE) + sys.exit(0) + elif setup['profile'] and have_profile: prof = profile.Profile() loop = prof.runcall(infoCore, setup['repo'], setup['plugins'], setup['password'], setup['encoding'], -- cgit v1.2.3-1-g7c22 From a19c87f8dc03132e7257a539c7e486b9c9298aff Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 17 May 2011 11:46:58 -0500 Subject: bcfg2-info: Fix traceback when no args specified Signed-off-by: Sol Jerome --- src/sbin/bcfg2-info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index fc36b2602..c36e1af42 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -487,7 +487,7 @@ if __name__ == '__main__': }) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) - if setup['args'][0] == 'help': + if setup['args'] and setup['args'][0] == 'help': print(USAGE) sys.exit(0) elif setup['profile'] and have_profile: -- cgit v1.2.3-1-g7c22 From b7234fe39d1f25eb55320fed8491781aef1ed9fe Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 17 May 2011 15:46:57 -0400 Subject: added bcfg2-lint MergeFiles plugin to suggest config files and probes that are very similar and could be merged added text wrapping to bcfg2-lint error handling --- src/lib/Server/Lint/MergeFiles.py | 71 +++++++++++++++++++++++++++++++++++++++ src/lib/Server/Lint/__init__.py | 43 ++++++++++++++++++------ 2 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 src/lib/Server/Lint/MergeFiles.py (limited to 'src') diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py new file mode 100644 index 000000000..1e177acff --- /dev/null +++ b/src/lib/Server/Lint/MergeFiles.py @@ -0,0 +1,71 @@ +import os +from copy import deepcopy +from difflib import SequenceMatcher +import Bcfg2.Options +import Bcfg2.Server.Lint + +class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): + """ find Probes or Cfg files with multiple similar files that + might be merged into one """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + if 'Probes' in self.core.plugins: + self.check_probes() + + def check_cfg(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for mset in self.get_similar(entryset.entries): + self.LintError("merge-cfg", + "The following files are similar: %s. " + "Consider merging them into a single Genshi " + "template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + + def check_probes(self): + probes = self.core.plugins['Probes'].probes.entries + for mset in self.get_similar(probes): + self.LintError("merge-cfg", + "The following probes are similar: %s. " + "Consider merging them into a single probe." % + ", ".join([p for p in mset])) + + def get_similar(self, entries): + if "threshold" in self.config: + # accept threshold either as a percent (e.g., "threshold=75") or + # as a ratio (e.g., "threshold=.75") + threshold = float(self.config['threshold']) + if threshold > 1: + threshold /= 100 + else: + threshold = 0.75 + rv = [] + elist = entries.items() + while elist: + result = self._find_similar(elist.pop(0), deepcopy(elist), + threshold) + if len(result) > 1: + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in result] + rv.append(result) + return rv + + def _find_similar(self, ftuple, others, threshold): + fname, fdata = ftuple + rv = [fname] + while others: + cname, cdata = others.pop(0) + sm = SequenceMatcher(None, fdata.data, cdata.data) + # perform progressively more expensive comparisons + if (sm.real_quick_ratio() > threshold and + sm.quick_ratio() > threshold and + sm.ratio() > threshold): + rv.extend(self._find_similar((cname, cdata), deepcopy(others), + threshold)) + return rv + + diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 3b89d1f9e..013cbf2ba 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -4,6 +4,7 @@ __all__ = ['Bundles', 'Comments', 'Duplicates', 'InfoXML', + 'MergeFiles', 'Pkgmgr', 'RequiredAttrs', 'Validate'] @@ -11,6 +12,7 @@ __all__ = ['Bundles', import logging import os.path from copy import copy +import textwrap import lxml.etree import Bcfg2.Logger @@ -84,7 +86,9 @@ class ErrorHandler (object): "properties-schema-not-found":"warning", "xml-failed-to-parse":"error", "xml-failed-to-read":"error", - "xml-failed-to-verify":"error",} + "xml-failed-to-verify":"error", + "merge-cfg":"warning", + "merge-probes":"warning",} def __init__(self, config=None): self.errors = 0 @@ -92,6 +96,9 @@ class ErrorHandler (object): self.logger = logging.getLogger('bcfg2-lint') + self._wrapper = textwrap.TextWrapper(initial_indent = " ", + subsequent_indent = " ") + self._handlers = {} if config is not None: for err, action in config.items(): @@ -116,26 +123,42 @@ class ErrorHandler (object): self._handlers[err](msg) self.logger.debug(" (%s)" % err) else: - self.logger.info("Unknown error %s" % err) + # assume that it's an error, but complain + self.error(msg) + self.logger.warning("Unknown error %s" % err) def error(self, msg): """ log an error condition """ self.errors += 1 - lines = msg.splitlines() - self.logger.error("ERROR: %s" % lines.pop()) - [self.logger.error(" %s" % l) for l in lines] + self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): """ log a warning condition """ self.warnings += 1 - lines = msg.splitlines() - self.logger.warning("WARNING: %s" % lines.pop()) - [self.logger.warning(" %s" % l) for l in lines] + self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): """ log a silent/debug condition """ - lines = msg.splitlines() - [self.logger.debug("%s" % l) for l in lines] + self._log(msg, self.logger.debug) + + def _log(self, msg, logfunc, prefix=""): + # a message may itself consist of multiple lines. wrap() will + # elide them all into a single paragraph, which we don't want. + # so we split the message into its paragraphs and wrap each + # paragraph individually. this means, unfortunately, that we + # lose textwrap's built-in initial indent functionality, + # because we want to only treat the very first line of the + # first paragraph specially. so we do some silliness. + rawlines = msg.splitlines() + firstline = True + for rawline in rawlines: + lines = self._wrapper.wrap(rawline) + for line in lines: + if firstline: + logfunc("%s%s" % (prefix, line.lstrip())) + firstline = False + else: + logfunc(line) class ServerlessPlugin (Plugin): -- cgit v1.2.3-1-g7c22 From 065724b38235cb8ba4aa493f43a3f4f426ac1abd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 18 May 2011 10:44:50 -0400 Subject: fixed several major bugs in bcfg2-lint Validate plugin --- src/lib/Server/Lint/Validate.py | 16 ++++++++++------ src/sbin/bcfg2-lint | 7 +++++-- 2 files changed, 15 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index 834608378..c0a400dd6 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -14,7 +14,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filesets = {"metadata:groups":"%s/metadata.xsd", "metadata:clients":"%s/clients.xsd", "info":"%s/info.xsd", - "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd", + "%s/Bundler/*.xml":"%s/bundle.xsd", + "%s/Bundler/*.genshi":"%s/bundle.xsd", "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", "%s/Base/*.xml":"%s/base.xsd", "%s/Rules/*.xml":"%s/rules.xsd", @@ -33,14 +34,16 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): - self.schemadir = self.config['schema'] + schemadir = self.config['schema'] - for schemaname, path in self.filesets.items(): + for path, schemaname in self.filesets.items(): try: filelist = self.filelists[path] except KeyError: filelist = [] - + + print "validating %s" % path + print "filelist = %s" % filelist if filelist: # avoid loading schemas for empty file lists try: @@ -48,8 +51,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): schemadir)) except: self.LintError("schema-failed-to-parse", - "Failed to process schema %s", - schemaname % schemadir) + "Failed to process schema %s" % + (schemaname % schemadir)) continue for filename in filelist: self.validate(filename, schemaname % schemadir, @@ -132,6 +135,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == 'info.xml']) else: + print "globbing for %s" % (path % self.config['repo']) self.filelists[path] = listfiles(path) self.filelists['props'] = listfiles("%s/Properties/*.xml") diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 6bc34433e..3c9bc44b3 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -50,10 +50,13 @@ def run_plugin(plugin, plugin_name, setup=None, errorhandler=None, errorhandler = get_errorhandler(config) if config is not None and config.has_section(plugin_name): - args.append(dict(config.items(plugin_name), **setup)) + arg = setup + for key, val in config.items(plugin_name): + arg[key] = val + args.append(arg) else: args.append(setup) - + # older versions of python do not support mixing *-magic and # non-*-magic (e.g., "plugin(*args, files=files)", so we do this # all with *-magic -- cgit v1.2.3-1-g7c22 From e081a2355c6afa182359bd44aa74ad45f16636f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Thu, 19 May 2011 23:08:48 +0200 Subject: Revert "bcfg2: Write cached config using encoding from Options.py" This reverts commit 5d69ff7e966c4ffa911c78d11a6879b48e90aef8. As the "rawconfig" variable now holds a UTF-8 encoded string, it would have to be decoded before re-encoding it using a different encoding. However, the cached configuration shouldn't be written using a non-UTF-8 encoding anyway, as "bcfg2 -f " currently doesn't accept any other encodings. (If this is to be changed, the XML encoding declaration of the configuration would have to be adjusted accordingly.) --- src/sbin/bcfg2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 996d773ff..eb41af640 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -252,7 +252,7 @@ class Client: if self.setup['cache']: try: - open(self.setup['cache'], 'w').write(rawconfig.encode(self.setup['encoding'])) + open(self.setup['cache'], 'w').write(rawconfig) os.chmod(self.setup['cache'], 33152) except IOError: self.logger.warning("Failed to write config cache file %s" % -- cgit v1.2.3-1-g7c22 From 8168776fb50858a0e531c3a874ee1e20a88e7690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Sun, 15 May 2011 15:23:59 +0200 Subject: APT: Add two filters for deprecated API accesses These accesses are triggered by configuration entries which don't have the "version" attribute set to "auto" or "any". --- src/lib/Client/Tools/APT.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Client/Tools/APT.py index a838f5e27..d7be44dc0 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Client/Tools/APT.py @@ -9,6 +9,8 @@ warnings.filterwarnings("ignore", "Accessed deprecated property Package.installe warnings.filterwarnings("ignore", "Accessed deprecated property Package.candidateVersion, please see the Version class for alternatives.", DeprecationWarning) warnings.filterwarnings("ignore", "Deprecated, please use 'is_installed' instead", DeprecationWarning) warnings.filterwarnings("ignore", "Attribute 'IsUpgradable' of the 'apt_pkg.DepCache' object is deprecated, use 'is_upgradable' instead.", DeprecationWarning) +warnings.filterwarnings("ignore", "Attribute 'VersionList' of the 'apt_pkg.Package' object is deprecated, use 'version_list' instead.", DeprecationWarning) +warnings.filterwarnings("ignore", "Attribute 'VerStr' of the 'apt_pkg.Version' object is deprecated, use 'ver_str' instead.", DeprecationWarning) import apt.cache import os -- cgit v1.2.3-1-g7c22 From 37c5d256bda2ffe037680c9d5bfc5d986a58f7ef Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 May 2011 09:12:59 -0400 Subject: removed debugging output from Validate plugin --- src/lib/Server/Lint/Validate.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c0a400dd6..c9b5688e1 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -42,8 +42,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): except KeyError: filelist = [] - print "validating %s" % path - print "filelist = %s" % filelist if filelist: # avoid loading schemas for empty file lists try: @@ -135,7 +133,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == 'info.xml']) else: - print "globbing for %s" % (path % self.config['repo']) self.filelists[path] = listfiles(path) self.filelists['props'] = listfiles("%s/Properties/*.xml") -- cgit v1.2.3-1-g7c22 From 4944977b131be888f4584b4b6b44dba2214d1818 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 May 2011 09:58:13 -0400 Subject: don't run lint server plugins if serverless plugins produced errors; avoids an ugly stack trace if a file fails to validate --- src/sbin/bcfg2-lint | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 3c9bc44b3..464e839e5 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -184,8 +184,20 @@ if __name__ == '__main__': config=config, setup=setup) if serverplugins: - run_server_plugins(serverplugins, errorhandler=errorhandler, - config=config, setup=setup) + if errorhandler.errors: + # it would be swell if we could try to start the server + # even if there were errors with the serverless plugins, + # but since XML parsing errors occur in the FAM thread + # (not in the core server thread), there's no way we can + # start the server and try to catch exceptions -- + # bcfg2-lint isn't in the same stack as the exceptions. + # so we're forced to assume that a serverless plugin error + # will prevent the server from starting + print("Serverless plugins encountered errors, skipping server " + "plugins") + else: + run_server_plugins(serverplugins, errorhandler=errorhandler, + config=config, setup=setup) if errorhandler.errors or errorhandler.warnings or setup['verbose']: print("%d errors" % errorhandler.errors) -- cgit v1.2.3-1-g7c22 From 81eeff2a4a9381811d6f3d20ebd6bcf18bf83553 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 24 May 2011 15:41:37 -0400 Subject: allow setting whitelist/blacklist mode in bcfg2.conf --- src/lib/Options.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Options.py b/src/lib/Options.py index d5304e696..26d0fa349 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -307,6 +307,7 @@ CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) CLIENT_DLIST = Option('Run client in server decision list mode', default=False, + cf=('client', 'decision'), cmd='-l', odesc='') CLIENT_FILE = Option('Configure from a file rather than querying the server', default=False, cmd='-f', odesc='') -- cgit v1.2.3-1-g7c22 From be72eb8b648672707822f891a103fc75ed54dbd2 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 25 May 2011 22:19:40 -0500 Subject: POSIX: Clarify normalization error (Reported by Tim Goodaire) Signed-off-by: Sol Jerome --- src/lib/Client/Tools/POSIX.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index af3d1a473..875db5ea7 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -55,7 +55,8 @@ def normGid(entry): except: return int(grp.getgrnam(entry.get('group'))[2]) except (OSError, KeyError): - log.error('GID normalization failed for %s' % (entry.get('name'))) + log.error('GID normalization failed for %s. Does group %s exist?' + % (entry.get('name'), entry.get('group'))) return False @@ -70,7 +71,8 @@ def normUid(entry): except: return int(pwd.getpwnam(entry.get('owner'))[2]) except (OSError, KeyError): - log.error('UID normalization failed for %s' % (entry.get('name'))) + log.error('UID normalization failed for %s. Does owner %s exist?' + % (entry.get('name'), entry.get('owner'))) return False -- cgit v1.2.3-1-g7c22 From 99495c559da8bb432274e94248f98d9b34c2d205 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 31 May 2011 10:36:15 -0500 Subject: Cfg: Fix traceback for non-ascii files Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 832f7ab41..5e3cca847 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -162,6 +162,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) logger.error("Please verify you are using the proper encoding.") raise Bcfg2.Server.Plugin.PluginExecutionError + except ValueError: + e = sys.exc_info()[1] + logger.error("Error in specification for %s" % entry.get('name')) + logger.error("%s" % e) + logger.error("You need to specify base64 encoding for %s." % + entry.get('name')) + raise Bcfg2.Server.Plugin.PluginExecutionError if entry.text in ['', None]: entry.set('empty', 'true') -- cgit v1.2.3-1-g7c22 From 3f13297b48c511de37c2a3233780d07d089c9993 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 1 Jun 2011 09:50:25 -0400 Subject: added -t option to set client timeout --- src/lib/Options.py | 3 +++ src/lib/Server/Admin/Perf.py | 4 +++- src/lib/Server/Admin/Xcmd.py | 5 +++-- src/sbin/bcfg2 | 10 ++++++---- 4 files changed, 15 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/lib/Options.py b/src/lib/Options.py index 26d0fa349..e6eebb808 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -317,6 +317,9 @@ CLIENT_USER = Option('The user to provide for authentication', default='root', cmd='-u', cf=('communication', 'user'), odesc='') CLIENT_SERVICE_MODE = Option('Set client service mode', default='default', cmd='-s', odesc='') +CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90, + cmd='-t', cf=('communication', 'timeout'), + odesc='') # APT client tool options CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path', diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py index af1c83072..d03b37d57 100644 --- a/src/lib/Server/Admin/Perf.py +++ b/src/lib/Server/Admin/Perf.py @@ -22,6 +22,7 @@ class Perf(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'server': Bcfg2.Options.SERVER_LOCATION, 'user': Bcfg2.Options.CLIENT_USER, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -30,7 +31,8 @@ class Perf(Bcfg2.Server.Admin.Mode): setup['password'], key=setup['key'], cert=setup['certificate'], - ca=setup['ca']) + ca=setup['ca'], + timeout=setup['timeout']) data = proxy.get_statistics() for key, value in list(data.items()): data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]]) diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py index fd5794f88..2cb085346 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Server/Admin/Xcmd.py @@ -20,7 +20,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'key': Bcfg2.Options.SERVER_KEY, 'certificate': Bcfg2.Options.CLIENT_CERT, - 'ca': Bcfg2.Options.CLIENT_CA + 'ca': Bcfg2.Options.CLIENT_CA, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -31,7 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): key=setup['key'], cert=setup['certificate'], ca=setup['ca'], - timeout=180) + timeout=setup['timeout']) if len(setup['args']) == 0: print("Usage: xcmd ") return diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index eb41af640..534ab8238 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -75,6 +75,7 @@ class Client: 'certificate': Bcfg2.Options.CLIENT_CERT, 'ca': Bcfg2.Options.CLIENT_CA, 'serverCN': Bcfg2.Options.CLIENT_SCNS, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } self.setup = Bcfg2.Options.OptionParser(optinfo) @@ -178,10 +179,11 @@ class Client: proxy = Bcfg2.Proxy.ComponentProxy(self.setup['server'], self.setup['user'], self.setup['password'], - key = self.setup['key'], - cert = self.setup['certificate'], - ca = self.setup['ca'], - allowedServerCNs = self.setup['serverCN']) + key=self.setup['key'], + cert=self.setup['certificate'], + ca=self.setup['ca'], + allowedServerCNs=self.setup['serverCN'], + timeout=self.setup['timeout']) if self.setup['profile']: try: -- cgit v1.2.3-1-g7c22 From ab956d76bd20b458195ade8efd2cafb590bb3217 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 1 Jun 2011 16:01:15 -0400 Subject: guarantee that timeout is a float --- src/lib/Proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index ed33316af..9656b166c 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -332,5 +332,5 @@ def ComponentProxy(url, user=None, password=None, else: newurl = url ssl_trans = XMLRPCTransport(key, cert, ca, - allowedServerCNs, timeout=timeout) + allowedServerCNs, timeout=float(timeout)) return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) -- cgit v1.2.3-1-g7c22 From 5da2ced6d355a61032e254356c88804c7a44ffc1 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 3 Jun 2011 13:28:46 -0500 Subject: bcfg2-server: Add the ability to listen on specific interfaces (#1013) Signed-off-by: Sol Jerome --- src/lib/Component.py | 13 +++++++++---- src/lib/Options.py | 19 +++++++++++++++++++ src/lib/SSLServer.py | 25 +++++++++++++++++-------- src/sbin/bcfg2-server | 2 ++ 4 files changed, 47 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/lib/Component.py b/src/lib/Component.py index 88dce906e..b73098d09 100644 --- a/src/lib/Component.py +++ b/src/lib/Component.py @@ -23,8 +23,8 @@ logger = logging.getLogger() class NoExposedMethod (Exception): """There is no method exposed with the given name.""" -def run_component(component_cls, location, daemon, pidfile_name, to_file, - cfile, argv=None, register=True, +def run_component(component_cls, listen_all, location, daemon, pidfile_name, + to_file, cfile, argv=None, register=True, state_name=False, cls_kwargs={}, extra_getopt='', time_out=10, protocol='xmlrpc/ssl', certfile=None, keyfile=None, ca=None): @@ -64,8 +64,13 @@ def run_component(component_cls, location, daemon, pidfile_name, to_file, port = tuple(up[1].split(':')) port = (port[0], int(port[1])) try: - server = XMLRPCServer(port, keyfile=keyfile, certfile=certfile, - register=register, timeout=time_out, ca=ca, + server = XMLRPCServer(listen_all, + port, + keyfile=keyfile, + certfile=certfile, + register=register, + timeout=time_out, + ca=ca, protocol=protocol) except: logger.error("Server startup failed") diff --git a/src/lib/Options.py b/src/lib/Options.py index e6eebb808..9980566fb 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -173,6 +173,18 @@ def colon_split(c_string): return c_string.split(':') return [] +def get_bool(s): + # these values copied from ConfigParser.RawConfigParser.getboolean + # with the addition of True and False + truelist = ["1", "yes", "True", "true", "on"] + falselist = ["0", "no", "False", "false", "off"] + if s in truelist: + return True + elif s in falselist: + return False + else: + raise ValueError + # General options CFILE = Option('Specify configuration file', DEFAULT_CONFIG_LOCATION, cmd='-C', odesc='') @@ -249,6 +261,13 @@ SERVER_MCONNECT = Option('Server Metadata Connector list', cook=list_split, cf=('server', 'connectors'), default=['Probes'], ) SERVER_FILEMONITOR = Option('Server file monitor', cf=('server', 'filemonitor'), default='default', odesc='File monitoring driver') +SERVER_LISTEN_ALL = Option('Listen on all interfaces', + cf=('server', 'listen_all'), + cmd='--listen-all', + default=False, + long_arg=True, + cook=get_bool, + odesc='True|False') SERVER_LOCATION = Option('Server Location', cf=('components', 'bcfg2'), default='https://localhost:6789', cmd='-S', odesc='https://server:port') diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index a89beabbb..8cac8a53f 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -79,9 +79,9 @@ class SSLServer (SocketServer.TCPServer, object): allow_reuse_address = True logger = logging.getLogger("Cobalt.Server.TCPServer") - def __init__(self, server_address, RequestHandlerClass, keyfile=None, - certfile=None, reqCert=False, ca=None, timeout=None, - protocol='xmlrpc/ssl'): + def __init__(self, listen_all, server_address, RequestHandlerClass, + keyfile=None, certfile=None, reqCert=False, ca=None, + timeout=None, protocol='xmlrpc/ssl'): """Initialize the SSL-TCP server. @@ -97,9 +97,12 @@ class SSLServer (SocketServer.TCPServer, object): """ - all_iface_address = ('', server_address[1]) + if listen_all: + listen_address = ('', server_address[1]) + else: + listen_address = (server_address[0], server_address[1]) try: - SocketServer.TCPServer.__init__(self, all_iface_address, + SocketServer.TCPServer.__init__(self, listen_address, RequestHandlerClass) except socket.error: self.logger.error("Failed to bind to socket") @@ -310,7 +313,7 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer, """ - def __init__(self, server_address, RequestHandlerClass=None, + def __init__(self, listen_all, server_address, RequestHandlerClass=None, keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', timeout=10, logRequests=False, @@ -339,8 +342,14 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer, """A subclassed request handler to prevent class-attribute conflicts.""" SSLServer.__init__(self, - server_address, RequestHandlerClass, ca=ca, - timeout=timeout, keyfile=keyfile, certfile=certfile, protocol=protocol) + listen_all, + server_address, + RequestHandlerClass, + ca=ca, + timeout=timeout, + keyfile=keyfile, + certfile=certfile, + protocol=protocol) self.logRequests = logRequests self.serve = False self.register = register diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index f4bd5e5b7..546d5a249 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -35,6 +35,7 @@ if __name__ == '__main__': OPTINFO.update({'key' : Bcfg2.Options.SERVER_KEY, 'cert' : Bcfg2.Options.SERVER_CERT, 'ca' : Bcfg2.Options.SERVER_CA, + 'listen_all' : Bcfg2.Options.SERVER_LISTEN_ALL, 'location' : Bcfg2.Options.SERVER_LOCATION, 'passwd' : Bcfg2.Options.SERVER_PASSWORD, 'static' : Bcfg2.Options.SERVER_STATIC, @@ -51,6 +52,7 @@ if __name__ == '__main__': print("Could not read %s" % setup['configfile']) sys.exit(1) Bcfg2.Component.run_component(Bcfg2.Server.Core.Core, + listen_all=setup['listen_all'], location=setup['location'], daemon = setup['daemon'], pidfile_name = setup['daemon'], -- cgit v1.2.3-1-g7c22 From 368b34bcdd509d24bcc82ccbdbd056455bba7c37 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 3 Jun 2011 19:10:18 -0500 Subject: Base: Deprecate Base in favor of Bundler Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Base.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 5e7d89727..5de57a87c 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = Bcfg2.Server.Plugin.StructFile + deprecated = True """Base creates independent clauses based on client metadata.""" def __init__(self, core, datastore): -- cgit v1.2.3-1-g7c22 From 80fb796b9a02403e5e90e7bde73fb2e44b3f5222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Sat, 4 Jun 2011 23:52:02 +0200 Subject: Document the "decision" option in bcfg2.conf(5) Add documentation for the "decision" option to the bcfg2.conf(5) man page. Also, note that it can be overridden using "-l none" on the bcfg2(1) command line. --- src/lib/Options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Options.py b/src/lib/Options.py index 9980566fb..5c0829df7 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -325,9 +325,9 @@ CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, cmd='-z') CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) -CLIENT_DLIST = Option('Run client in server decision list mode', default=False, +CLIENT_DLIST = Option('Run client in server decision list mode', default='none', cf=('client', 'decision'), - cmd='-l', odesc='') + cmd='-l', odesc='') CLIENT_FILE = Option('Configure from a file rather than querying the server', default=False, cmd='-f', odesc='') CLIENT_QUICK = Option('Disable some checksum verification', default=False, -- cgit v1.2.3-1-g7c22 From 260cd8caafdd26ccdcf1f999eaaa8ff2efde25a6 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 4 Jun 2011 17:49:01 -0500 Subject: Proxy: Catch traceback when name resolution fails (#1012) Signed-off-by: Sol Jerome --- src/lib/Proxy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index 9656b166c..e4a0f6a3d 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -193,7 +193,13 @@ class SSLHTTPConnection(httplib.HTTPConnection): ca_certs=self.ca, suppress_ragged_eofs=True, keyfile=self.key, certfile=self.cert, ssl_version=ssl_protocol_ver) - self.sock.connect((self.host, self.port)) + try: + self.sock.connect((self.host, self.port)) + except socket.gaierror: + e = sys.exc_info()[1] + self.logger.error("Unable to connect to %s:%s\n%s" % + (self.host, self.port, e.strerror)) + sys.exit(1) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] -- cgit v1.2.3-1-g7c22 From 70d480f2a77902347601596a7ab8f606a4b4290d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sun, 5 Jun 2011 14:00:56 -0500 Subject: Validate: Unsuppress errors for invalid schema paths (#1007) Signed-off-by: Sol Jerome --- src/lib/Server/Lint/Validate.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c9b5688e1..a561232b6 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -1,10 +1,12 @@ +import fnmatch import glob import lxml.etree import os -import fnmatch +from subprocess import Popen, PIPE, STDOUT +import sys + import Bcfg2.Options import Bcfg2.Server.Lint -from subprocess import Popen, PIPE, STDOUT class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that the repo validates """ @@ -47,6 +49,10 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname % schemadir)) + except IOError: + e = sys.exc_info()[1] + self.LintError("input/output error", e.message) + continue except: self.LintError("schema-failed-to-parse", "Failed to process schema %s" % -- cgit v1.2.3-1-g7c22 From b2e9d11b0dca1af6e0ce9a9f21558b35b35f5777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 16:32:53 +0200 Subject: Accept non-ASCII diffs Currently, client reports don't include diffs of files which aren't US-ASCII encoded. The client transmits such files as Base64 blobs. As we'd like to change that, this commit teaches the server to properly handle non-ASCII diffs. --- src/lib/Server/Admin/Reports.py | 6 ++++++ src/lib/Server/Admin/__init__.py | 5 +++-- src/lib/Server/Plugins/Cfg.py | 7 ++++++- src/lib/Server/Plugins/DBStats.py | 1 + src/lib/Server/Reports/importscript.py | 18 +++++++++++++++--- 5 files changed, 31 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 942477a49..c9f3d3f58 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode): except (IOError, XMLSyntaxError): self.errExit("StatReports: Failed to parse %s" % (stats_file)) + try: + encoding = self.cfp.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientspath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: load_stats(clientsdata, statsdata, + encoding, verb, self.log, quick=quick, diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index 8915492a3..b34d7108c 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -113,7 +113,8 @@ class MetadataCore(Mode): def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None): Mode.__init__(self, configfile) options = {'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'configfile': Bcfg2.Options.CFILE} + 'configfile': Bcfg2.Options.CFILE, + 'encoding': Bcfg2.Options.ENCODING} setup = Bcfg2.Options.OptionParser(options) setup.hm = usage setup.parse(sys.argv[1:]) @@ -126,7 +127,7 @@ class MetadataCore(Mode): try: self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), setup['plugins'], - 'foo', 'UTF-8') + 'foo', setup['encoding']) except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed because %s" % msg) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 5e3cca847..c08e8c4b6 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -197,7 +197,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(new_entry['text']) + try: + etext = new_entry['text'].encode(self.encoding) + except: + logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) + raise Bcfg2.Server.Plugin.PluginExecutionError + open(name, 'w').write(etext) if log: logger.info("Wrote file %s" % name) badattr = [attr for attr in ['owner', 'group', 'perms'] diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py index 5ef1920e1..103fb7353 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Server/Plugins/DBStats.py @@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, try: Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, container, + self.core.encoding, 0, logger, True, diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index b6a3c2599..68774cec6 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -38,7 +38,7 @@ import platform from Bcfg2.Bcfg2Py3k import ConfigParser -def build_reason_kwargs(r_ent): +def build_reason_kwargs(r_ent, encoding, logger): binary_file = False if r_ent.get('current_bfile', False): binary_file = True @@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent): rc_diff = r_ent.get('current_diff') else: rc_diff = '' + if not binary_file: + try: + rc_diff = rc_diff.decode(encoding) + except: + logger.error("Reason isn't %s encoded, cannot decode it" % encoding) + rc_diff = '' return dict(owner=r_ent.get('owner', default=""), current_owner=r_ent.get('current_owner', default=""), group=r_ent.get('group', default=""), @@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent): is_binary=binary_file) -def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): +def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): clients = {} [clients.__setitem__(c.name, c) \ for c in Client.objects.all()] @@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): for (xpath, type) in pattern: for x in statistics.findall(xpath): counter_fields[type] = counter_fields[type] + 1 - kargs = build_reason_kwargs(x) + kargs = build_reason_kwargs(x, encoding, logger) try: rr = None @@ -270,6 +276,11 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s" % (statpath)) raise SystemExit(1) + try: + encoding = cf.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -288,6 +299,7 @@ if __name__ == '__main__': update_database() load_stats(clientsdata, statsdata, + encoding, verb, logger, quick=q, -- cgit v1.2.3-1-g7c22 From c55c7f92d4971eedd2e7eafa18e0cac424b637db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 17:06:16 +0200 Subject: Create non-ASCII diffs The client now also transmits diffs of files which include non-US-ASCII data (using the encoding setting from Options.py), unless they look like binary files. In the past, non-ASCII files were transmitted as Base64 blobs. In addition, "bcfg2 -I" no longer refuses to display non-ASCII diffs. Resolves ticket #999. --- src/lib/Client/Frame.py | 3 ++- src/lib/Client/Tools/POSIX.py | 28 +++++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index 60d158eb1..57844ab19 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -5,6 +5,7 @@ installs entries, and generates statistics. __revision__ = '$Revision$' import logging +import sys import time import Bcfg2.Client.Tools @@ -29,7 +30,7 @@ def promptFilter(prompt, entries): try: # py3k compatibility try: - ans = raw_input(iprompt) + ans = raw_input(iprompt.encode(sys.stdout.encoding, 'replace')) except NameError: ans = input(iprompt) if ans in ['y', 'Y']: diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 875db5ea7..862e0bc04 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -76,6 +76,21 @@ def normUid(entry): return False +def isString(strng, encoding): + """ + Returns true if the string contains no ASCII control characters + and can be decoded from the specified encoding. + """ + for char in strng: + if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: + return False + try: + strng.decode(encoding) + return True + except: + return False + + class POSIX(Bcfg2.Client.Tools.Tool): """POSIX File support code.""" name = 'POSIX' @@ -458,12 +473,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): # md5sum so it would be faster for big binary files contentStatus = content == tempdata if not contentStatus: - try: - content.decode('ascii') - isstring = True - except: - isstring = False - if tbin or not isstring: + if tbin or not isString(content, self.setup['encoding']): entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') nqtext += '\nBinary file, no printable diff' @@ -493,15 +503,15 @@ class POSIX(Bcfg2.Client.Tools.Tool): difflib.unified_diff(content.split('\n'), \ tempdata.split('\n'))]) try: - eudiff = udiff.encode('ascii') + dudiff = udiff.decode(self.setup['encoding']) except: - eudiff = "Binary file: no diff printed" + dudiff = "Binary file: no diff printed" nqtext = entry.get('qtext', '') if nqtext: nqtext += '\n' - nqtext += eudiff + nqtext += dudiff else: entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') -- cgit v1.2.3-1-g7c22 From 129aa6d584765fba5b0ff686d5d8da62b5ae405c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 6 Jun 2011 12:36:56 -0500 Subject: Validate: Create and use new error type for missing schemas Signed-off-by: Sol Jerome --- src/lib/Server/Lint/Validate.py | 2 +- src/lib/Server/Lint/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index a561232b6..8a8406e73 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -51,7 +51,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): schemadir)) except IOError: e = sys.exc_info()[1] - self.LintError("input/output error", e.message) + self.LintError("input-output-error", e.message) continue except: self.LintError("schema-failed-to-parse", diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 013cbf2ba..63cb2463b 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -88,7 +88,8 @@ class ErrorHandler (object): "xml-failed-to-read":"error", "xml-failed-to-verify":"error", "merge-cfg":"warning", - "merge-probes":"warning",} + "merge-probes":"warning", + "input-output-error": "error"} def __init__(self, config=None): self.errors = 0 -- cgit v1.2.3-1-g7c22 From 00888897e4a07f9f83b1c890f70382609c856199 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 6 Jun 2011 18:30:46 -0500 Subject: =?UTF-8?q?Options:=20Set=20default=20encoding=20to=20UTF-8=20(as?= =?UTF-8?q?=20per=20Holger=20Wei=C3=9F's=20suggestion)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sol Jerome --- src/lib/Options.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Options.py b/src/lib/Options.py index 5c0829df7..0d978c519 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -203,8 +203,10 @@ SENDMAIL_PATH = Option('Path to sendmail', cf=('reports', 'sendmailpath'), default='/usr/lib/sendmail') INTERACTIVE = Option('Prompt the user for each change', default=False, cmd='-I', ) -ENCODING = Option('Encoding of cfg files', default=sys.getdefaultencoding(), - cmd='-E', odesc='', +ENCODING = Option('Encoding of cfg files', + default='UTF-8', + cmd='-E', + odesc='', cf=('components', 'encoding')) PARANOID_PATH = Option('Specify path for paranoid file backups', default='/var/cache/bcfg2', cf=('paranoid', 'path'), -- cgit v1.2.3-1-g7c22 From fdda835c1c6181589f1eb5fa889056ac26a09347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Tue, 7 Jun 2011 13:22:00 +0200 Subject: bcfg2-reports: Show modified entries bcfg2-reports now shows modified entries if it's called with --modifiedentry, -m, or -s. --- src/sbin/bcfg2-reports | 95 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 33a291395..b5609db9f 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -104,6 +104,7 @@ def print_entry(item, max_name): fields = "" sort = "" badentry = "" +modifiedentry = "" extraentry = "" expire = "" singlehost = "" @@ -114,8 +115,8 @@ result = list() entrydict = dict() args = sys.argv[1:] -opts, pargs = getopt(args, 'ab:cde:hs:x:', - ['stale', 'sort=', 'fields=', 'badentry=', 'extraentry=']) +opts, pargs = getopt(args, 'ab:cde:hm:s:x:', + ['stale', 'sort=', 'fields=', 'badentry=', 'modifiedentry=', 'extraentry=']) for option in opts: if len(option) > 0: @@ -125,11 +126,13 @@ for option in opts: sort = option[1] if option[0] == '--badentry': badentry = option[1] + if option[0] == '--modifiedentry': + modifiedentry = option[1] if option[0] == '--extraentry': extraentry = option[1] if option[0] == '-x': expire = option[1] - if option[0] == '-s' or option[0] == '-b' or option[0] == '-e': + if option[0] == '-s' or option[0] == '-b' or option[0] == '-m' or option[0] == '-e': singlehost = option[1] if expire != "": @@ -147,29 +150,35 @@ elif '-h' in args: print("""Usage: bcfg2-reports [option] ... Options and arguments (and corresponding environment variables): --a : shows all hosts, including expired hosts --b NAME : single-host mode - shows bad entries from the - current interaction of NAME --c : shows only clean hosts --d : shows only dirty hosts --e NAME : single-host mode - shows extra entries from the - current interaction of NAME --h : shows help and usage info about bcfg2-reports --s NAME : single-host mode - shows bad and extra entries from - the current interaction of NAME --x NAME : toggles expired/unexpired state of NAME ---badentry=KIND,NAME : shows only hosts whose current interaction has bad - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 ---extraentry=KIND,NAME : shows only hosts whose current interaction has extra - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 ---fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... - (name,time,state) ---sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) ---stale : shows hosts which haven't run in the last 24 hours +-a : shows all hosts, including expired hosts +-b NAME : single-host mode - shows bad entries from the + current interaction of NAME +-c : shows only clean hosts +-d : shows only dirty hosts +-e NAME : single-host mode - shows extra entries from the + current interaction of NAME +-h : shows help and usage info about bcfg2-reports +-m NAME : single-host mode - shows modified entries from the + current interaction of NAME +-s NAME : single-host mode - shows bad, modified, and extra + entries from the current interaction of NAME +-x NAME : toggles expired/unexpired state of NAME +--badentry=KIND,NAME : shows only hosts whose current interaction has bad + entries in of KIND kind and NAME name; if a single + argument ARG1 is given, then KIND,NAME pairs will be + read from a file of name ARG1 +--modifiedentry=KIND,NAME : shows only hosts whose current interaction has + modified entries in of KIND kind and NAME name; if a + single argument ARG1 is given, then KIND,NAME pairs + will be read from a file of name ARG1 +--extraentry=KIND,NAME : shows only hosts whose current interaction has extra + entries in of KIND kind and NAME name; if a single + argument ARG1 is given, then KIND,NAME pairs will be + read from a file of name ARG1 +--fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... + (name,time,state) +--sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) +--stale : shows hosts which haven't run in the last 24 hours """) elif singlehost != "": for c_inst in c_list: @@ -183,6 +192,15 @@ elif singlehost != "": max_name = len(item.entry.name) for item in baditems: print_entry(item, max_name) + modifieditems = c_inst.current_interaction.modified() + if len(modifieditems) > 0 and ('-m' in args or '-s' in args): + print "Modified Entries:" + max_name = -1 + for item in modifieditems: + if len(item.entry.name) > max_name: + max_name = len(item.entry.name) + for item in modifieditems: + print_entry(item, max_name) extraitems = c_inst.current_interaction.extra() if len(extraitems) > 0 and ('-e' in args or '-s' in args): print("Extra Entries:") @@ -206,6 +224,9 @@ else: if badentry != "": badentry = badentry.split(',') + if modifiedentry != "": + modifiedentry = modifiedentry.split(',') + if extraentry != "": extraentry = extraentry.split(',') @@ -247,6 +268,28 @@ else: if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: result.append(c_inst) break + elif modifiedentry != "": + if len(modifiedentry) == 1: + fileread = fileinput.input(modifiedentry[0]) + for line in fileread: + modifiedentry = line.strip().split(',') + for c_inst in c_list: + modifieditems = c_inst.current_interaction.modified() + for item in modifieditems: + if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]: + result.append(c_inst) + if c_inst in entrydict: + entrydict.get(c_inst).append(modifiedentry[1]) + else: + entrydict[c_inst] = [modifiedentry[1]] + break + else: + for c_inst in c_list: + modifieditems = c_inst.current_interaction.modified() + for item in modifieditems: + if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]: + result.append(c_inst) + break elif extraentry != "": if len(extraentry) == 1: fileread = fileinput.input(extraentry[0]) -- cgit v1.2.3-1-g7c22 From 80929a6e73eeece44997b84f2beeb38eb3d7645d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 8 Jun 2011 17:16:44 +0200 Subject: bcfg2-reports: Show total numbers of entries Add a "-t NAME" option which reports the total (and good) number of managed entries on the host NAME. Also, allow for specifying "total", "good", and "bad" fields via --fields and --sort. --- src/sbin/bcfg2-reports | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index b5609db9f..36283f87d 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#!/usr/bin/python """Query reporting system for client status.""" __revision__ = '$Revision$' @@ -47,6 +47,23 @@ def statecompare(client1, client2): else: return 0 +def totalcompare(client1, client2): + """Compares two clients by their total entry counts.""" + return cmp(client2.current_interaction.totalcount, \ + client1.current_interaction.totalcount) + +def goodcompare(client1, client2): + """Compares two clients by their good entry counts.""" + return cmp(client2.current_interaction.goodcount, \ + client1.current_interaction.goodcount) + +def badcompare(client1, client2): + """Compares two clients by their bad entry counts.""" + return cmp(client2.current_interaction.totalcount - \ + client2.current_interaction.goodcount, \ + client1.current_interaction.totalcount - \ + client1.current_interaction.goodcount) + def crit_compare(criterion, client1, client2): """Compares two clients by the criteria provided in criterion.""" for crit in criterion: @@ -57,6 +74,12 @@ def crit_compare(criterion, client1, client2): comp = statecompare(client1, client2) elif crit == 'time': comp = timecompare(client1, client2) + elif crit == 'total': + comp = totalcompare(client1, client2) + elif crit == 'good': + comp = goodcompare(client1, client2) + elif crit == 'bad': + comp = badcompare(client1, client2) if comp != 0: return comp @@ -83,6 +106,13 @@ def print_fields(fields, cli, max_name, entrydict): fdata.append("clean") else: fdata.append("dirty") + elif field == 'total': + fdata.append("%5d" % cli.current_interaction.totalcount) + elif field == 'good': + fdata.append("%5d" % cli.current_interaction.goodcount) + elif field == 'bad': + fdata.append("%5d" % cli.current_interaction.totalcount \ + - cli.current_interaction.goodcount) else: try: fdata.append(getattr(cli, field)) @@ -115,7 +145,7 @@ result = list() entrydict = dict() args = sys.argv[1:] -opts, pargs = getopt(args, 'ab:cde:hm:s:x:', +opts, pargs = getopt(args, 'ab:cde:hm:s:t:x:', ['stale', 'sort=', 'fields=', 'badentry=', 'modifiedentry=', 'extraentry=']) for option in opts: @@ -132,7 +162,11 @@ for option in opts: extraentry = option[1] if option[0] == '-x': expire = option[1] - if option[0] == '-s' or option[0] == '-b' or option[0] == '-m' or option[0] == '-e': + if option[0] == '-s' or \ + option[0] == '-t' or \ + option[0] == '-b' or \ + option[0] == '-m' or \ + option[0] == '-e': singlehost = option[1] if expire != "": @@ -162,6 +196,8 @@ Options and arguments (and corresponding environment variables): current interaction of NAME -s NAME : single-host mode - shows bad, modified, and extra entries from the current interaction of NAME +-t NAME : single-host mode - shows total number of managed and + good entries from the current interaction of NAME -x NAME : toggles expired/unexpired state of NAME --badentry=KIND,NAME : shows only hosts whose current interaction has bad entries in of KIND kind and NAME name; if a single @@ -183,6 +219,10 @@ Options and arguments (and corresponding environment variables): elif singlehost != "": for c_inst in c_list: if singlehost == c_inst.name: + if '-t' in args: + managed = c_inst.current_interaction.totalcount + good = c_inst.current_interaction.goodcount + print("Total managed entries: %d (good: %d)" % (managed, good)) baditems = c_inst.current_interaction.bad() if len(baditems) > 0 and ('-b' in args or '-s' in args): print("Bad Entries:") -- cgit v1.2.3-1-g7c22 From 0133621823bdc7623875d926f235607c7f71b5ff Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 8 Jun 2011 19:40:54 -0500 Subject: bcfg2-reports: Revert shebang line modification Signed-off-by: Sol Jerome --- src/sbin/bcfg2-reports | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 36283f87d..9a4c6e60d 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """Query reporting system for client status.""" __revision__ = '$Revision$' -- cgit v1.2.3-1-g7c22 From bacc03705454a8e3c5f8c670b4823927587f1963 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 09:57:50 -0500 Subject: Frame: Fix 'important' entry installation in decision mode (Reported by m4z on irc) Signed-off-by: Sol Jerome --- src/lib/Client/Frame.py | 86 ++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 36 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index 57844ab19..e33e1a7df 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -123,22 +123,7 @@ class Frame: self.logger.info("Loaded tool drivers:") self.logger.info([tool.name for tool in self.tools]) - if not self.dryrun and not self.setup['bundle']: - for cfile in [cfl for cfl in config.findall(".//Path") \ - if cfl.get('name') in self.__important__ and \ - cfl.get('type') == 'file']: - tl = [t for t in self.tools if t.handlesEntry(cfile) \ - and t.canVerify(cfile)] - if tl: - if not tl[0].VerifyPath(cfile, []): - if self.setup['interactive'] and not \ - promptFilter("Install %s: %s? (y/N):", [cfile]): - continue - try: - self.states[cfile] = tl[0].InstallPath(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) + # find entries not handled by any tools problems = [entry for struct in config for \ entry in struct if entry not in self.handled] @@ -177,6 +162,54 @@ class Frame: return self.__dict__[name] raise AttributeError(name) + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode works with it + self.whitelist = [entry for entry in self.states \ + if not self.states[entry]] + if self.setup['decision'] == 'whitelist': + dwl = self.setup['decision_list'] + w_to_rem = [e for e in self.whitelist \ + if not matches_white_list(e, dwl)] + if w_to_rem: + self.logger.info("In whitelist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) + self.whitelist = [x for x in self.whitelist \ + if x not in w_to_rem] + + elif self.setup['decision'] == 'blacklist': + b_to_rem = [e for e in self.whitelist \ + if not passes_black_list(e, self.setup['decision_list'])] + if b_to_rem: + self.logger.info("In blacklist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist if x not in b_to_rem] + + # take care of important entries first + if not self.dryrun and not self.setup['bundle']: + for cfile in [cfl for cfl in self.config.findall(".//Path") \ + if cfl.get('name') in self.__important__ and \ + cfl.get('type') == 'file']: + if cfile not in self.whitelist: + continue + tl = [t for t in self.tools if t.handlesEntry(cfile) \ + and t.canVerify(cfile)] + if tl: + if not tl[0].VerifyPath(cfile, []): + if self.setup['interactive'] and not \ + promptFilter("Install %s: %s? (y/N):", [cfile]): + continue + try: + self.states[cfile] = tl[0].InstallPath(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + def Inventory(self): """ Verify all entries, @@ -210,26 +243,6 @@ class Frame: candidates = [entry for entry in self.states \ if not self.states[entry]] - self.whitelist = [entry for entry in self.states \ - if not self.states[entry]] - # Need to process decision stuff early so that dryrun mode works with it - if self.setup['decision'] == 'whitelist': - dwl = self.setup['decision_list'] - w_to_rem = [e for e in self.whitelist \ - if not matches_white_list(e, dwl)] - if w_to_rem: - self.logger.info("In whitelist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) - self.whitelist = [x for x in self.whitelist \ - if x not in w_to_rem] - - elif self.setup['decision'] == 'blacklist': - b_to_rem = [e for e in self.whitelist \ - if not passes_black_list(e, self.setup['decision_list'])] - if b_to_rem: - self.logger.info("In blacklist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) - self.whitelist = [x for x in self.whitelist if x not in b_to_rem] if self.dryrun: if self.whitelist: @@ -389,6 +402,7 @@ class Frame: self.Inventory() self.times['inventory'] = time.time() self.CondDisplayState('initial') + self.InstallImportant() self.Decide() self.Install() self.times['install'] = time.time() -- cgit v1.2.3-1-g7c22 From 0b074d1d8dff851081a892e6a6755c36f51c189f Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 10:34:13 -0500 Subject: Init: Remove Base from default plugin list Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index fff8bcd1c..a9170530e 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,7 +77,6 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', - 'Base', 'Bundler', 'Bzr', 'Cfg', -- cgit v1.2.3-1-g7c22 From 733f3e51f2e1986005b37caeedca8edf4c464471 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 10:41:58 -0500 Subject: YUM: Make YUMng the preferred client tool on RPM systems Signed-off-by: Sol Jerome --- src/lib/Client/Tools/YUM24.py | 2 -- src/lib/Client/Tools/YUMng.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Client/Tools/YUM24.py index 04d9f5c07..5387fdd6a 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Client/Tools/YUM24.py @@ -76,8 +76,6 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): __new_gpg_ireq__ = {'Package': ['name'], 'Instance': ['version', 'release']} - conflicts = ['YUMng', 'RPMng'] - def __init__(self, logger, setup, config): Bcfg2.Client.Tools.RPMng.RPMng.__init__(self, logger, setup, config) self.__important__ = self.__important__ + \ diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index c9e7aa15e..7299ab4c0 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -141,7 +141,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): 'Path': ['type']} __ireq__ = {'Package': ['name']} - conflicts = ['RPMng'] + conflicts = ['YUMng', 'RPMng'] def __init__(self, logger, setup, config): self.yb = yum.YumBase() -- cgit v1.2.3-1-g7c22 From fa45b31482f86cc75ca58123096d08dde015f48b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 10:47:14 -0500 Subject: YUMng: Fix typo in last commit Signed-off-by: Sol Jerome --- src/lib/Client/Tools/YUMng.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 7299ab4c0..34d463941 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -141,7 +141,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): 'Path': ['type']} __ireq__ = {'Package': ['name']} - conflicts = ['YUMng', 'RPMng'] + conflicts = ['YUM24', 'RPMng'] def __init__(self, logger, setup, config): self.yb = yum.YumBase() -- cgit v1.2.3-1-g7c22 From e4f2ac90cb173a75d7e39d643b3e8147085ecca1 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 10 Jun 2011 10:47:46 -0500 Subject: POSIX: Detect missing cache directory failures Signed-off-by: Sol Jerome --- src/lib/Client/Tools/POSIX.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 862e0bc04..64821f621 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -14,6 +14,7 @@ import os import pwd import shutil import stat +import sys import time import Bcfg2.Client.Tools import Bcfg2.Options @@ -559,8 +560,14 @@ class POSIX(Bcfg2.Client.Tools.Tool): (entry.get('current_exists', 'true') == 'false'): bkupnam = entry.get('name').replace('/', '_') # current list of backups for this file - bkuplist = [f for f in os.listdir(self.ppath) if - f.startswith(bkupnam)] + try: + bkuplist = [f for f in os.listdir(self.ppath) if + f.startswith(bkupnam)] + except OSError: + e = sys.exc_info()[1] + self.logger.error("Failed to create backup list in %s: %s" % + (self.ppath, e.strerror)) + return False bkuplist.sort() while len(bkuplist) >= int(self.max_copies): # remove the oldest backup available -- cgit v1.2.3-1-g7c22 From 36aadd161281334c53b39ae028afbb477e0228ec Mon Sep 17 00:00:00 2001 From: Jack Neely Date: Fri, 10 Jun 2011 13:40:41 -0400 Subject: Make -q turn off package verification in YUMng This make the YUMng verification behavior like the APT tool. Before hand we just turned off package checksums which only worked with very new versions of Yum. --- src/lib/Client/Tools/YUMng.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 34d463941..324307f7f 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -485,6 +485,9 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): qtext_versions.append("U(%s)" % str(POs[0])) continue + if self.setup.get('quick', False): + # Passed -q on the command line + continue if not (pkg_verify and \ inst.get('pkg_verify', 'true').lower() == 'true'): continue -- cgit v1.2.3-1-g7c22 From da0fb10e340f3fd8e97ef29810fd0f59173492e7 Mon Sep 17 00:00:00 2001 From: Jack Neely Date: Fri, 10 Jun 2011 16:45:30 -0400 Subject: Addition Yum error handling Address ticket #955 and attempt to capture the exceptions that may possibly be generated from Yum or Yum's plugins and return some sort of error message. --- src/lib/Client/Tools/YUMng.py | 50 +++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 324307f7f..0435ca0d7 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -170,7 +170,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): except yum.Errors.RepoError, e: self.logger.error("YUMng Repository error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError - except yum.Errors.YumBaseError, e: + except Exception, e: self.logger.error("YUMng error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError @@ -671,38 +671,56 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): return True def _runYumTransaction(self): + def cleanup(): + self.yb.closeRpmDB() + self.RefreshPackages() + rDisplay = RPMDisplay(self.logger) yDisplay = YumDisplay(self.logger) # Run the Yum Transaction - rescode, restring = self.yb.buildTransaction() + try: + rescode, restring = self.yb.buildTransaction() + except yum.Errors.YumBaseError, e: + self.logger.error("Yum transaction error: %s" % str(e)) + cleanup() + return + self.logger.debug("Initial Yum buildTransaction() run said:") self.logger.debug(" resultcode: %s, msgs: %s" \ % (rescode, restring)) if rescode != 1: # Transaction built successfully, run it - self.yb.processTransaction(callback=yDisplay, - rpmDisplay=rDisplay) - self.logger.info("Single Pass for Install Succeeded") + try: + self.yb.processTransaction(callback=yDisplay, + rpmDisplay=rDisplay) + self.logger.info("Single Pass for Install Succeeded") + except yum.Errors.YumBaseError, e: + self.logger.error("Yum transaction error: %s" % str(e)) + cleanup() + return else: # The yum command failed. No packages installed. # Try installing instances individually. self.logger.error("Single Pass Install of Packages Failed") skipBroken = self.yb.conf.skip_broken self.yb.conf.skip_broken = True - rescode, restring = self.yb.buildTransaction() - if rescode != 1: - self.yb.processTransaction(callback=yDisplay, - rpmDisplay=rDisplay) - self.logger.debug( - "Second pass install did not install all packages") - else: - self.logger.error("Second pass yum install failed.") - self.logger.debug(" %s" % restring) + try: + rescode, restring = self.yb.buildTransaction() + if rescode != 1: + self.yb.processTransaction(callback=yDisplay, + rpmDisplay=rDisplay) + self.logger.debug( + "Second pass install did not install all packages") + else: + self.logger.error("Second pass yum install failed.") + self.logger.debug(" %s" % restring) + except yum.Errors.YumBaseError, e: + self.logger.error("Yum transaction error: %s" % str(e)) + self.yb.conf.skip_broken = skipBroken - self.yb.closeRpmDB() - self.RefreshPackages() + cleanup() def Install(self, packages, states): """ -- cgit v1.2.3-1-g7c22 From 99a4c8ed71fd15523fc9c6120e0a2fdac38ca092 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 08:11:16 -0500 Subject: bcfg2-info: Fix print formatting Signed-off-by: Sol Jerome --- src/sbin/bcfg2-info | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index c36e1af42..07953ae69 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -375,21 +375,21 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): except: print("Client %s not defined" % client) continue - print("Hostname:\t", client_meta.hostname) - print("Profile:\t", client_meta.profile) - print("Groups:\t\t", list(client_meta.groups)[0]) + print("Hostname:\t%s" % client_meta.hostname) + print("Profile:\t%s" % client_meta.profile) + print("Groups:\t\t%s" % list(client_meta.groups)[0]) for grp in list(client_meta.groups)[1:]: - print('\t\t%s' % grp) + print("\t\t%s" % grp) if client_meta.bundles: - print("Bundles:\t", list(client_meta.bundles)[0]) + print("Bundles:\t%s" % list(client_meta.bundles)[0]) for bnd in list(client_meta.bundles)[1:]: - print('\t\t%s' % bnd) + print("\t\t%s" % bnd) if client_meta.connectors: print("Connector data") print("=" * 80) for conn in client_meta.connectors: if getattr(client_meta, conn): - print("%s:\t" % (conn), getattr(client_meta, conn)) + print("%s:\t%s" % (conn, getattr(client_meta, conn))) print("=" * 80) def do_mappings(self, args): -- cgit v1.2.3-1-g7c22 From c1dfee4c7ed9eb1d8ca62d95182552cc847ec52e Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 11:24:42 -0500 Subject: Revert "Init: Remove Base from default plugin list" This reverts commit 611ce16c8cef81a6fc754c46dcb5cbe618b20b67. --- src/lib/Server/Admin/Init.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index a9170530e..fff8bcd1c 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,6 +77,7 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', + 'Base', 'Bundler', 'Bzr', 'Cfg', -- cgit v1.2.3-1-g7c22 From 2c9a76cdfcce02f8d89129405f1f477753c47d3c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 11:25:13 -0500 Subject: Init: Remove Base from default plugins list Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index fff8bcd1c..eab030cf8 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -103,8 +103,7 @@ plugin_list = ['Account', 'TGenshi'] # Default list of plugins to use -default_plugins = ['Base', - 'Bundler', +default_plugins = ['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', -- cgit v1.2.3-1-g7c22 From b1ab3e2bab9f07c13daf5dcfd4a9502eb84dcf0d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 14 Jun 2011 15:23:31 -0500 Subject: PY3K: Finish server-side code fixes Signed-off-by: Sol Jerome --- src/lib/SSLServer.py | 18 ++++++++++++++---- src/lib/Server/Core.py | 4 ++-- src/lib/Server/Plugin.py | 3 +++ src/lib/Server/Plugins/Cfg.py | 5 +++-- src/lib/Server/Plugins/Metadata.py | 2 +- src/lib/Server/Plugins/Probes.py | 5 +++-- src/lib/Server/Plugins/SSHbase.py | 21 +++++++++++++-------- 7 files changed, 39 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index 8cac8a53f..bb0f61198 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -46,7 +46,11 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher): if '.' not in method: params = (address, ) + params response = self.instance._dispatch(method, params, self.funcs) - response = (response, ) + # py3k compatibility + if isinstance(response, bool) or isinstance(response, str): + response = (response, ) + else: + response = (response.decode('utf-8'), ) raw_response = xmlrpclib.dumps(response, methodresponse=1, allow_none=self.allow_none, encoding=self.encoding) @@ -191,9 +195,14 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): self.logger.error("No authentication data presented") return False auth_type, auth_content = header.split() - auth_content = base64.standard_b64decode(auth_content) + auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) try: - username, password = auth_content.split(":") + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(b":") + password = pw.decode('utf-8') except ValueError: username = auth_content password = "" @@ -234,11 +243,12 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): print("got select timeout") raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size)) + L.append(self.rfile.read(chunk_size).decode('utf-8')) size_remaining -= len(L[-1]) data = ''.join(L) response = self.server._marshaled_dispatch(self.client_address, data) + response = response.encode('utf-8') except: try: self.send_response(500) diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 8f9d3e746..5adfa5381 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -384,7 +384,7 @@ class Core(Component): # clear dynamic groups self.metadata.cgroups[meta.hostname] = [] try: - xpdata = lxml.etree.XML(probedata) + xpdata = lxml.etree.XML(probedata.encode('utf-8')) except: self.logger.error("Failed to parse probe data from client %s" % \ (address[0])) @@ -433,7 +433,7 @@ class Core(Component): @exposed def RecvStats(self, address, stats): """Act on statistics upload.""" - sdata = lxml.etree.XML(stats) + sdata = lxml.etree.XML(stats.encode('utf-8')) client = self.metadata.resolve_client(address) self.process_statistics(client, sdata) return "" diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3b331b300..3a36baf8e 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -693,6 +693,9 @@ class Specificity: self.prio = prio self.delta = delta + def __lt__(self, other): + return self.__cmp__(other) < 0 + def matches(self, metadata): return self.all or \ self.hostname == metadata.hostname or \ diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c08e8c4b6..4a0fd2dfe 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import binascii import logging import lxml +import operator import os import os.path import re @@ -25,7 +26,7 @@ logger = logging.getLogger('Bcfg2.Plugins.Cfg') def u_str(string, encoding): if sys.hexversion >= 0x03000000: - return str(string, encoding) + return string.encode(encoding) else: return unicode(string, encoding) @@ -105,7 +106,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] - matching.sort(self.sort_by_specific) + matching.sort(key=operator.attrgetter('specific')) non_delta = [matching.index(m) for m in matching if not m.specific.delta] if not non_delta: diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ca6e43851..6570f2912 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (address[0])) return False # populate the session cache - if user != 'root': + if user.decode('utf-8') != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index ea2e79ccc..3599f2af1 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -1,4 +1,5 @@ import lxml.etree +import operator import re import Bcfg2.Server.Plugin @@ -27,7 +28,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret = [] build = dict() candidates = self.get_matching(metadata) - candidates.sort(lambda x, y: cmp(x.specific, y.specific)) + candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: rem = specific_probe_matcher.match(entry.name) if not rem: @@ -90,7 +91,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data) + datafile.write(data.decode('utf-8')) def load_data(self): try: diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cf0998aaa..4a33c0cb0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", - "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"] + "ssh_host_rsa_key.pub.H_%s", + "ssh_host_key.pub.H_%s"] hostkeys = ["ssh_host_dsa_key.H_%s", - "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"] - keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key', - 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub', - 'ssh_host_key.pub'] + "ssh_host_rsa_key.H_%s", + "ssh_host_key.H_%s"] + keypatterns = ["ssh_host_dsa_key", + "ssh_host_rsa_key", + "ssh_host_key", + "ssh_host_dsa_key.pub", + "ssh_host_rsa_key.pub", + "ssh_host_key.pub"] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([str(value.data) for key, value in \ + self.__skn = "\n".join([value.data.decode() for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() @@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname) continue self.__skn += "%s %s" % (','.join(names[hostname]), - self.entries[pubkey].data) + self.entries[pubkey].data.decode()) return self.__skn def set_skn(self, value): @@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, hostkeys.sort() for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - self.entries[hostkey].data) + self.entries[hostkey].data.decode()) permdata = {'owner': 'root', 'group': 'root', 'type': 'file', -- cgit v1.2.3-1-g7c22 From a1bc75c200fb9237ecc0dd5ccd5b8bf94214cf4b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 09:23:19 -0500 Subject: PY3K: Add compatibility comments Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 1 + src/lib/Server/Plugins/Snapshots.py | 1 + src/lib/Server/Reports/reports/templatetags/syntax_coloring.py | 1 + src/lib/Server/Snapshots/model.py | 1 + 4 files changed, 4 insertions(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 4a0fd2dfe..c93b76488 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -24,6 +24,7 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') +# py3k compatibility def u_str(string, encoding): if sys.hexversion >= 0x03000000: return string.encode(encoding) diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index 8b6bad574..aeb3b9f74 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -28,6 +28,7 @@ datafields = { } +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py index 291528e2e..2e30125f9 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py @@ -15,6 +15,7 @@ try: except: colorize = False +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 2aa35f1ec..f30c38a05 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string -- cgit v1.2.3-1-g7c22 From ac578f15ca785d9b1fcd0d42bfa3ffd017985e78 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 11:32:29 -0400 Subject: make Bcfg2 automatically handle JSON, XML, and YAML output from probes --- src/lib/Server/Plugins/Probes.py | 93 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 3599f2af1..4a7171297 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -2,12 +2,96 @@ import lxml.etree import operator import re +try: + import json + has_json = True +except ImportError: + has_json = False + +try: + import syck + has_syck = True +except ImportError: + has_syck = False + +try: + import yaml + has_yaml = True +except ImportError: + has_yaml = False + import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P\S+)(.(?P[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P\S+)") +class ProbeData (object): + """ a ProbeData object emulates a str object, but also has .xdata + and .json properties to provide convenient ways to use ProbeData + objects as XML or JSON data """ + def __init__(self, data): + self.data = data + self._xdata = None + self._json = None + self._yaml = None + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __getattr__(self, name): + """ make ProbeData act like a str object """ + return getattr(self.data, name) + + def __complex__(self): + return complex(self.data) + + def __int__(self): + return int(self.data) + + def __long__(self): + return long(self.data) + + def __float__(self): + return float(self.data) + + @property + def xdata(self): + if self._xdata is None: + try: + self._xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + pass + return self._xdata + + @property + def json(self): + if self._json is None and has_json: + try: + self._json = json.loads(self.data) + except ValueError: + pass + return self._json + + @property + def yaml(self): + if self._yaml is None: + if has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml.YAMLError: + pass + elif has_syck: + try: + self._yaml = syck.load(self.data) + except syck.error: + pass + return self._yaml + + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") @@ -106,7 +190,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') + self.probedata[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get('value')) elif (pdata.tag == 'Group'): self.cgroups[client.get('name')].append(pdata.get('name')) @@ -142,11 +227,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, if newgroup not in self.cgroups[client.hostname]: self.cgroups[client.hostname].append(newgroup) dlines.remove(line) - dtext = "\n".join(dlines) + dobj = ProbeData("\n".join(dlines)) try: - self.probedata[client.hostname].update({data.get('name'): dtext}) + self.probedata[client.hostname].update({data.get('name'): dobj}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): dtext} + self.probedata[client.hostname] = {data.get('name'): dobj} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) -- cgit v1.2.3-1-g7c22 From 5f93d780fc2dcd6ba35179acd61c05754d1e4fbc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 12:21:17 -0400 Subject: made StructFile.Match() work with Group/Client tags inside other tags --- src/lib/Server/Plugin.py | 80 +++++++++++++++++------------------- src/lib/Server/Plugins/Properties.py | 45 ++++---------------- 2 files changed, 46 insertions(+), 79 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3a36baf8e..aff9af12b 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -457,51 +457,45 @@ class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" def __init__(self, name): XMLFileBacked.__init__(self, name) - self.fragments = {} - - def Index(self): - """Build internal data structures.""" - try: - xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - logger.error("Failed to parse file %s" % self.name) - return - self.fragments = {} - work = {lambda x: True: xdata.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + self.matches = {} + + def _match(self, item, metadata): + """ recursive helper for Match() """ + if isinstance(item, lxml.etree._Comment): + return [] + elif item.tag == 'Group': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') not in metadata.groups) or + item.get('name') in metadata.groups): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + elif item.tag == 'Client': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') != metadata.hostname) or + item.get('name') == metadata.hostname): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + else: + rv = copy.deepcopy(item) + for child in rv.iterchildren(): + rv.remove(child) + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return [rv] + def Match(self, metadata): """Return matching fragments of independent.""" - matching = [frag for (pred, frag) in list(self.fragments.items()) - if pred(metadata)] - if matching: - return reduce(lambda x, y: x + y, matching) - logger.error("File %s got null match" % (self.name)) - return [] + rv = [] + if metadata.hostname not in self.matches: + for child in self.entries(): + rv.extend(self._match(child, metadata)) + if not rv: + logger.error("File %s got null match" % (self.name)) + return rv class INode: diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index dea797a10..95565f2e4 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -7,42 +7,15 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" def Index(self): - """Build internal data structures.""" - if type(self.data) is not lxml.etree._Element: - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % - self.name) - - self.fragments = {} - work = {lambda x: True: self.data.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + """Build local data structures.""" + try: + xdata = XML(self.data) + except XMLSyntaxError: + self.label = None + self.entries = [] + return + self.label = xdata.attrib[self.__identifier__] + self.entries = xdata.getchildren() class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): -- cgit v1.2.3-1-g7c22 From bc1a6b8d0a46e37a108a752a7b6f54e637ff804d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 13:22:19 -0400 Subject: bug fixes --- src/lib/Server/Plugin.py | 20 ++++++++++++-------- src/lib/Server/Plugins/Properties.py | 12 +----------- 2 files changed, 13 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index aff9af12b..e535802c7 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked): def Index(self): """Build local data structures.""" try: - xdata = XML(self.data) + self.xdata = XML(self.data) except XMLSyntaxError: logger.error("Failed to parse %s" % (self.name)) return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() + self.entries = self.xdata.getchildren() + if self.__identifier__ is not None: + self.label = self.xdata.attrib[self.__identifier__] def __iter__(self): return iter(self.entries) @@ -455,6 +456,8 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" + __identifier__ = None + def __init__(self, name): XMLFileBacked.__init__(self, name) self.matches = {} @@ -489,13 +492,14 @@ class StructFile(XMLFileBacked): def Match(self, metadata): """Return matching fragments of independent.""" - rv = [] if metadata.hostname not in self.matches: - for child in self.entries(): + rv = [] + for child in self.entries: rv.extend(self._match(child, metadata)) - if not rv: - logger.error("File %s got null match" % (self.name)) - return rv + if not rv: + logger.error("File %s got null match" % (self.name)) + self.matches[metadata.hostname] = rv + return self.matches[metadata.hostname] class INode: diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index 95565f2e4..54c5def57 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,17 +6,7 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - def Index(self): - """Build local data structures.""" - try: - xdata = XML(self.data) - except XMLSyntaxError: - self.label = None - self.entries = [] - return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() - + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile -- cgit v1.2.3-1-g7c22 From df2fbddbe59b21f41eeddc84d75d852a477454b0 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 13:20:20 -0500 Subject: SSLServer: Fix syntax error for python 2 Signed-off-by: Sol Jerome --- src/lib/SSLServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index bb0f61198..fa7d4e145 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -201,7 +201,7 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): try: username, password = auth_content.split(":") except TypeError: - username, pw = auth_content.split(b":") + username, pw = auth_content.split(bytes(":", encoding='utf-8')) password = pw.decode('utf-8') except ValueError: username = auth_content -- cgit v1.2.3-1-g7c22 From 0366ab745df68660f5966c7fb901766bc2c2d4d4 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 13:38:04 -0500 Subject: SSLServer: Fix another syntax error for python 2 Signed-off-by: Sol Jerome --- src/lib/SSLServer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index fa7d4e145..d5d2f2b3a 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -195,7 +195,11 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): self.logger.error("No authentication data presented") return False auth_type, auth_content = header.split() - auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) + try: + # py3k compatibility + auth_content = base64.standard_b64decode(auth_content) + except TypeError: + auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) try: # py3k compatibility try: -- cgit v1.2.3-1-g7c22 From dfee52f047c4d50b9b7120b4310c11397d1a0620 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 15:04:04 -0400 Subject: cleaned up some ProbeData stuff, make null probe storage more consistent --- src/lib/Server/Plugins/Probes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 4a7171297..b07c4dfd3 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -165,7 +165,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, cx = lxml.etree.SubElement(top, 'Client', name=client) for probe in sorted(probed): lxml.etree.SubElement(cx, 'Probe', name=probe, - value=self.probedata[client][probe]) + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) data = lxml.etree.tostring(top, encoding='UTF-8', @@ -214,9 +214,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.logger.error("Got null response to probe %s from %s" % \ (data.get('name'), client.hostname)) try: - self.probedata[client.hostname].update({data.get('name'): ''}) + self.probedata[client.hostname].update({data.get('name'): + ProbeData('')}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): ''} + self.probedata[client.hostname] = \ + {data.get('name'): ProbeData('')} return dlines = data.text.split('\n') self.logger.debug("%s:probe:%s:%s" % (client.hostname, -- cgit v1.2.3-1-g7c22 From 7be5239be45d6ee52d0b80d2a5993d3bdb6b933b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 14:15:52 -0500 Subject: Client Tools: Add PY3K compatibility Signed-off-by: Sol Jerome --- src/lib/Client/Tools/APT.py | 3 ++- src/lib/Client/Tools/POSIX.py | 28 ++++++++++++++++++++-------- src/lib/Client/Tools/Pacman.py | 3 ++- src/lib/Client/Tools/RPMng.py | 3 ++- src/lib/Client/Tools/YUM24.py | 6 ++++-- src/lib/Client/Tools/YUMng.py | 28 +++++++++++++++++++--------- 6 files changed, 49 insertions(+), 22 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Client/Tools/APT.py index d7be44dc0..2b8cc3304 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Client/Tools/APT.py @@ -73,7 +73,8 @@ class APT(Bcfg2.Client.Tools.Tool): self.cmd.run("%s clean" % APTGET) try: self.pkg_cache = apt.cache.Cache() - except SystemError, e: + except SystemError: + e = sys.exc_info()[1] self.logger.info("Failed to initialize APT cache: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError self.pkg_cache.update() diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 64821f621..a079571e5 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -16,6 +16,10 @@ import shutil import stat import sys import time +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str + import Bcfg2.Client.Tools import Bcfg2.Options from Bcfg2.Client import XML @@ -145,7 +149,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: content = open(entry.get('name')).read() entry.set('current_bfile', binascii.b2a_base64(content)) - except IOError, error: + except IOError: + error = sys.exc_info()[1] self.logger.error("Failed to read %s: %s" % (error.filename, error.strerror)) @@ -457,12 +462,14 @@ class POSIX(Bcfg2.Client.Tools.Tool): if type(tempdata) == unicode: try: tempdata = tempdata.encode(self.setup['encoding']) - except UnicodeEncodeError, e: + except UnicodeEncodeError: + e = sys.exc_info()[1] self.logger.error("Error encoding file %s:\n %s" % \ (entry.get('name'), e)) try: content = open(entry.get('name')).read() - except IOError, error: + except IOError: + error = sys.exc_info()[1] if error.strerror == "No such file or directory": # print diff for files that don't exist (yet) content = '' @@ -586,7 +593,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): datetime.isoformat(datetime.now()))) self.logger.info("Backup of %s saved to %s" % (entry.get('name'), self.ppath)) - except IOError, e: + except IOError: + e = sys.exc_info()[1] self.logger.error("Failed to create backup file for %s" % \ (entry.get('name'))) self.logger.error(e) @@ -622,7 +630,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): % (entry.get('name'))) return False return True - except (OSError, IOError), err: + except (OSError, IOError): + err = sys.exc_info()[1] if err.errno == errno.EACCES: self.logger.info("Failed to open %s for writing" % (entry.get('name'))) else: @@ -702,7 +711,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): return False try: shutil.rmtree(ename) - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) else: @@ -710,14 +720,16 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: os.rmdir(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False try: os.remove(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Client/Tools/Pacman.py index 082897934..c8c05061c 100644 --- a/src/lib/Client/Tools/Pacman.py +++ b/src/lib/Client/Tools/Pacman.py @@ -78,5 +78,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): try: self.logger.debug("Running : %s -S %s" % (self.pkgtool, pkgline)) self.cmd.run("%s -S %s" % (self.pkgtool, pkgline)) - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("Error occurred during installation: %s" % e) diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Client/Tools/RPMng.py index a1e14b3a6..5376118c2 100644 --- a/src/lib/Client/Tools/RPMng.py +++ b/src/lib/Client/Tools/RPMng.py @@ -2,11 +2,12 @@ __revision__ = '$Revision$' -import ConfigParser import os.path import rpm import rpmtools import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Client/Tools/YUM24.py index 5387fdd6a..66768fb34 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Client/Tools/YUM24.py @@ -1,13 +1,14 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision: $' -import ConfigParser import copy import os.path import sys import yum import Bcfg2.Client.XML import Bcfg2.Client.Tools.RPMng +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -151,7 +152,8 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): try: pkgDict = dict([(i.name, i) for i in \ self.yb.returnPackagesByDep(entry.get('name'))]) - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error('Yum Error Depsolving for %s: %s' % \ (entry.get('name'), str(e))) pkgDict = {} diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 0435ca0d7..6aab9817c 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -1,9 +1,9 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision$' -import ConfigParser import copy import os.path +import sys import yum import yum.packages import yum.rpmtrans @@ -13,6 +13,8 @@ import yum.misc import rpmUtils.arch import Bcfg2.Client.XML import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -167,10 +169,12 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): self.yb.doConfigSetup() self.yb.doTsSetup() self.yb.doRpmDBSetup() - except yum.Errors.RepoError, e: + except yum.Errors.RepoError: + e = sys.exc_info()[1] self.logger.error("YUMng Repository error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("YUMng error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError @@ -505,7 +509,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): try: vResult = self._verifyHelper(_POs[0]) - except Exception, e: + except Exception: + e = sys.exc_info()[1] # Unknown Yum exception self.logger.warning(" Verify Exception: %s" % str(e)) package_fail = True @@ -680,7 +685,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): # Run the Yum Transaction try: rescode, restring = self.yb.buildTransaction() - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) cleanup() return @@ -695,7 +701,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): self.yb.processTransaction(callback=yDisplay, rpmDisplay=rDisplay) self.logger.info("Single Pass for Install Succeeded") - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) cleanup() return @@ -822,7 +829,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.install(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error installing some packages: %s" % yume) if len(upgrade_pkgs) > 0: @@ -832,7 +840,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.update(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" % yume) if len(reinstall_pkgs) > 0: @@ -841,7 +850,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.reinstall(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" \ % yume) -- cgit v1.2.3-1-g7c22 From 707a17cf8bcf0e91a2c74f671f0d3fffff3c294e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 15 Jun 2011 15:26:53 -0400 Subject: fixed bugs with handling of negate in new StructFile.Match() --- src/lib/Server/Plugin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index e535802c7..740de247a 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -470,7 +470,8 @@ class StructFile(XMLFileBacked): rv = [] if ((item.get('negate', 'false').lower() == 'true' and item.get('name') not in metadata.groups) or - item.get('name') in metadata.groups): + (item.get('negate', 'false').lower() == 'false' and + item.get('name') in metadata.groups)): for child in item.iterchildren(): rv.extend(self._match(child, metadata)) return rv @@ -478,7 +479,8 @@ class StructFile(XMLFileBacked): rv = [] if ((item.get('negate', 'false').lower() == 'true' and item.get('name') != metadata.hostname) or - item.get('name') == metadata.hostname): + (item.get('negate', 'false').lower() == 'false' and + item.get('name') == metadata.hostname)): for child in item.iterchildren(): rv.extend(self._match(child, metadata)) return rv -- cgit v1.2.3-1-g7c22 From 644d34552c4e2bc6a1da1daef5e3ac734b06b205 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 14:49:52 -0500 Subject: Hostbase: More PY3K fixes Signed-off-by: Sol Jerome --- src/lib/Server/Hostbase/backends.py | 6 ++++-- src/lib/Server/Hostbase/ldapauth.py | 12 ++++++++---- src/lib/Server/Hostbase/settings.py | 5 +++-- 3 files changed, 15 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py index aa822409c..bf774f695 100644 --- a/src/lib/Server/Hostbase/backends.py +++ b/src/lib/Server/Hostbase/backends.py @@ -57,12 +57,14 @@ class NISBackend(object): return user - except NISAUTHError, e: + except NISAUTHError: + e = sys.exc_info()[1] return None def get_user(self, user_id): try: return User.objects.get(pk=user_id) - except User.DoesNotExist, e: + except User.DoesNotExist: + e = sys.exc_info()[1] return None diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py index 21b462c86..f3db26f67 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Server/Hostbase/ldapauth.py @@ -69,7 +69,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -86,7 +87,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -108,7 +110,8 @@ class ldapauth(object): raw_obj = result_data[0][1] distinguishedName = raw_obj['distinguishedName'] return ('success', distinguishedName[0],) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -134,7 +137,8 @@ class ldapauth(object): self.is_superuser = False return - except KeyError, e: + except KeyError: + e = sys.exc_info()[1] raise LDAPAUTHError("Portions of the LDAP User profile not present") def member_of(self): diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py index c44c7bf16..4e641f13c 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Server/Hostbase/settings.py @@ -1,9 +1,10 @@ -from ConfigParser import ConfigParser, NoSectionError, NoOptionError import os.path +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) -c = ConfigParser() +c = ConfigParser.ConfigParser() #This needs to be configurable one day somehow c.read(['./bcfg2.conf']) -- cgit v1.2.3-1-g7c22 From 5e4ad67b39a175759251d9e9bfc4b93c64d5dd36 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Jun 2011 18:44:12 -0500 Subject: SSLServer: Fix errors reported by emias on IRC Signed-off-by: Sol Jerome --- src/lib/SSLServer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index d5d2f2b3a..21bf48d3e 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -47,7 +47,8 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher): params = (address, ) + params response = self.instance._dispatch(method, params, self.funcs) # py3k compatibility - if isinstance(response, bool) or isinstance(response, str): + if isinstance(response, bool) or isinstance(response, str) \ + or isinstance(response, list): response = (response, ) else: response = (response.decode('utf-8'), ) @@ -252,7 +253,8 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): data = ''.join(L) response = self.server._marshaled_dispatch(self.client_address, data) - response = response.encode('utf-8') + if sys.hexversion >= 0x03000000: + response = response.encode('utf-8') except: try: self.send_response(500) -- cgit v1.2.3-1-g7c22 From edb09c53992ef3aca8bfe440df66717290b1771f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 16 Jun 2011 07:33:18 -0400 Subject: change error to debug --- src/lib/Server/Plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 740de247a..30c4f9686 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -498,8 +498,7 @@ class StructFile(XMLFileBacked): rv = [] for child in self.entries: rv.extend(self._match(child, metadata)) - if not rv: - logger.error("File %s got null match" % (self.name)) + logger.debug("File %s got %d match(es)" % (self.name, len(rv))) self.matches[metadata.hostname] = rv return self.matches[metadata.hostname] -- cgit v1.2.3-1-g7c22 From 18368164103d047cf742f0679b4a8cb9ab5cb9c2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 16 Jun 2011 12:41:30 -0400 Subject: updated Lint.Comments for new properties XML handling --- src/lib/Server/Lint/Comments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8e86cc564..09443d4c0 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): props = self.core.plugins['Properties'] for propfile, pdata in props.store.entries.items(): if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.data, 'properties') + self.check_xml(pdata.name, pdata.xdata, 'properties') def check_metadata(self): """ check metadata files for required headers """ -- cgit v1.2.3-1-g7c22 From 3ec2fc27deefc647127c9e729221826a90fd7a96 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 17 Jun 2011 08:03:26 -0400 Subject: Improved handling of JSON data from probes --- src/lib/Server/Plugins/Probes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index b07c4dfd3..ec0f294dd 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -6,19 +6,22 @@ try: import json has_json = True except ImportError: - has_json = False + try: + import simplejson as json + has_json = True + except ImportError: + has_json = False try: import syck has_syck = True except ImportError: has_syck = False - -try: - import yaml - has_yaml = True -except ImportError: - has_yaml = False + try: + import yaml + has_yaml = True + except ImportError: + has_yaml = False import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 76366b0bf116b0d320ec4a7168de8f62cc50ec98 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 18 Jun 2011 19:41:19 -0500 Subject: POSIX: Add recursive permissions (Ticket #871) This allows for a recursive='true' attribute such that the owner/group can be set recursively for a directory when using Path type='permissions'. Signed-off-by: Sol Jerome --- src/lib/Client/Tools/POSIX.py | 59 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index a079571e5..faec2e251 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -736,6 +736,47 @@ class POSIX(Bcfg2.Client.Tools.Tool): def Verifypermissions(self, entry, _): """Verify Path type='permissions' entry""" + if entry.get('perms') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % (entry.get('name'))) + return False + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid: + # owner mismatch for path + entry.set('current_owner', str(pstat.st_uid)) + self.logger.debug("%s %s ownership wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Owner for path %s is incorrect. " + "Current owner is %s but should be %s\n" % \ + (path, pstat.st_uid, entry.get('owner'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False + if group != pstat.st_gid: + # group mismatch for path + entry.set('current_group', str(pstat.st_gid)) + self.logger.debug("%s %s group wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Group for path %s is incorrect. " + "Current group is %s but should be %s\n" % \ + (path, pstat.st_gid, entry.get('group'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False return self.Verifydirectory(entry, _) def Installpermissions(self, entry): @@ -746,9 +787,23 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.error('Entry %s not completely specified. ' 'Try running bcfg2-lint.' % (entry.get('name'))) return False + plist = [entry.get('name')] + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid or group != pstat.st_gid: + # owner mismatch for path + plist.append(path) try: - os.chown(entry.get('name'), normUid(entry), normGid(entry)) - os.chmod(entry.get('name'), calcPerms(S_IFDIR, entry.get('perms'))) + for p in plist: + os.chown(p, normUid(entry), normGid(entry)) + os.chmod(p, calcPerms(S_IFDIR, entry.get('perms'))) return True except (OSError, KeyError): self.logger.error('Permission fixup failed for %s' % \ -- cgit v1.2.3-1-g7c22 From 4f762722925113c56582a10dd4abead6cd84facc Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 18 Jun 2011 22:41:29 -0500 Subject: Version bump to 1.2.0pre3 Signed-off-by: Sol Jerome --- src/lib/Server/Reports/reports/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html index 6ef4c9aff..ec9a17468 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Server/Reports/reports/templates/base.html @@ -87,7 +87,7 @@
-- cgit v1.2.3-1-g7c22 From 19586bc42aa90543cf27e33ec32bd7df222138e8 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 20 Jun 2011 12:07:16 -0500 Subject: Cfg: Add support for perms='inherit' (Ticket #642) This feature allows you to use the on-disk permissions of the file in the Cfg repository rather than specifying them using the traditional means in info.xml. Note that this only works for the octal permissions of the file on disk since the owner/group may not exist on the destination machine. Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c93b76488..23ba0a745 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -8,6 +8,7 @@ import operator import os import os.path import re +import stat import sys import tempfile @@ -97,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) self.specific = CfgMatcher(path.split('/')[-1]) + path = path def sort_by_specific(self, one, other): return cmp(one.specific, other.specific) @@ -121,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): self.bind_info_to_entry(entry, metadata) used = self.get_pertinent_entries(metadata) basefile = used.pop(0) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = "%s/%s" % (self.path, entry.get('name')) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): -- cgit v1.2.3-1-g7c22 From 0957c3c73880448ca461aad90cebe7cf85717ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Tue, 21 Jun 2011 12:38:40 +0200 Subject: Correct the description of bcfg2(1)'s "-z" option The "-z" option is for use with Independent collections instead of Bundles. --- src/lib/Options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Options.py b/src/lib/Options.py index 0d978c519..619b16787 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -323,7 +323,7 @@ CLIENT_BUNDLE = Option('Only configure the given bundle(s)', default=[], cmd='-b', odesc='', cook=colon_split) CLIENT_BUNDLEQUICK = Option('only verify/configure the given bundle(s)', default=False, cmd='-Q') -CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, +CLIENT_INDEP = Option('Only configure independent entries, ignore bundles', default=False, cmd='-z') CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) -- cgit v1.2.3-1-g7c22 From b3bad727bc0968cd8b7fb29573e74d551a13c74d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 21 Jun 2011 07:42:23 -0500 Subject: Frame: Fix 'important' interactive installation (#1015) Signed-off-by: Sol Jerome --- src/lib/Client/Frame.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index e33e1a7df..d8e308ee0 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -200,15 +200,18 @@ class Frame: tl = [t for t in self.tools if t.handlesEntry(cfile) \ and t.canVerify(cfile)] if tl: - if not tl[0].VerifyPath(cfile, []): - if self.setup['interactive'] and not \ - promptFilter("Install %s: %s? (y/N):", [cfile]): - continue - try: - self.states[cfile] = tl[0].InstallPath(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) + if self.setup['interactive'] and not \ + promptFilter("Install %s: %s? (y/N):", [cfile]): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tl[0].InstallPath(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tl[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) def Inventory(self): """ -- cgit v1.2.3-1-g7c22 From 45f7e202f6b7860214d709d0d470f2c8938c2993 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 21 Jun 2011 11:27:19 -0500 Subject: bcfg2: Catch tracebacks (Tickets #1016 and #1017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This also moves the exception handling added in #1012 to the bcfg2 code for consistency (as per Holger Weiß's suggestion). Signed-off-by: Sol Jerome --- src/lib/Proxy.py | 11 ++++------- src/sbin/bcfg2 | 8 ++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index e4a0f6a3d..4cb0bbe80 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -47,6 +47,9 @@ __all__ = ["ComponentProxy", class CertificateError(Exception): def __init__(self, commonName): self.commonName = commonName + def __str__(self): + return ("Got unallowed commonName %s from server" + % self.commonName) class RetryMethod(xmlrpclib._Method): @@ -193,13 +196,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): ca_certs=self.ca, suppress_ragged_eofs=True, keyfile=self.key, certfile=self.cert, ssl_version=ssl_protocol_ver) - try: - self.sock.connect((self.host, self.port)) - except socket.gaierror: - e = sys.exc_info()[1] - self.logger.error("Unable to connect to %s:%s\n%s" % - (self.host, self.port, e.strerror)) - sys.exit(1) + self.sock.connect((self.host, self.port)) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 534ab8238..5ddfd8791 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -7,6 +7,7 @@ import fcntl import logging import os import signal +import socket import stat import sys import tempfile @@ -199,6 +200,13 @@ class Client: self.logger.error("Failed to download probes from bcfg2") self.logger.error(flt.faultString) raise SystemExit(1) + except (Bcfg2.Proxy.CertificateError, + socket.gaierror, + socket.error): + e = sys.exc_info()[1] + self.logger.error("Failed to download probes from bcfg2: %s" + % e) + raise SystemExit(1) times['probe_download'] = time.time() -- cgit v1.2.3-1-g7c22 From 772382d68d6d9014f7f590b3dbd68a87dc918e32 Mon Sep 17 00:00:00 2001 From: Kristian Kostecky Date: Tue, 21 Jun 2011 16:36:35 -0400 Subject: Added auth attrib to list of valid attribs that the Client object can add/update --- src/lib/Server/Admin/Client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py index 3af25b15a..81fc4a1b1 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Server/Admin/Client.py @@ -27,7 +27,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val @@ -41,7 +42,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val -- cgit v1.2.3-1-g7c22 From 29e43063c2e8eb58e916da2dfbb375b4c64d3919 Mon Sep 17 00:00:00 2001 From: Jack Neely Date: Wed, 22 Jun 2011 13:41:40 -0400 Subject: Packages are uniquely identified by both name and arch So a package is not "installed" if a package of the same name but different arch is. This will enable YUMng to handle multilib package installs. --- src/lib/Client/Tools/YUMng.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 6aab9817c..f67859ee5 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -451,8 +451,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): verify_flags = inst.get('verify_flags', self.verifyFlags) verify_flags = verify_flags.lower().replace(' ', ',').split(',') - if len(POs) == 0: - # Package not installed + if len([ p in POs if p.arch = nevra['arch'] ]) == 0: + # Package (name, arch) not installed self.logger.debug(" %s is not installed" % nevraString(nevra)) stat['installed'] = False package_fail = True -- cgit v1.2.3-1-g7c22 From 837142c765b99035821cc08f171b231eba55639e Mon Sep 17 00:00:00 2001 From: Jack Neely Date: Wed, 22 Jun 2011 14:03:09 -0400 Subject: YUMng: correct syntax in list comp --- src/lib/Client/Tools/YUMng.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index f67859ee5..08cfd3289 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -451,7 +451,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): verify_flags = inst.get('verify_flags', self.verifyFlags) verify_flags = verify_flags.lower().replace(' ', ',').split(',') - if len([ p in POs if p.arch = nevra['arch'] ]) == 0: + if len([ p for p in POs if p.arch == nevra['arch'] ]) == 0: # Package (name, arch) not installed self.logger.debug(" %s is not installed" % nevraString(nevra)) stat['installed'] = False -- cgit v1.2.3-1-g7c22 From b4c708c2f9ea913d28c2c8bc75c65c525b00c7ed Mon Sep 17 00:00:00 2001 From: Jack Neely Date: Wed, 22 Jun 2011 14:10:31 -0400 Subject: YUMng: arch may not be specified If the arch is specified use it to select the package in VerifyPackage() otherwise just work with the default arch. --- src/lib/Client/Tools/YUMng.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 08cfd3289..24605ca44 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -451,7 +451,12 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): verify_flags = inst.get('verify_flags', self.verifyFlags) verify_flags = verify_flags.lower().replace(' ', ',').split(',') - if len([ p for p in POs if p.arch == nevra['arch'] ]) == 0: + if 'arch' in nevra: + # If arch is specified use it to select the package + _POs = [ p for p in POs if p.arch == nevra['arch'] ] + else: + _POs = POs + if len(_POs) == 0: # Package (name, arch) not installed self.logger.debug(" %s is not installed" % nevraString(nevra)) stat['installed'] = False -- cgit v1.2.3-1-g7c22