From 8938b4025400d150db4b6fb32181cd36c294e07c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 4 May 2011 13:35:38 -0500 Subject: SSHbase: PY3K string join method fix Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/SSHbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index ce08235eb..a8bf596de 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -73,7 +73,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([value.data for key, value in \ + self.__skn = "\n".join([str(value.data) for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() -- cgit v1.2.3-1-g7c22 From 071ef1a1fbe6368b1abb81855e1ab95e316e6911 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 08:35:54 -0500 Subject: TGenshi: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TGenshi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index 83b60c958..bc5e00400 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -3,7 +3,11 @@ __revision__ = '$Revision$' import binascii import logging +import sys import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TGenshi') @@ -76,9 +80,6 @@ class TemplateFile: def bind_entry(self, entry, metadata): """Build literal file information.""" fname = entry.get('realname', entry.get('name')) - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if entry.tag == 'Path': entry.set('type', 'file') try: -- cgit v1.2.3-1-g7c22 From c7f6cb353df45bb3662b78a49815a6b1a54a7f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 13 Apr 2011 15:16:06 +0200 Subject: Cfg: Fix the output encoding of Genshi templates Encode the configuration files generated from Genshi templates according to the encoding setting from Options.py instead of unconditionally using UTF-8. --- src/lib/Server/Plugins/Cfg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 41cf6c9c1..a97cbd550 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -134,9 +134,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): metadata=metadata, path=basefile.name).filter(removecomment) try: - data = stream.render('text', strip_whitespace=False) + data = stream.render('text', encoding=self.encoding, + strip_whitespace=False) except TypeError: - data = stream.render('text') + data = stream.render('text', encoding=self.encoding) if data == '': entry.set('empty', 'true') except Exception: -- cgit v1.2.3-1-g7c22 From 5952ca5ebf0908b29e2fb09091249dc6ce67540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Sun, 1 May 2011 23:50:09 +0200 Subject: Fix bcfg2-reports --badentry and --extraentry The reporting system schema has been changed. This change lead to exceptions such as | AttributeError: 'Entries_interactions' object has no attribute 'name' when running bcfg2-reports with the --badentry or --extraentry options. --- src/sbin/bcfg2-reports | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 20288fc5e..33a291395 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -233,7 +233,7 @@ else: for c_inst in c_list: baditems = c_inst.current_interaction.bad() for item in baditems: - if item.name == badentry[1] and item.kind == badentry[0]: + if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: result.append(c_inst) if c_inst in entrydict: entrydict.get(c_inst).append(badentry[1]) @@ -244,7 +244,7 @@ else: for c_inst in c_list: baditems = c_inst.current_interaction.bad() for item in baditems: - if item.name == badentry[1] and item.kind == badentry[0]: + if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: result.append(c_inst) break elif extraentry != "": @@ -255,7 +255,7 @@ else: for c_inst in c_list: extraitems = c_inst.current_interaction.extra() for item in extraitems: - if item.name == extraentry[1] and item.kind == extraentry[0]: + if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]: result.append(c_inst) if c_inst in entrydict: entrydict.get(c_inst).append(extraentry[1]) @@ -266,7 +266,7 @@ else: for c_inst in c_list: extraitems = c_inst.current_interaction.extra() for item in extraitems: - if item.name == extraentry[1] and item.kind == extraentry[0]: + if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]: result.append(c_inst) break -- cgit v1.2.3-1-g7c22 From 3f3bc1cb0d539b6651605b48300946d0d7ec6560 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Thu, 5 May 2011 23:13:31 -0400 Subject: Change preflight script to clean up man{1,5} files as well as man8 --- osx/preflight | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osx/preflight b/osx/preflight index 169551b35..64555480f 100644 --- a/osx/preflight +++ b/osx/preflight @@ -8,4 +8,6 @@ /bin/rm -Rvf "${3}"{SITELIBDIR}/Bcfg2* /bin/rm -Rvf "${3}"/usr/local/bin/bcfg2* /bin/rm -Rvf "${3}{DATADIR}/share/bcfg2" +/bin/rm -Rvf "${3}{DATADIR}/share/man/man1/bcfg2*" +/bin/rm -Rvf "${3}{DATADIR}/share/man/man5/bcfg2*" /bin/rm -Rvf "${3}{DATADIR}/share/man/man8/bcfg2*" -- cgit v1.2.3-1-g7c22 From af8bf14ee0079649d4583fb0850ec25b74242344 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 12:46:57 -0500 Subject: TCheetah: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TCheetah.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py index 151cc6543..49be88881 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Server/Plugins/TCheetah.py @@ -6,6 +6,9 @@ import logging import sys import traceback import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TCheetah') @@ -53,9 +56,6 @@ class TemplateFile: if entry.tag == 'Path': entry.set('type', 'file') try: - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if type(self.template) == unicode: entry.text = self.template else: -- cgit v1.2.3-1-g7c22 From 417f502abe96ab7d784cbc872aeb65add9a509ba Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:43:43 -0400 Subject: Add missing import. --- src/lib/Server/Admin/Init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index e69412a5e..f450e5303 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -4,6 +4,7 @@ import random import socket import stat import string +import sys import subprocess import Bcfg2.Server.Admin import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 4838c1477108fd61b2f65f79f8e3d31776cd1ba6 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:48:49 -0400 Subject: Pass keypath to create_conf function. --- src/lib/Server/Admin/Init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index f450e5303..fff8bcd1c 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -142,7 +142,7 @@ def create_key(hostname, keypath, certpath, country, state, location): os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 -def create_conf(confpath, confdata): +def create_conf(confpath, confdata, keypath): # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): # py3k compatibility @@ -402,7 +402,7 @@ class Init(Bcfg2.Server.Admin.Mode): self.server_uri) # Create the configuration file and SSL key - create_conf(self.configfile, confdata) + create_conf(self.configfile, confdata, keypath) kpath = keypath + '/bcfg2.key' cpath = keypath + '/bcfg2.crt' create_key(self.shostname, kpath, cpath, self.country, -- cgit v1.2.3-1-g7c22 From 88249702412125c36179f5ca9f65b45aa0dc42a5 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:53:00 -0400 Subject: Add missing import sys. --- src/lib/Server/Plugins/SSHbase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index a8bf596de..cf0998aaa 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -5,6 +5,7 @@ import binascii import os import socket import shutil +import sys import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From e810fa812c23e87fc43908f8f72c4c6d751df625 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:47:46 +0200 Subject: add missing sys import --- src/lib/Server/Plugins/Ldap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index b904dbe02..7d6d0b609 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,4 +1,5 @@ import imp +import sys import time import ldap import Bcfg2.Options -- cgit v1.2.3-1-g7c22 From 31cfa9f3a7b9f4ffd5fa32e22042681015489149 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:54:42 +0200 Subject: log more useful error message if python-ldap is not installed --- src/lib/Server/Plugins/Ldap.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index 7d6d0b609..06ecaed7b 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,10 +1,18 @@ import imp +import logging import sys import time -import ldap import Bcfg2.Options import Bcfg2.Server.Plugin +logger = logging.getLogger('Bcfg2.Plugins.Ldap') + +try: + import ldap +except: + logger.error("Unable to load ldap module. Is python-ldap installed?") + raise ImportError + # time in seconds between retries after failed LDAP connection RETRY_DELAY = 5 # how many times to try reaching the LDAP server if a connection is broken -- cgit v1.2.3-1-g7c22 From 9ec715f1e25f5274d6472536ce18a899358d3e66 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 25 Apr 2011 21:33:46 -0500 Subject: DBStats: Stop duplicating data in reports_reason --- src/lib/Server/Reports/importscript.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 1781e2fac..b6a3c2599 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -133,12 +133,9 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): try: rr = None - if not quick: - try: - rr = Reason.objects.filter(**kargs)[0] - except IndexError: - pass - if not rr: + try: + rr = Reason.objects.filter(**kargs)[0] + except IndexError: rr = Reason(**kargs) rr.save() if vlevel > 0: -- cgit v1.2.3-1-g7c22 From 0326069de94cdc8300a8b917ae7c440f02793d6e Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:32:44 +0200 Subject: add another missing sys import --- src/lib/Server/Plugins/SGenshi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py index 04942c2bd..efd981956 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Server/Plugins/SGenshi.py @@ -5,6 +5,7 @@ import genshi.input import genshi.template import lxml.etree import logging +import sys import Bcfg2.Server.Plugin import Bcfg2.Server.Plugins.TGenshi -- cgit v1.2.3-1-g7c22 From 84152d61ad34b0c9e1ebae7c4e00e100b41d211f Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 9 May 2011 09:46:47 -0500 Subject: Upstart: Fix typo reported by justintime on IRC Signed-off-by: Sol Jerome --- src/lib/Client/Tools/Upstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Client/Tools/Upstart.py b/src/lib/Client/Tools/Upstart.py index a9d4b166b..41a585c23 100644 --- a/src/lib/Client/Tools/Upstart.py +++ b/src/lib/Client/Tools/Upstart.py @@ -74,7 +74,7 @@ class Upstart(Bcfg2.Client.Tools.SvcTool): if entry.get('mode', 'default') == 'manual': self.logger.info("Service %s mode set to manual. Skipping " "installation." % (entry.get('name'))) - return Fasle + return False if entry.get('status') == 'on': pstatus = self.cmd.run(self.get_svc_command(entry, 'start'))[0] elif entry.get('status') == 'off': -- cgit v1.2.3-1-g7c22 From 92d4cfc6e8e2225b3b8d00578d83397313983c0b Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:53:44 +0200 Subject: add yet another missing sys import --- src/lib/Server/Plugins/Bundler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 1a8e7348b..01ad3c78b 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import copy import lxml.etree import re +import sys import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 382d96a8146a888c6edca7a20ee6cb348b202c4b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 9 May 2011 11:54:07 -0500 Subject: Logger: Use bytes for PY3K Signed-off-by: Sol Jerome --- src/lib/Logger.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/Logger.py b/src/lib/Logger.py index b49a7069f..0a72e038c 100644 --- a/src/lib/Logger.py +++ b/src/lib/Logger.py @@ -12,6 +12,8 @@ import sys import termios # Compatibility import from Bcfg2.Bcfg2Py3k import fprint +if sys.hexversion >= 0x03000000: + str = bytes logging.raiseExceptions = 0 @@ -118,7 +120,7 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): def emit(self, record): """Chunk and deliver records.""" record.name = self.procname - if len(record.msg) > 250: + if str(record.msg) > 250: msgs = [] error = record.exc_info record.exc_info = None -- cgit v1.2.3-1-g7c22 From ffce8cbd14a0ddc2cd266d80d63b36a786c08cb0 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 9 May 2011 15:20:11 -0500 Subject: Logger: Statement always was True in python 2 Signed-off-by: Sol Jerome --- src/lib/Logger.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/lib/Logger.py b/src/lib/Logger.py index 0a72e038c..8920ba747 100644 --- a/src/lib/Logger.py +++ b/src/lib/Logger.py @@ -12,8 +12,6 @@ import sys import termios # Compatibility import from Bcfg2.Bcfg2Py3k import fprint -if sys.hexversion >= 0x03000000: - str = bytes logging.raiseExceptions = 0 @@ -120,19 +118,16 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): def emit(self, record): """Chunk and deliver records.""" record.name = self.procname - if str(record.msg) > 250: - msgs = [] - error = record.exc_info - record.exc_info = None - msgdata = record.msg - while msgdata: - newrec = copy.deepcopy(record) - newrec.msg = msgdata[:250] - msgs.append(newrec) - msgdata = msgdata[250:] - msgs[0].exc_info = error - else: - msgs = [record] + msgs = [] + error = record.exc_info + record.exc_info = None + msgdata = record.msg + while msgdata: + newrec = copy.deepcopy(record) + newrec.msg = msgdata[:250] + msgs.append(newrec) + msgdata = msgdata[250:] + msgs[0].exc_info = error for newrec in msgs: msg = self.log_format_string % (self.encodePriority(self.facility, newrec.levelname.lower()), -- cgit v1.2.3-1-g7c22 From e6383bdb7d19729d340565fb091a8b67831a5835 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 10 May 2011 08:55:53 -0500 Subject: Logger: Fix non-string logging Signed-off-by: Sol Jerome --- src/lib/Logger.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/lib/Logger.py b/src/lib/Logger.py index 8920ba747..9fe81f47e 100644 --- a/src/lib/Logger.py +++ b/src/lib/Logger.py @@ -118,16 +118,19 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): def emit(self, record): """Chunk and deliver records.""" record.name = self.procname - msgs = [] - error = record.exc_info - record.exc_info = None - msgdata = record.msg - while msgdata: - newrec = copy.deepcopy(record) - newrec.msg = msgdata[:250] - msgs.append(newrec) - msgdata = msgdata[250:] - msgs[0].exc_info = error + if isinstance(record.msg, str): + msgs = [] + error = record.exc_info + record.exc_info = None + msgdata = record.msg + while msgdata: + newrec = copy.deepcopy(record) + newrec.msg = msgdata[:250] + msgs.append(newrec) + msgdata = msgdata[250:] + msgs[0].exc_info = error + else: + msgs = [record] for newrec in msgs: msg = self.log_format_string % (self.encodePriority(self.facility, newrec.levelname.lower()), -- cgit v1.2.3-1-g7c22 From 30b1b4be5e9ff88b254814cda2b4d8184e80d6db Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 10 May 2011 09:46:40 -0500 Subject: man: Fix typos reported by emias on IRC Signed-off-by: Sol Jerome --- man/bcfg2-admin.8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8 index 0b3829b7e..829d00f03 100644 --- a/man/bcfg2-admin.8 +++ b/man/bcfg2-admin.8 @@ -55,7 +55,7 @@ Build structure entries based on client statistics extra entries. Install configuration information into repo based on client bad entries. .RE -.B report [init|load_stats|purge|scrub|update] +.B reports [init|load_stats|purge|scrub|update] .RS Interact with the dynamic reporting system. .RE @@ -154,7 +154,7 @@ Specify the type of the entry to pull. .RS Specify the name of the entry to pull. .RE -.SH REPORT OPTIONS +.SH REPORTS OPTIONS .PP .B init .RS -- cgit v1.2.3-1-g7c22 From aa8c53f8cf7d2e8bcf63176c45c44b19cbba2bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 11 May 2011 14:51:29 +0200 Subject: bcfg2: Convert specification from Unicode to UTF-8 The client receives the configuration specification as a Unicode string and then hands it over to the XML() function, which expects a UTF-8 encoded string. Therefore, the configuration specification is now converted to UTF-8. Resolves ticket #1009. --- src/sbin/bcfg2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 7f7d8f5c6..996d773ff 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -243,7 +243,7 @@ class Client: raise SystemExit(1) try: - rawconfig = proxy.GetConfig() + rawconfig = proxy.GetConfig().encode('UTF-8') except xmlrpclib.Fault: self.logger.error("Failed to download configuration from Bcfg2") raise SystemExit(2) -- cgit v1.2.3-1-g7c22 From 716168325c7221b33e3100b8a903bc5d075f6cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 11 May 2011 15:33:14 +0200 Subject: Fix a typo in an SSL error message --- src/lib/Proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index 8a1ad683e..ed33316af 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -181,7 +181,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): other_side_required = ssl.CERT_NONE self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") if self.cert and not self.key: - self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") self.cert = None if self.key and not self.cert: self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") @@ -226,7 +226,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): # authentication to the server ctx.load_cert(self.cert, self.key) elif self.cert: - self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") elif self.key: self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") -- cgit v1.2.3-1-g7c22 From fa70c550a90c369ac48862d976e3d4294181be89 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 09:37:50 -0400 Subject: Improved Svn2 error handling -- ClientError may not have a message attribute. --- src/lib/Server/Plugins/Svn2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py index 35f555294..b8a8e6b7e 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Server/Plugins/Svn2.py @@ -75,9 +75,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: @@ -95,9 +95,9 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, except Exception, err: # try to be smart about the error we got back details = None - if "callback_ssl_server_trust_prompt" in err.message: + if "callback_ssl_server_trust_prompt" in str(err): details = "SVN server certificate is not trusted" - elif "callback_get_login" in err.message: + elif "callback_get_login" in str(err): details = "SVN credentials not cached" if details is None: -- cgit v1.2.3-1-g7c22 From 564a0ae955286194094ea5e6eda0afb7f755ca91 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 10:12:45 -0400 Subject: Fixed error messages from info.xml bcfg2-lint check --- src/lib/Server/Lint/InfoXML.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 7725ad748..c88e54e95 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -13,12 +13,13 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if self.HandlesFile(infoxml_fname): if (hasattr(entryset, "infoxml") and entryset.infoxml is not None): - self.check_infoxml(entryset.infoxml.pnode.data) + self.check_infoxml(infoxml_fname, + entryset.infoxml.pnode.data) else: self.LintError("no-infoxml", "No info.xml found for %s" % filename) - def check_infoxml(self, xdata): + def check_infoxml(self, fname, xdata): for info in xdata.getroottree().findall("//Info"): required = [] if "required_attrs" in self.config: @@ -28,8 +29,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if missing: self.LintError("required-infoxml-attrs-missing", "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), infoxml_fname, - self.RenderXML(info))) + (",".join(missing), fname, self.RenderXML(info))) if ((Bcfg2.Options.MDATA_PARANOID.value and info.get("paranoid") is not None and @@ -39,5 +39,5 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): info.get("paranoid").lower() != "true"))): self.LintError("paranoid-false", "Paranoid must be true in %s:%s" % - (infoxml_fname, self.RenderXML(info))) + (fname, self.RenderXML(info))) -- cgit v1.2.3-1-g7c22 From 64a49246688d0e8cb1481497ebbceac104220eda Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 11 May 2011 09:19:06 -0500 Subject: setup: Don't include py3 libs in py2 setups Including the python3 libraries when running distutils on python2 leads to syntax errors. Since the libraries are only imported conditionally, it is safe to exclude these libraries completely on setups where they are not needed. Signed-off-by: Sol Jerome --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 52ec93d10..18590cc34 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,9 @@ from distutils.core import setup from distutils.core import Command from fnmatch import fnmatch from glob import glob +import os import os.path +import sys class BuildDTDDoc (Command): """Build DTD documentation""" @@ -113,6 +115,10 @@ try: except ImportError: pass +py3lib = 'src/lib/Bcfg2Py3Incompat.py' +if sys.hexversion < 0x03000000 and os.path.exists(py3lib): + os.remove(py3lib) + setup(cmdclass=cmdclass, name="Bcfg2", version="1.2.0pre2", @@ -133,7 +139,7 @@ setup(cmdclass=cmdclass, "Bcfg2.Server.Reports.reports.templatetags", "Bcfg2.Server.Snapshots", ], - package_dir = {'Bcfg2':'src/lib'}, + package_dir = {'Bcfg2': 'src/lib'}, package_data = {'Bcfg2.Server.Reports.reports':['fixtures/*.xml', 'templates/*.html', 'templates/*/*.html', 'templates/*/*.inc' ] }, -- cgit v1.2.3-1-g7c22 From 34e5e12fd58ec4b0015abdf1a04a4f684c9194ba Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 11 May 2011 10:30:33 -0400 Subject: Added FileProbes plugin. --- doc/server/plugins/probes/index.txt | 57 +++++++++++ schemas/fileprobes.xsd | 34 +++++++ src/lib/Server/Lint/Validate.py | 4 +- src/lib/Server/Plugins/FileProbes.py | 177 +++++++++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 schemas/fileprobes.xsd create mode 100644 src/lib/Server/Plugins/FileProbes.py diff --git a/doc/server/plugins/probes/index.txt b/doc/server/plugins/probes/index.txt index 33ec6f444..b9f698a0f 100644 --- a/doc/server/plugins/probes/index.txt +++ b/doc/server/plugins/probes/index.txt @@ -143,3 +143,60 @@ Other examples :hidden: ohai + +.. _server-plugins-probes-fileprobes: + +FileProbes +========== + +The FileProbes plugin allows you to probe a client for a file, +which is then added to the :ref:`server-plugins-generators-cfg` +specification. If the file changes on the client, FileProbes can +either update it in the specification or allow Cfg to replace it. + +FileProbes will not probe a file if there's already a file in Cfg that +will apply to the client. So if, for instance, you have a generic +file in ``Cfg/etc/foo.conf/foo.conf`` that applies to all hosts, +FileProbes will not retrieve ``/etc/foo.conf`` from the client (unless +``update`` is enabled; see Configuration_ below). + +When a new config file is first probed, an ``info.xml`` file is also +written to enforce the permissions from that client. Subsequent +probes from other clients will not modify or overwrite the data in +``info.xml``. (This ensures that any manual changes you make to +``info.xml`` for that file are not circumvented.) + +Configuration +------------- + +FileProbes is configured in ``FileProbes/config.xml``, which might +look something like: + +.. code-block:: xml + + + + + + + + + + +This will result in ``/etc/foo.conf`` being retrieved from all +clients; if it changes on a client, it will be overwritten by the +version that was retrieved initially. + +Clients in the ``blah-servers`` group will be probed for +``/etc/blah.conf``; if it changes on a client, those changes will be +written into the Bcfg2 specification. If the file is deleted from a +client, it will be rewritten from Bcfg2. + +``bar.example.com`` will be probed for ``/var/lib/bar.gz``, which +contains non-ASCII characters and so needs to use base64 encoding when +transferring the file. + +The paths probed by FileProbes must also be included as Path entries +in your bundles in order to be handled properly by Cfg. Permissions +are handled as usual, with ``info.xml`` files in Cfg. diff --git a/schemas/fileprobes.xsd b/schemas/fileprobes.xsd new file mode 100644 index 000000000..112047836 --- /dev/null +++ b/schemas/fileprobes.xsd @@ -0,0 +1,34 @@ + + + + FileProbes plugin config schema for bcfg2 + Chris St. Pierre + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c87c55ee9..834608378 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -24,7 +24,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "%s/Decisions/*.xml":"%s/decisions.xsd", "%s/Packages/config.xml":"%s/packages.xsd", "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd"} + "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", + "%s/FileProbes/config.xml":"%s/fileprobes.xsd", + } self.filelists = {} self.get_filelists() diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py new file mode 100644 index 000000000..4df5a7f4a --- /dev/null +++ b/src/lib/Server/Plugins/FileProbes.py @@ -0,0 +1,177 @@ +""" This module allows you to probe a client for a file, which is then +added to the specification. On subsequent runs, the file will be +replaced on the client if it is missing; if it has changed on the +client, it can either be updated in the specification or replaced on +the client """ +__revision__ = '$Revision: 1465 $' + +import os +import errno +import binascii +import lxml.etree +import Bcfg2.Options +import Bcfg2.Server.Plugin + +probecode = """#!/usr/bin/env python + +import os +import pwd +import grp +import binascii +import lxml.etree + +path = "%s" + +if not os.path.exists(path): + print "%%s does not exist" %% path + raise SystemExit(1) + +stat = os.stat(path) +data = lxml.etree.Element("ProbedFileData", + name=path, + owner=pwd.getpwuid(stat[4])[0], + group=grp.getgrgid(stat[5])[0], + perms=oct(stat[0] & 07777)) +data.text = binascii.b2a_base64(open(path).read()) +print lxml.etree.tostring(data) +""" + +class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, + Bcfg2.Server.Plugin.StructFile): + """ Config file handler for FileProbes """ + def __init__(self, filename, fam): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename) + + +class FileProbes(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Probing): + """ This module allows you to probe a client for a file, which is then + added to the specification. On subsequent runs, the file will be + replaced on the client if it is missing; if it has changed on the + client, it can either be updated in the specification or replaced on + the client """ + + name = 'FileProbes' + experimental = True + __version__ = '$Id$' + __author__ = 'stpierreca@ornl.gov' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Probing.__init__(self) + self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'), + core.fam) + self.entries = dict() + self.probes = dict() + + def GetProbes(self, metadata): + """Return a set of probes for execution on client.""" + if metadata.hostname not in self.probes: + cfg = self.core.plugins['Cfg'] + self.entries[metadata.hostname] = dict() + self.probes[metadata.hostname] = [] + for entry in self.config.Match(metadata): + path = entry.get("name") + # do not probe for files that are already in Cfg and + # for which update is false; we can't possibly do + # anything with the data we get from such a probe + try: + if (cfg.entries[path].get_pertinent_entries(metadata) and + entry.get('update', 'false').lower() == "false"): + continue + except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError): + pass + self.entries[metadata.hostname][path] = entry + probe = lxml.etree.Element('probe', name=path, + source=self.name, + interpreter="/usr/bin/env python") + probe.text = probecode % path + self.probes[metadata.hostname].append(probe) + self.logger.debug("Adding file probe for %s to %s" % + (path, metadata.hostname)) + return self.probes[metadata.hostname] + + def ReceiveData(self, metadata, datalist): + """Receive data from probe.""" + self.logger.debug("Receiving file probe data from %s" % + metadata.hostname) + + for data in datalist: + if data.text is None: + self.logger.error("Got null response to %s file probe from %s" % + (data.get('name'), metadata.hostname)) + else: + self.logger.debug("%s:fileprobe:%s:%s" % + (metadata.hostname, + data.get("name"), + data.text)) + try: + filedata = lxml.etree.XML(data.text) + self.write_file(filedata, metadata) + except lxml.etree.XMLSyntaxError: + # if we didn't get XML back from the probe, assume + # it's an error message + self.logger.error(data.text) + + def write_file(self, data, metadata): + """Write the probed file data to the bcfg2 specification.""" + filename = data.get("name") + contents = binascii.a2b_base64(data.text) + entry = self.entries[metadata.hostname][filename] + cfg = self.core.plugins['Cfg'] + specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) + # we can't use os.path.join() for this because specific + # already has a leading /, which confuses os.path.join() + fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific)) + if filename not in cfg.entries.keys(): + self.logger.info("Writing new probed file %s" % fileloc) + try: + os.makedirs(os.path.dirname(fileloc)) + except OSError, err: + if err.errno == errno.EEXIST: + pass + else: + raise + open(fileloc, 'wb').write(contents) + + infoxml = os.path.join("%s%s" % (cfg.data, filename), + "info.xml") + self.write_infoxml(infoxml, entry, data) + else: + try: + cfgentry = \ + cfg.entries[filename].get_pertinent_entries(metadata)[0] + except Bcfg2.Server.Plugin.PluginExecutionError: + self.logger.info("Writing new probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + return + + if cfgentry.data == contents: + self.logger.debug("Existing %s contents match probed contents" % + filename) + elif (entry.get('update', 'false').lower() == "true"): + self.logger.info("Writing updated probed file %s" % fileloc) + open(fileloc, 'wb').write(contents) + else: + self.logger.info("Skipping updated probed file %s" % fileloc) + + def write_infoxml(self, infoxml, entry, data): + """ write an info.xml for the file """ + self.logger.info("Writing info.xml at %s for %s" % + (infoxml, data.get("name"))) + info = \ + lxml.etree.Element("Info", + owner=data.get("owner", + Bcfg2.Options.MDATA_OWNER.value), + group=data.get("group", + Bcfg2.Options.MDATA_GROUP.value), + perms=data.get("perms", + Bcfg2.Options.MDATA_PERMS.value), + encoding=entry.get("encoding", + Bcfg2.Options.ENCODING.value)) + + root = lxml.etree.Element("FileInfo") + root.append(info) + open(infoxml, "w").write(lxml.etree.tostring(root, + pretty_print=True)) -- cgit v1.2.3-1-g7c22 From fe17a4a28a9adcfd0b5365425f498dd4f6816511 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Wed, 11 May 2011 10:35:24 -0500 Subject: man: Fix incorrect purge/scrub command parameters --- man/bcfg2-admin.8 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/man/bcfg2-admin.8 b/man/bcfg2-admin.8 index 829d00f03..4f6528e0e 100644 --- a/man/bcfg2-admin.8 +++ b/man/bcfg2-admin.8 @@ -164,11 +164,11 @@ Initialize the database. .RS Load statistics data. .RE -.B purge +.B purge [--client [n]] [--days [n]] [--expired] .RS -Purge records. +Purge historic and expired data. .RE -.B scrub [--client [n]] [--days [n]] [--expired] +.B scrub .RS Scrub the database for duplicate reasons and orphaned entries. .RE -- cgit v1.2.3-1-g7c22 From a408e7cbe36beacc2aefe291ac3f5caec36ddf35 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Wed, 11 May 2011 11:37:18 -0500 Subject: Cfg: Fix PluginExecutionError Replaces PluginExecutionError with Bcfg2.Server.Plugin.PluginExecutionError. Reported by emias. --- src/lib/Server/Plugins/Cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index a97cbd550..7fc35cec1 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -188,7 +188,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): name = self.build_filename(specific) if name.endswith(".genshi"): logger.error("Cfg: Unable to pull data for genshi types") - raise PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) if log: logger.info("Wrote file %s" % name) -- cgit v1.2.3-1-g7c22 From 32659c415a2c438eaa2dbf160d118465439da6dd Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 11 May 2011 15:24:26 -0500 Subject: Plugin: Fix nasty list comprehension bug It appears as though Python 2 kept around variables assigned within a list comprehensions which caused this to go unnoticed. Signed-off-by: Sol Jerome --- src/lib/Server/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index cd2b63656..3b331b300 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -644,9 +644,9 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): def BindEntry(self, entry, metadata): """Check package lists of package entries.""" - [src.Cache(metadata) for src in list(self.entries.values())] name = entry.get('name') - if not src.cache: + if False in [src.Cache(metadata) for src in + list(self.entries.values())]: self.logger.error("Called before data loaded") raise PluginExecutionError matching = [src for src in list(self.entries.values()) -- cgit v1.2.3-1-g7c22 From 34d9c0652dbbaaf28fbc311ef168e29fc58155f7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 13 May 2011 13:10:00 -0500 Subject: Cfg: Fix bcfg2-admin pull behavior for genshi templates (#1010) Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 7fc35cec1..832f7ab41 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -5,6 +5,7 @@ import binascii import logging import lxml import os +import os.path import re import sys import tempfile @@ -186,7 +187,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): def write_update(self, specific, new_entry, log): if 'text' in new_entry: name = self.build_filename(specific) - if name.endswith(".genshi"): + if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError open(name, 'w').write(new_entry['text']) -- cgit v1.2.3-1-g7c22 From da0da295fc5f0d9336a8668a48c78e73248d56fa Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 14 May 2011 16:12:27 -0500 Subject: man: Point to the correct sections Signed-off-by: Sol Jerome --- man/bcfg2-lint.conf.5 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/man/bcfg2-lint.conf.5 b/man/bcfg2-lint.conf.5 index 0ae7a27ac..49a32bb22 100644 --- a/man/bcfg2-lint.conf.5 +++ b/man/bcfg2-lint.conf.5 @@ -43,7 +43,7 @@ section. A comma-delimited list of plugins to run. By default, all plugins are run. This can be overridden by listing plugins on the command line. See -.B bcfg2-lint(1) +.B bcfg2-lint(8) for a list of the available plugins. .SH ERROR HANDLING @@ -56,10 +56,10 @@ section. Each option should be the name of an error and one of , or .I "silent" , which tells -.B bcfg2-lint(1) +.B bcfg2-lint(8) how to handle the warning. Error names and their defaults can be displayed by running -.B bcfg2-lint(1) +.B bcfg2-lint(8) with the .B --list-errors option. @@ -162,5 +162,5 @@ The full path to the XML Schema files. Default is command-line option .SH SEE ALSO -.BR bcfg2-lint(1) +.BR bcfg2-lint(8) -- cgit v1.2.3-1-g7c22 From 87aca88c41efaaa5a37fb415ced67cbbcfc34ff5 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 14 May 2011 16:15:25 -0500 Subject: doc: Fix groups in centos guide (#1006) Signed-off-by: Sol Jerome --- doc/appendix/guides/centos.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/appendix/guides/centos.txt b/doc/appendix/guides/centos.txt index d788891de..0b5b83211 100644 --- a/doc/appendix/guides/centos.txt +++ b/doc/appendix/guides/centos.txt @@ -195,17 +195,17 @@ line of ``bcfg2.conf``. Then create Packages layout (as per - centos5.4 + centos-5.4 http://mrepo/centos5-x86_64/RPMS.os x86_64 - centos5.4 + centos-5.4 http://mrepo/centos5-x86_64/RPMS.updates x86_64 - centos5.4 + centos-5.4 http://mrepo/centos5-x86_64/RPMS.extras x86_64 @@ -227,9 +227,9 @@ file should look something like this - + - + -- cgit v1.2.3-1-g7c22 From fa3431d204add1baab2dd1e6ca5326f2bf1749fe Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 16 May 2011 11:19:17 -0500 Subject: doc: Better document PostInstall entries Signed-off-by: Sol Jerome --- doc/server/configurationentries.txt | 18 +++++++++++++++++- schemas/bundle.xsd | 9 +++++++++ schemas/rules.xsd | 5 +++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/doc/server/configurationentries.txt b/doc/server/configurationentries.txt index 5602da189..91aed99b6 100644 --- a/doc/server/configurationentries.txt +++ b/doc/server/configurationentries.txt @@ -34,6 +34,22 @@ Non-POSIX entries | Service | System Services | name, type, status, target | +-------------+---------------------+-----------------------------+ +.. note:: + + PostInstall entries are deprecated in favor of Action entries. In + fact, a PostInstall entry is simply a specific type of Action. + Basically, the following are equivalent:: + + .. code-block:: xml + + + + and + + .. code-block:: xml + + + POSIX entries ============= @@ -48,7 +64,7 @@ will only contain a *name* attribute. The type will be added by the plugin that handles the entry in the case of `Cfg`_, `TGenshi`_, or `TCheetah`_. If the entry is handled by the `Rules`_ plugin (i.e. it is a device, directory, hardlink, symlink, etc), then you will specify both -the *type* and any other necessary attributes in `Rules`_. +the *type* and any other necessary attributes in `Rules`_. Running ``bcfg2-lint`` will check your configuration specification for the presence of any mandatory attributes that are necessary for the diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd index b226e1078..c0a7e08ac 100644 --- a/schemas/bundle.xsd +++ b/schemas/bundle.xsd @@ -55,6 +55,15 @@ + + + + PostInstall entries are deprecated in favor of Action + entries. Actions can do everything PostInstall entries can + do and more. + + + diff --git a/schemas/rules.xsd b/schemas/rules.xsd index bc8a4af80..101b62384 100644 --- a/schemas/rules.xsd +++ b/schemas/rules.xsd @@ -24,6 +24,10 @@ + + + + @@ -68,6 +72,7 @@ + -- cgit v1.2.3-1-g7c22 From 010bc893324ddcf8be5a3b00d01bbeb9786a120b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 16 May 2011 12:44:34 -0400 Subject: bcfg2-info help returns help without starting a server instance --- src/sbin/bcfg2-info | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 161fee441..fc36b2602 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -487,7 +487,10 @@ if __name__ == '__main__': }) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) - if setup['profile'] and have_profile: + if setup['args'][0] == 'help': + print(USAGE) + sys.exit(0) + elif setup['profile'] and have_profile: prof = profile.Profile() loop = prof.runcall(infoCore, setup['repo'], setup['plugins'], setup['password'], setup['encoding'], -- cgit v1.2.3-1-g7c22 From d94c449ed7d1e68ffa119572f1b53080523e74e5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 17 May 2011 10:20:46 -0400 Subject: added documentation about manually troubleshooting TGenshi and Bundler templates --- doc/server/plugins/generators/tgenshi/index.txt | 61 +++++++++++++++++++++++-- doc/server/plugins/structures/bundler/index.txt | 17 +++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/doc/server/plugins/generators/tgenshi/index.txt b/doc/server/plugins/generators/tgenshi/index.txt index 0fc541b52..d7e0b3bf2 100644 --- a/doc/server/plugins/generators/tgenshi/index.txt +++ b/doc/server/plugins/generators/tgenshi/index.txt @@ -54,7 +54,10 @@ Inside of templates available when used in conjunction with the :ref:`server-plugins-connectors-properties` plugin) * **name** is the path name specified in bcfg -* **path** is the path to the TGenshi template +* **path** is the path to the TGenshi template. It starts with a + leading slash, and is relative to the Bcfg2 specification root. + E.g., ``/Cfg/etc/foo.conf/foo.conf.genshi`` or + ``/TGenshi/etc/foo.conf/template.newtxt.H_foo.example.com`` See the genshi `documentation `_ for examples of @@ -91,6 +94,56 @@ Produces: This flexibility provides the ability to build much more compact and succinct definitions of configuration contents than Cfg can. +Troubleshooting +=============== + +When developing a template, you can see what the template would +generate on a client with :ref:`bcfg2-info `:: + + bcfg2-info buildfile + +E.g.:: + + bcfg2-info buildfile /etc/foo.conf foo.example.com + +Sometimes, it's useful to be able to do more in-depth troubleshooting +by running the template manually. (This is also necessary if you want +to generate a template that depends on an :ref:`altsrc +` tag.) To do this, run ``bcfg2-info +debug``, and, once in the Python interpreter, run:: + + metadata = self.build_metadata("") + path = "" + bcfg2root = "" + +``path`` should be set to the path to the template file with a leading +slash, relative to the Bcfg2 specification root. See `Inside of +Templates`_ for examples. + +``bcfg2root`` should be set to the absolute path to the Bcfg2 +specification. (This is ``/var/lib/bcfg2`` by default.) + +Then, run:: + + import os + name = os.path.dirname(path[path.find('/', 1):]) + from genshi.template import TemplateLoader, NewTextTemplate + template = TemplateLoader().load(bcfg2root + path, cls=NewTextTemplate) + print template.generate(metadata=metadata, path=path, name=name).render() + +This gives you more fine-grained control over how your template is +rendered. + +You can also use this approach to render templates that depend on +:ref:`altsrc ` tags by setting +``path`` to the path to the template, and setting ``name`` to the path +to the file to be generated, e.g.:: + + metadata = self.build_metadata("foo.example.com") + path = "/Cfg/etc/sysconfig/network-scripts/ifcfg-template/ifcfg-template.genshi" + bcfg2root = "/var/lib/bcfg2" + name = "/etc/sysconfig/network-scripts/ifcfg-bond0" + File permissions ================ @@ -101,10 +154,10 @@ Permissions entry and a Path entry to handle the same file. Error handling ================ -Situations may arrise where a templated file cannot be generated due to +Situations may arise where a templated file cannot be generated due to missing or incomplete information. A TemplateError can be raised to -force a bind failure and prevent sending an incomplete file to the client. -For example, this template:: +force a bind failure and prevent sending an incomplete file to the +client. For example, this template:: {% python from genshi.template import TemplateError diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 9fd897385..6b5c246aa 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -156,6 +156,23 @@ format is XML. A Genshi template looks much like a Bundler file, except the Bundle tag has an additional `xmlns:py` attribute. See the examples. +Troubleshooting +--------------- + +There is no :ref:`bcfg2-info ` command like +``buildfile`` for Bundler templates, so if you want to generate a +Bundler template for a given client, you have to do so manually by +first invoking ``bcfg2-info debug``, then run:: + + metadata = self.build_metadata("") + path = "" + from genshi.template import TemplateLoader, MarkupTemplate + template = TemplateLoader().load(path, cls=MarkupTemplate) + print template.generate(metadata=metadata).render('xml') + +``path`` needs to be the full path to the template file on the +filesystem, not just within the Bcfg2 repo. + Altsrc ====== -- cgit v1.2.3-1-g7c22 From 58e8891a16b31c23a09e019ae0de8bef7a37b6a5 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 17 May 2011 11:10:58 -0500 Subject: doc: Fix formatting error Signed-off-by: Sol Jerome --- doc/server/configurationentries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/server/configurationentries.txt b/doc/server/configurationentries.txt index 91aed99b6..10eccf6be 100644 --- a/doc/server/configurationentries.txt +++ b/doc/server/configurationentries.txt @@ -38,7 +38,7 @@ Non-POSIX entries PostInstall entries are deprecated in favor of Action entries. In fact, a PostInstall entry is simply a specific type of Action. - Basically, the following are equivalent:: + Basically, the following are equivalent: .. code-block:: xml -- cgit v1.2.3-1-g7c22 From a19c87f8dc03132e7257a539c7e486b9c9298aff Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 17 May 2011 11:46:58 -0500 Subject: bcfg2-info: Fix traceback when no args specified Signed-off-by: Sol Jerome --- src/sbin/bcfg2-info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index fc36b2602..c36e1af42 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -487,7 +487,7 @@ if __name__ == '__main__': }) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) - if setup['args'][0] == 'help': + if setup['args'] and setup['args'][0] == 'help': print(USAGE) sys.exit(0) elif setup['profile'] and have_profile: -- cgit v1.2.3-1-g7c22 From b7234fe39d1f25eb55320fed8491781aef1ed9fe Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 17 May 2011 15:46:57 -0400 Subject: added bcfg2-lint MergeFiles plugin to suggest config files and probes that are very similar and could be merged added text wrapping to bcfg2-lint error handling --- examples/bcfg2-lint.conf | 5 ++- man/bcfg2-lint.8 | 5 +++ man/bcfg2-lint.conf.5 | 8 +++++ src/lib/Server/Lint/MergeFiles.py | 71 +++++++++++++++++++++++++++++++++++++++ src/lib/Server/Lint/__init__.py | 43 ++++++++++++++++++------ 5 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 src/lib/Server/Lint/MergeFiles.py diff --git a/examples/bcfg2-lint.conf b/examples/bcfg2-lint.conf index abf969496..9c0d2c72a 100644 --- a/examples/bcfg2-lint.conf +++ b/examples/bcfg2-lint.conf @@ -1,5 +1,5 @@ [lint] -plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate +plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate,MergeFiles [errors] no-infoxml=error @@ -23,3 +23,6 @@ probe_comments = Maintainer,Purpose,Groups,Other Output [Validate] schema=/usr/share/bcfg2/schema + +[MergeFiles] +threshold=85 diff --git a/man/bcfg2-lint.8 b/man/bcfg2-lint.8 index b1fa9244b..f2d4f9e88 100644 --- a/man/bcfg2-lint.8 +++ b/man/bcfg2-lint.8 @@ -119,6 +119,11 @@ are specified. Can also require that an exists for all Cfg files, and that paranoid mode be enabled for all files. +.TP +.BR MergeFiles +Suggest that similar probes and config files be merged into single +probes or TGenshi templates. + .TP .BR Pkgmgr Check for duplicate packages specified in Pkgmgr. diff --git a/man/bcfg2-lint.conf.5 b/man/bcfg2-lint.conf.5 index 49a32bb22..10a812874 100644 --- a/man/bcfg2-lint.conf.5 +++ b/man/bcfg2-lint.conf.5 @@ -151,6 +151,14 @@ A comma-delimited list of attributes to require on .I tags. Default is "owner,group,perms". +.TP +.BR MergeFiles + +\(bu +.B threshold +The threshold at which MergeFiles will suggest merging config files +and probes. Default is 75% similar. + .TP .BR Validate diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py new file mode 100644 index 000000000..1e177acff --- /dev/null +++ b/src/lib/Server/Lint/MergeFiles.py @@ -0,0 +1,71 @@ +import os +from copy import deepcopy +from difflib import SequenceMatcher +import Bcfg2.Options +import Bcfg2.Server.Lint + +class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): + """ find Probes or Cfg files with multiple similar files that + might be merged into one """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + if 'Probes' in self.core.plugins: + self.check_probes() + + def check_cfg(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for mset in self.get_similar(entryset.entries): + self.LintError("merge-cfg", + "The following files are similar: %s. " + "Consider merging them into a single Genshi " + "template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + + def check_probes(self): + probes = self.core.plugins['Probes'].probes.entries + for mset in self.get_similar(probes): + self.LintError("merge-cfg", + "The following probes are similar: %s. " + "Consider merging them into a single probe." % + ", ".join([p for p in mset])) + + def get_similar(self, entries): + if "threshold" in self.config: + # accept threshold either as a percent (e.g., "threshold=75") or + # as a ratio (e.g., "threshold=.75") + threshold = float(self.config['threshold']) + if threshold > 1: + threshold /= 100 + else: + threshold = 0.75 + rv = [] + elist = entries.items() + while elist: + result = self._find_similar(elist.pop(0), deepcopy(elist), + threshold) + if len(result) > 1: + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in result] + rv.append(result) + return rv + + def _find_similar(self, ftuple, others, threshold): + fname, fdata = ftuple + rv = [fname] + while others: + cname, cdata = others.pop(0) + sm = SequenceMatcher(None, fdata.data, cdata.data) + # perform progressively more expensive comparisons + if (sm.real_quick_ratio() > threshold and + sm.quick_ratio() > threshold and + sm.ratio() > threshold): + rv.extend(self._find_similar((cname, cdata), deepcopy(others), + threshold)) + return rv + + diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 3b89d1f9e..013cbf2ba 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -4,6 +4,7 @@ __all__ = ['Bundles', 'Comments', 'Duplicates', 'InfoXML', + 'MergeFiles', 'Pkgmgr', 'RequiredAttrs', 'Validate'] @@ -11,6 +12,7 @@ __all__ = ['Bundles', import logging import os.path from copy import copy +import textwrap import lxml.etree import Bcfg2.Logger @@ -84,7 +86,9 @@ class ErrorHandler (object): "properties-schema-not-found":"warning", "xml-failed-to-parse":"error", "xml-failed-to-read":"error", - "xml-failed-to-verify":"error",} + "xml-failed-to-verify":"error", + "merge-cfg":"warning", + "merge-probes":"warning",} def __init__(self, config=None): self.errors = 0 @@ -92,6 +96,9 @@ class ErrorHandler (object): self.logger = logging.getLogger('bcfg2-lint') + self._wrapper = textwrap.TextWrapper(initial_indent = " ", + subsequent_indent = " ") + self._handlers = {} if config is not None: for err, action in config.items(): @@ -116,26 +123,42 @@ class ErrorHandler (object): self._handlers[err](msg) self.logger.debug(" (%s)" % err) else: - self.logger.info("Unknown error %s" % err) + # assume that it's an error, but complain + self.error(msg) + self.logger.warning("Unknown error %s" % err) def error(self, msg): """ log an error condition """ self.errors += 1 - lines = msg.splitlines() - self.logger.error("ERROR: %s" % lines.pop()) - [self.logger.error(" %s" % l) for l in lines] + self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): """ log a warning condition """ self.warnings += 1 - lines = msg.splitlines() - self.logger.warning("WARNING: %s" % lines.pop()) - [self.logger.warning(" %s" % l) for l in lines] + self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): """ log a silent/debug condition """ - lines = msg.splitlines() - [self.logger.debug("%s" % l) for l in lines] + self._log(msg, self.logger.debug) + + def _log(self, msg, logfunc, prefix=""): + # a message may itself consist of multiple lines. wrap() will + # elide them all into a single paragraph, which we don't want. + # so we split the message into its paragraphs and wrap each + # paragraph individually. this means, unfortunately, that we + # lose textwrap's built-in initial indent functionality, + # because we want to only treat the very first line of the + # first paragraph specially. so we do some silliness. + rawlines = msg.splitlines() + firstline = True + for rawline in rawlines: + lines = self._wrapper.wrap(rawline) + for line in lines: + if firstline: + logfunc("%s%s" % (prefix, line.lstrip())) + firstline = False + else: + logfunc(line) class ServerlessPlugin (Plugin): -- cgit v1.2.3-1-g7c22 From 96d93d3c16c1e6cb0aa4748f8b392b84197a644a Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 17 May 2011 18:20:48 -0500 Subject: doc: Fix Cfg example Signed-off-by: Sol Jerome --- doc/server/plugins/generators/cfg.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 612b14bec..1039ff556 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -132,16 +132,16 @@ server and we have the following configuration files:: motd.G01_web-server motd.G01_mail-server.cat motd.G02_file-server.cat - motd.H_foo.example.cat + motd.H_foo.example.com.cat -If our machine isn't *foo.example.com* then here's what would happen: +If our machine **isn't** *foo.example.com* then here's what would happen: Bcfg2 would choose ``motd.G01_web-server`` as the base file. It is the most specific base file for this host. Bcfg2 would apply the ``motd.G01_mail-server.cat`` delta to the ``motd.G01_web-server`` base file. It is the least specific delta. Bcfg2 would then apply the ``motd.G02_file-server.cat`` delta to the result of the delta before -it. If our machine is foo.example.com then here's what would happen: +it. If our machine **is** *foo.example.com* then here's what would happen: Bcfg2 would choose ``motd.G01_web-server`` as the base file. It is the most specific base file for this host. Bcfg2 would apply the -- cgit v1.2.3-1-g7c22 From 7ea350c0571ea8f551efa9fcfcc2eafc456dd9d9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 18 May 2011 10:44:29 -0400 Subject: Fixed fileprobes schema --- schemas/fileprobes.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/fileprobes.xsd b/schemas/fileprobes.xsd index 112047836..e10dc51dd 100644 --- a/schemas/fileprobes.xsd +++ b/schemas/fileprobes.xsd @@ -18,7 +18,7 @@ - + -- cgit v1.2.3-1-g7c22 From 065724b38235cb8ba4aa493f43a3f4f426ac1abd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 18 May 2011 10:44:50 -0400 Subject: fixed several major bugs in bcfg2-lint Validate plugin --- src/lib/Server/Lint/Validate.py | 16 ++++++++++------ src/sbin/bcfg2-lint | 7 +++++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index 834608378..c0a400dd6 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -14,7 +14,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filesets = {"metadata:groups":"%s/metadata.xsd", "metadata:clients":"%s/clients.xsd", "info":"%s/info.xsd", - "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd", + "%s/Bundler/*.xml":"%s/bundle.xsd", + "%s/Bundler/*.genshi":"%s/bundle.xsd", "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", "%s/Base/*.xml":"%s/base.xsd", "%s/Rules/*.xml":"%s/rules.xsd", @@ -33,14 +34,16 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): - self.schemadir = self.config['schema'] + schemadir = self.config['schema'] - for schemaname, path in self.filesets.items(): + for path, schemaname in self.filesets.items(): try: filelist = self.filelists[path] except KeyError: filelist = [] - + + print "validating %s" % path + print "filelist = %s" % filelist if filelist: # avoid loading schemas for empty file lists try: @@ -48,8 +51,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): schemadir)) except: self.LintError("schema-failed-to-parse", - "Failed to process schema %s", - schemaname % schemadir) + "Failed to process schema %s" % + (schemaname % schemadir)) continue for filename in filelist: self.validate(filename, schemaname % schemadir, @@ -132,6 +135,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == 'info.xml']) else: + print "globbing for %s" % (path % self.config['repo']) self.filelists[path] = listfiles(path) self.filelists['props'] = listfiles("%s/Properties/*.xml") diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 6bc34433e..3c9bc44b3 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -50,10 +50,13 @@ def run_plugin(plugin, plugin_name, setup=None, errorhandler=None, errorhandler = get_errorhandler(config) if config is not None and config.has_section(plugin_name): - args.append(dict(config.items(plugin_name), **setup)) + arg = setup + for key, val in config.items(plugin_name): + arg[key] = val + args.append(arg) else: args.append(setup) - + # older versions of python do not support mixing *-magic and # non-*-magic (e.g., "plugin(*args, files=files)", so we do this # all with *-magic -- cgit v1.2.3-1-g7c22 From e081a2355c6afa182359bd44aa74ad45f16636f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Thu, 19 May 2011 23:08:48 +0200 Subject: Revert "bcfg2: Write cached config using encoding from Options.py" This reverts commit 5d69ff7e966c4ffa911c78d11a6879b48e90aef8. As the "rawconfig" variable now holds a UTF-8 encoded string, it would have to be decoded before re-encoding it using a different encoding. However, the cached configuration shouldn't be written using a non-UTF-8 encoding anyway, as "bcfg2 -f " currently doesn't accept any other encodings. (If this is to be changed, the XML encoding declaration of the configuration would have to be adjusted accordingly.) --- src/sbin/bcfg2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 996d773ff..eb41af640 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -252,7 +252,7 @@ class Client: if self.setup['cache']: try: - open(self.setup['cache'], 'w').write(rawconfig.encode(self.setup['encoding'])) + open(self.setup['cache'], 'w').write(rawconfig) os.chmod(self.setup['cache'], 33152) except IOError: self.logger.warning("Failed to write config cache file %s" % -- cgit v1.2.3-1-g7c22 From 8168776fb50858a0e531c3a874ee1e20a88e7690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Sun, 15 May 2011 15:23:59 +0200 Subject: APT: Add two filters for deprecated API accesses These accesses are triggered by configuration entries which don't have the "version" attribute set to "auto" or "any". --- src/lib/Client/Tools/APT.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Client/Tools/APT.py index a838f5e27..d7be44dc0 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Client/Tools/APT.py @@ -9,6 +9,8 @@ warnings.filterwarnings("ignore", "Accessed deprecated property Package.installe warnings.filterwarnings("ignore", "Accessed deprecated property Package.candidateVersion, please see the Version class for alternatives.", DeprecationWarning) warnings.filterwarnings("ignore", "Deprecated, please use 'is_installed' instead", DeprecationWarning) warnings.filterwarnings("ignore", "Attribute 'IsUpgradable' of the 'apt_pkg.DepCache' object is deprecated, use 'is_upgradable' instead.", DeprecationWarning) +warnings.filterwarnings("ignore", "Attribute 'VersionList' of the 'apt_pkg.Package' object is deprecated, use 'version_list' instead.", DeprecationWarning) +warnings.filterwarnings("ignore", "Attribute 'VerStr' of the 'apt_pkg.Version' object is deprecated, use 'ver_str' instead.", DeprecationWarning) import apt.cache import os -- cgit v1.2.3-1-g7c22 From 37c5d256bda2ffe037680c9d5bfc5d986a58f7ef Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 May 2011 09:12:59 -0400 Subject: removed debugging output from Validate plugin --- src/lib/Server/Lint/Validate.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c0a400dd6..c9b5688e1 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -42,8 +42,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): except KeyError: filelist = [] - print "validating %s" % path - print "filelist = %s" % filelist if filelist: # avoid loading schemas for empty file lists try: @@ -135,7 +133,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == 'info.xml']) else: - print "globbing for %s" % (path % self.config['repo']) self.filelists[path] = listfiles(path) self.filelists['props'] = listfiles("%s/Properties/*.xml") -- cgit v1.2.3-1-g7c22 From bd65a65555b5a70a43b1cdc8a7309cf85d84aa87 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 May 2011 09:16:02 -0400 Subject: fixed typo in bcfg2-lint man page (thanks jsbillings) --- man/bcfg2-lint.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/bcfg2-lint.8 b/man/bcfg2-lint.8 index f2d4f9e88..25fa30f9e 100644 --- a/man/bcfg2-lint.8 +++ b/man/bcfg2-lint.8 @@ -71,7 +71,7 @@ Require property files to have matching schema files .SH "PLUGINS" See -.BR bcfg-lint.conf(5) +.BR bcfg2-lint.conf(5) for more information on the configuration of the plugins listed below. .TP -- cgit v1.2.3-1-g7c22 From 4944977b131be888f4584b4b6b44dba2214d1818 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 May 2011 09:58:13 -0400 Subject: don't run lint server plugins if serverless plugins produced errors; avoids an ugly stack trace if a file fails to validate --- src/sbin/bcfg2-lint | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 3c9bc44b3..464e839e5 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -184,8 +184,20 @@ if __name__ == '__main__': config=config, setup=setup) if serverplugins: - run_server_plugins(serverplugins, errorhandler=errorhandler, - config=config, setup=setup) + if errorhandler.errors: + # it would be swell if we could try to start the server + # even if there were errors with the serverless plugins, + # but since XML parsing errors occur in the FAM thread + # (not in the core server thread), there's no way we can + # start the server and try to catch exceptions -- + # bcfg2-lint isn't in the same stack as the exceptions. + # so we're forced to assume that a serverless plugin error + # will prevent the server from starting + print("Serverless plugins encountered errors, skipping server " + "plugins") + else: + run_server_plugins(serverplugins, errorhandler=errorhandler, + config=config, setup=setup) if errorhandler.errors or errorhandler.warnings or setup['verbose']: print("%d errors" % errorhandler.errors) -- cgit v1.2.3-1-g7c22 From 7792959d08beb998df138f5dd8f4451faea50c40 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 21 May 2011 23:13:45 -0500 Subject: schemas: Fix schema for important attribute Signed-off-by: Sol Jerome --- schemas/info.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/info.xsd b/schemas/info.xsd index 96ccbe56c..2ff1d937e 100644 --- a/schemas/info.xsd +++ b/schemas/info.xsd @@ -11,7 +11,7 @@ - + -- cgit v1.2.3-1-g7c22 From 81eeff2a4a9381811d6f3d20ebd6bcf18bf83553 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 24 May 2011 15:41:37 -0400 Subject: allow setting whitelist/blacklist mode in bcfg2.conf --- doc/server/plugins/generators/decisions.txt | 37 ++++++++++++++++++++--------- src/lib/Options.py | 1 + 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/doc/server/plugins/generators/decisions.txt b/doc/server/plugins/generators/decisions.txt index ba01f7fc2..d75c298ef 100644 --- a/doc/server/plugins/generators/decisions.txt +++ b/doc/server/plugins/generators/decisions.txt @@ -22,6 +22,11 @@ should be corrected during the current run of the installation tool. The Decisions plugin is the only stock plugin that generates entries for client's whitelists or blacklists. +.. note:: If the client is not explicitly configured to run in + whitelist or blacklist mode, the list of entries is not + downloaded and decisions is not used. See ``Decision + Mode``_ below. + The Decisions plugin uses a directory in the Bcfg2 repository called Decisions. Files in the Decisions subdirectory are named similarly to files managed by Cfg, probes, TCheetah, and TGenshi (so you can use host- @@ -37,12 +42,6 @@ the following is an example. -.. note:: To add syntax highlighting in vim, you can add a modeline such as this: - - - This example, included as a whitelist due to its name, enables all services, and the path entry named ``/etc/apt/apt.conf``. All these entries must already be present in your repository, the Decisions plugin just references @@ -55,11 +54,27 @@ When a client asks for its whitelist or blacklist, all of the files pertaining to that client of the correct type are aggregated into a single list. This list is sent to the client. -.. note:: This list is only generated when a client is explicitly run with - the appropriate option (``-l (whitelist|blacklist)``); client - behavior is not controlled unless this option is used. If you do - not use Decisions, all your entries will be installed normally. -.. note:: Also, using this plugin does not present additional prompts or +.. note:: Using this plugin does not present additional prompts or safety nets to the administrator running the client, you have to control these via their respective options (``-I`` or ``-n``, for example). + +To add syntax highlighting to Decisions files in vim and emacs, you +can add comments such as this:: + + + + + +============= +Decision Mode +============= + +The whitelist or blacklist is only generated when a client is run in +whitelist or blacklist mode. This can either be set at the command +line with the appropriate option (``-l (whitelist|blacklist)``), or in +``bcfg2.conf`` by setting ``decision`` in the ``client`` section to +``whitelist`` or ``blacklist``). + +Client behavior is not controlled unless the decision mode is set. If +you do not use Decisions, all your entries will be installed normally. diff --git a/src/lib/Options.py b/src/lib/Options.py index d5304e696..26d0fa349 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -307,6 +307,7 @@ CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) CLIENT_DLIST = Option('Run client in server decision list mode', default=False, + cf=('client', 'decision'), cmd='-l', odesc='') CLIENT_FILE = Option('Configure from a file rather than querying the server', default=False, cmd='-f', odesc='') -- cgit v1.2.3-1-g7c22 From ac1e8e89733d1420afb69c8c7e96835243ffc323 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 25 May 2011 22:18:59 -0500 Subject: doc: Fix inline literal reference Signed-off-by: Sol Jerome --- doc/server/plugins/generators/decisions.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/server/plugins/generators/decisions.txt b/doc/server/plugins/generators/decisions.txt index d75c298ef..d75a9fede 100644 --- a/doc/server/plugins/generators/decisions.txt +++ b/doc/server/plugins/generators/decisions.txt @@ -22,10 +22,11 @@ should be corrected during the current run of the installation tool. The Decisions plugin is the only stock plugin that generates entries for client's whitelists or blacklists. -.. note:: If the client is not explicitly configured to run in - whitelist or blacklist mode, the list of entries is not - downloaded and decisions is not used. See ``Decision - Mode``_ below. +.. note:: + + If the client is not explicitly configured to run in whitelist or + blacklist mode, the list of entries is not downloaded and decisions + is not used. See `Decision Mode`_ below. The Decisions plugin uses a directory in the Bcfg2 repository called Decisions. Files in the Decisions subdirectory are named similarly to -- cgit v1.2.3-1-g7c22 From be72eb8b648672707822f891a103fc75ed54dbd2 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 25 May 2011 22:19:40 -0500 Subject: POSIX: Clarify normalization error (Reported by Tim Goodaire) Signed-off-by: Sol Jerome --- doc/help/troubleshooting.txt | 54 ++++++++++++++++++++++++++++--------------- src/lib/Client/Tools/POSIX.py | 6 +++-- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/doc/help/troubleshooting.txt b/doc/help/troubleshooting.txt index 88a0febc5..010c80de5 100644 --- a/doc/help/troubleshooting.txt +++ b/doc/help/troubleshooting.txt @@ -109,46 +109,56 @@ be taken to remedy them. | authenticate the server with | | unable to verify | | | SSL. | | the server | | +------------------------------+----------+---------------------+--------------+ -| Failed to bind entry: | Server | The server was | [4]_ | +| GID normalization failed for | Client | The client is | [4]_ | +| FILENAME. Does group GROUP | | unable to convert | | +| exist? | | the group GROUP to | | +| | | a usable GID | | ++------------------------------+----------+---------------------+--------------+ +| UID normalization failed for | Client | The client is | [5]_ | +| FILENAME. Does owner OWNER | | unable to convert | | +| exist? | | the owner OWNER to | | +| | | a usable UID | | ++------------------------------+----------+---------------------+--------------+ +| Failed to bind entry: | Server | The server was | [6]_ | | | | unable to find a | | | | | suitable version of | | | | | entry for client. | | +------------------------------+----------+---------------------+--------------+ -| Failed to bind to socket | Server | The server was | [5]_ | +| Failed to bind to socket | Server | The server was | [7]_ | | | | unable to bind to | | | | | the tcp server | | | | | socket. | | +------------------------------+----------+---------------------+--------------+ -| Failed to load | Server | The server was | [6]_ | +| Failed to load | Server | The server was | [8]_ | | ssl key | | unable to read and | | | | | process the ssl key.| | +------------------------------+----------+---------------------+--------------+ -| Failed to read file | Server | The server failed | [7]_ | +| Failed to read file | Server | The server failed | [9]_ | | | | to read the | | | | | specified file | | +------------------------------+----------+---------------------+--------------+ -| Failed to parse file | Server | The server failed | [8]_ | +| Failed to parse file | Server | The server failed | [10]_ | | | | to parse the | | | | | specified XML file | | +------------------------------+----------+---------------------+--------------+ -| Client metadata resolution | Server | The server cannot | [9]_ | +| Client metadata resolution | Server | The server cannot | [11]_ | | error for | | resolve the client | | | | | hostname or the | | | | | client is | | | | | associated with a | | | | | non-profile group. | | +------------------------------+----------+---------------------+--------------+ -| Failed to decode | Server | The encoding being | [10]_ | +| Failed to decode | Server | The encoding being | [12]_ | | Please verify you are using | | used is unable to | | | the proper encoding | | decode the | | | | | character present | | | | | in this file. | | +------------------------------+----------+---------------------+--------------+ -| Got unknown entries | Server | The Packages plugin | [11]_ | +| Got unknown entries | Server | The Packages plugin | [13]_ | | [list of unknown entries] | | has no knowledge of | | | | | the listed entries | | +------------------------------+----------+---------------------+--------------+ -| Failed to import lxml | Server | The server cannot | [12]_ | +| Failed to import lxml | Server | The server cannot | [14]_ | | dependency. Shutting | | import lxml | | | down server. | | | | +------------------------------+----------+---------------------+--------------+ @@ -162,20 +172,26 @@ be taken to remedy them. .. [3] Copy the Bcfg2 server's CA certificate to the client and specify it using the **ca** option in the [communication] section of ``bcfg2.conf`` -.. [4] This entry is not being bound. Ensure that a version of this +.. [4] If the group doesn't exist, you need to specify the correct one + in an :ref:`info.xml ` file or set the default group + appropriately. +.. [5] If the owner doesn't exist, you need to specify the correct one + in an :ref:`info.xml ` file or set the default owner + appropriately. +.. [6] This entry is not being bound. Ensure that a version of this entry applies to this client. -.. [5] Ensure that another instance of the daemon (or any other process) +.. [7] Ensure that another instance of the daemon (or any other process) is not listening on the same port. -.. [6] Ensure that the key is readable by the user running the daemon +.. [8] Ensure that the key is readable by the user running the daemon and that it is well-formed. -.. [7] Ensure that this file still exists; a frequent cause is the +.. [9] Ensure that this file still exists; a frequent cause is the deletion of a temp file. -.. [8] Ensure that the file is properly formed XML. -.. [9] Fix hostname resolution for the client or ensure that the profile - group is properly setup. -.. [10] Ensure the correct encoding is specified in the [components] +.. [10] Ensure that the file is properly formed XML. +.. [11] Fix hostname resolution for the client or ensure that the profile + group is properly setup. +.. [12] Ensure the correct encoding is specified in the [components] section of ``bcfg2.conf``. -.. [11] For packages listed other than **gpg-pubkey**, this error means +.. [13] For packages listed other than **gpg-pubkey**, this error means that the Packages plugin is unable to find the package in any of the sources listed in ``Packages/config.xml``. The issue often arises when the client is not in one of the groups necessary for @@ -183,7 +199,7 @@ be taken to remedy them. ignore the message as the Packages plugin has no knowledge of these packages (however, note that this package is most often specified as a BoundPackage entry). -.. [12] Ensure that you have installed all the necessary +.. [14] Ensure that you have installed all the necessary :ref:`installation-prerequisites`. FAQs diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index af3d1a473..875db5ea7 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -55,7 +55,8 @@ def normGid(entry): except: return int(grp.getgrnam(entry.get('group'))[2]) except (OSError, KeyError): - log.error('GID normalization failed for %s' % (entry.get('name'))) + log.error('GID normalization failed for %s. Does group %s exist?' + % (entry.get('name'), entry.get('group'))) return False @@ -70,7 +71,8 @@ def normUid(entry): except: return int(pwd.getpwnam(entry.get('owner'))[2]) except (OSError, KeyError): - log.error('UID normalization failed for %s' % (entry.get('name'))) + log.error('UID normalization failed for %s. Does owner %s exist?' + % (entry.get('name'), entry.get('owner'))) return False -- cgit v1.2.3-1-g7c22 From 99495c559da8bb432274e94248f98d9b34c2d205 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 31 May 2011 10:36:15 -0500 Subject: Cfg: Fix traceback for non-ascii files Signed-off-by: Sol Jerome --- doc/help/troubleshooting.txt | 6 ++++++ src/lib/Server/Plugins/Cfg.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/doc/help/troubleshooting.txt b/doc/help/troubleshooting.txt index 010c80de5..ecf53527c 100644 --- a/doc/help/troubleshooting.txt +++ b/doc/help/troubleshooting.txt @@ -162,6 +162,10 @@ be taken to remedy them. | dependency. Shutting | | import lxml | | | down server. | | | | +------------------------------+----------+---------------------+--------------+ +| You need to specify base64 | Server | The server cannot | [15]_ | +| encoding for | | send the file as | | +| | | ascii text | | ++------------------------------+----------+---------------------+--------------+ .. [1] This entry is not being bound. Ensure that a version of this @@ -201,6 +205,8 @@ be taken to remedy them. specified as a BoundPackage entry). .. [14] Ensure that you have installed all the necessary :ref:`installation-prerequisites`. +.. [15] You likely need to specify a base64 encoding using an + :ref:`server-info` file for this entry. FAQs ==== diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 832f7ab41..5e3cca847 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -162,6 +162,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) logger.error("Please verify you are using the proper encoding.") raise Bcfg2.Server.Plugin.PluginExecutionError + except ValueError: + e = sys.exc_info()[1] + logger.error("Error in specification for %s" % entry.get('name')) + logger.error("%s" % e) + logger.error("You need to specify base64 encoding for %s." % + entry.get('name')) + raise Bcfg2.Server.Plugin.PluginExecutionError if entry.text in ['', None]: entry.set('empty', 'true') -- cgit v1.2.3-1-g7c22 From 6e55609dd1857183648c794cc56338891bfff409 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 31 May 2011 12:40:50 -0500 Subject: export.py: Update documentation version numbers Signed-off-by: Sol Jerome --- tools/export.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/export.py b/tools/export.py index d637c166c..43ec03a23 100755 --- a/tools/export.py +++ b/tools/export.py @@ -85,6 +85,13 @@ for line in fileinput.input('src/lib/Server/Reports/reports/templates/base.html' if 'Bcfg2 Version' in line: line = line.replace(line, ' Bcfg2 Version %s\n' % version) sys.stdout.write(line) +# update the version in the docs +for line in fileinput.input('doc/conf.py', inplace=1): + if line.startswith('version ='): + line = line.replace(line, 'version = \'%s\'\n' % majorver[0:3]) + if line.startswith('release ='): + line = line.replace(line, 'release = \'%s\'\n' % (majorver + minorver)) + sys.stdout.write(line) # tag the release #FIXME: do this using python-dulwich -- cgit v1.2.3-1-g7c22 From 3dfcbe27a11988512a0315b013adbc31cd24d3f8 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 31 May 2011 14:01:06 -0500 Subject: export.py: Refactor find and replace bits Signed-off-by: Sol Jerome --- tools/export.py | 61 ++++++++++++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/tools/export.py b/tools/export.py index 43ec03a23..47b23469a 100755 --- a/tools/export.py +++ b/tools/export.py @@ -25,6 +25,17 @@ tarname = '/tmp/%s-%s.tar.gz' % (pkgname, version) def run(command): return Popen(command, shell=True, stdout=PIPE).communicate() +def find_and_replace(f, iftest, rline, startswith=False): + for line in fileinput.input(f, inplace=1): + if startswith: + if line.startswith(iftest): + line = line.replace(line, rline) + sys.stdout.write(line) + else: + if iftest in line and line != "Version: %{version}\n": + line = line.replace(line, rline) + sys.stdout.write(line) + # update the version majorver = version[:5] minorver = version[5:] @@ -49,16 +60,6 @@ with open('debian/changelog', 'r+') as f: f.seek(0) f.write(newchangelog + old) f.close() -# set new version in setup.py -for line in fileinput.input('setup.py', inplace=1): - if 'version' in line: - line = line.replace(line, ' version="%s",\n' % version) - sys.stdout.write(line) -# replace version in misc/bcfg2.spec -for line in fileinput.input('misc/bcfg2.spec', inplace=1): - if 'Version:' in line and line != "Version: %{version}\n": - line = line.replace(line, 'Version: %s\n' % version) - sys.stdout.write(line) # Update redhat directory versions with open('redhat/VERSION', 'w') as f: f.write("%s\n" % majorver) @@ -67,31 +68,25 @@ with open('redhat/RELEASE', 'w') as f: f.write("0.0%s\n" % minorver) f.close() # update solaris version -for line in fileinput.input('solaris/Makefile', inplace=1): - if line.startswith('VERS='): - line = line.replace(line, 'VERS=%s-1\n' % version) - sys.stdout.write(line) -for line in fileinput.input('solaris/pkginfo.bcfg2', inplace=1): - if line.startswith('VERSION='): - line = line.replace(line, 'VERSION="%s"\n' % version) - sys.stdout.write(line) -for line in fileinput.input('solaris/pkginfo.bcfg2-server', inplace=1): - if line.startswith('VERSION='): - line = line.replace(line, 'VERSION="%s"\n' % version) - sys.stdout.write(line) +find_and_replace('solaris/Makefile', 'VERS=', + 'VERS=%s-1\n' % version, startswith=True) +find_and_replace('solaris/pkginfo.bcfg2', 'VERSION=', + 'VERSION="%s"\n' % version, startswith=True) +find_and_replace('solaris/pkginfo.bcfg2-server', 'VERSION=', + 'VERSION="%s"\n' % version, startswith=True) +# set new version in setup.py +find_and_replace('setup.py', 'version=', ' version="%s",\n' % version) +# replace version in misc/bcfg2.spec +find_and_replace('misc/bcfg2.spec', 'Version:', + 'Version: %s\n' % version) # update the version in reports -for line in fileinput.input('src/lib/Server/Reports/reports/templates/base.html', - inplace=1): - if 'Bcfg2 Version' in line: - line = line.replace(line, ' Bcfg2 Version %s\n' % version) - sys.stdout.write(line) +find_and_replace('src/lib/Server/Reports/reports/templates/base.html', + 'Bcfg2 Version', ' Bcfg2 Version %s\n' % version) # update the version in the docs -for line in fileinput.input('doc/conf.py', inplace=1): - if line.startswith('version ='): - line = line.replace(line, 'version = \'%s\'\n' % majorver[0:3]) - if line.startswith('release ='): - line = line.replace(line, 'release = \'%s\'\n' % (majorver + minorver)) - sys.stdout.write(line) +find_and_replace('doc/conf.py', 'version =', + 'version = \'%s\'\n' % majorver[0:3], startswith=True) +find_and_replace('doc/conf.py', 'release =', + 'release = \'%s\'\n' % (majorver + minorver), startswith=True) # tag the release #FIXME: do this using python-dulwich -- cgit v1.2.3-1-g7c22 From 3f13297b48c511de37c2a3233780d07d089c9993 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 1 Jun 2011 09:50:25 -0400 Subject: added -t option to set client timeout --- man/bcfg2.1 | 5 +++++ src/lib/Options.py | 3 +++ src/lib/Server/Admin/Perf.py | 4 +++- src/lib/Server/Admin/Xcmd.py | 5 +++-- src/sbin/bcfg2 | 10 ++++++---- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/man/bcfg2.1 b/man/bcfg2.1 index 938d41dfe..7f108ed8f 100644 --- a/man/bcfg2.1 +++ b/man/bcfg2.1 @@ -146,6 +146,11 @@ Attempt to authenticate as 'user'. .BR "\-x " Use 'password' for client communication. +.TP +.BR "\-t " +Set the timeout (in seconds) for client communication. Default is 90 +seconds. + .TP .BR "\-v" Run bcfg2 in verbose mode. diff --git a/src/lib/Options.py b/src/lib/Options.py index 26d0fa349..e6eebb808 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -317,6 +317,9 @@ CLIENT_USER = Option('The user to provide for authentication', default='root', cmd='-u', cf=('communication', 'user'), odesc='') CLIENT_SERVICE_MODE = Option('Set client service mode', default='default', cmd='-s', odesc='') +CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90, + cmd='-t', cf=('communication', 'timeout'), + odesc='') # APT client tool options CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path', diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py index af1c83072..d03b37d57 100644 --- a/src/lib/Server/Admin/Perf.py +++ b/src/lib/Server/Admin/Perf.py @@ -22,6 +22,7 @@ class Perf(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'server': Bcfg2.Options.SERVER_LOCATION, 'user': Bcfg2.Options.CLIENT_USER, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -30,7 +31,8 @@ class Perf(Bcfg2.Server.Admin.Mode): setup['password'], key=setup['key'], cert=setup['certificate'], - ca=setup['ca']) + ca=setup['ca'], + timeout=setup['timeout']) data = proxy.get_statistics() for key, value in list(data.items()): data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]]) diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py index fd5794f88..2cb085346 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Server/Admin/Xcmd.py @@ -20,7 +20,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'key': Bcfg2.Options.SERVER_KEY, 'certificate': Bcfg2.Options.CLIENT_CERT, - 'ca': Bcfg2.Options.CLIENT_CA + 'ca': Bcfg2.Options.CLIENT_CA, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -31,7 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): key=setup['key'], cert=setup['certificate'], ca=setup['ca'], - timeout=180) + timeout=setup['timeout']) if len(setup['args']) == 0: print("Usage: xcmd ") return diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index eb41af640..534ab8238 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -75,6 +75,7 @@ class Client: 'certificate': Bcfg2.Options.CLIENT_CERT, 'ca': Bcfg2.Options.CLIENT_CA, 'serverCN': Bcfg2.Options.CLIENT_SCNS, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } self.setup = Bcfg2.Options.OptionParser(optinfo) @@ -178,10 +179,11 @@ class Client: proxy = Bcfg2.Proxy.ComponentProxy(self.setup['server'], self.setup['user'], self.setup['password'], - key = self.setup['key'], - cert = self.setup['certificate'], - ca = self.setup['ca'], - allowedServerCNs = self.setup['serverCN']) + key=self.setup['key'], + cert=self.setup['certificate'], + ca=self.setup['ca'], + allowedServerCNs=self.setup['serverCN'], + timeout=self.setup['timeout']) if self.setup['profile']: try: -- cgit v1.2.3-1-g7c22 From ab956d76bd20b458195ade8efd2cafb590bb3217 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 1 Jun 2011 16:01:15 -0400 Subject: guarantee that timeout is a float --- src/lib/Proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index ed33316af..9656b166c 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -332,5 +332,5 @@ def ComponentProxy(url, user=None, password=None, else: newurl = url ssl_trans = XMLRPCTransport(key, cert, ca, - allowedServerCNs, timeout=timeout) + allowedServerCNs, timeout=float(timeout)) return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) -- cgit v1.2.3-1-g7c22 From 5da2ced6d355a61032e254356c88804c7a44ffc1 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 3 Jun 2011 13:28:46 -0500 Subject: bcfg2-server: Add the ability to listen on specific interfaces (#1013) Signed-off-by: Sol Jerome --- man/bcfg2.conf.5 | 6 ++++++ src/lib/Component.py | 13 +++++++++---- src/lib/Options.py | 19 +++++++++++++++++++ src/lib/SSLServer.py | 25 +++++++++++++++++-------- src/sbin/bcfg2-server | 2 ++ 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index f2e47b7ac..b1acce7c3 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -37,6 +37,12 @@ using the 'bcfg2-admin init' command. The file monitor used to watch for changes in the repository. Values of 'gamin', 'fam', or 'pseudo' are valid. +.TP +.B listen_all +This setting tells the server to listen on all available interfaces. The +default is to only listen on those interfaces specified by the bcfg2 +setting in the components section of bcfg2.conf. + .TP .B plugins A comma-delimited list of enabled server plugins. Currently available diff --git a/src/lib/Component.py b/src/lib/Component.py index 88dce906e..b73098d09 100644 --- a/src/lib/Component.py +++ b/src/lib/Component.py @@ -23,8 +23,8 @@ logger = logging.getLogger() class NoExposedMethod (Exception): """There is no method exposed with the given name.""" -def run_component(component_cls, location, daemon, pidfile_name, to_file, - cfile, argv=None, register=True, +def run_component(component_cls, listen_all, location, daemon, pidfile_name, + to_file, cfile, argv=None, register=True, state_name=False, cls_kwargs={}, extra_getopt='', time_out=10, protocol='xmlrpc/ssl', certfile=None, keyfile=None, ca=None): @@ -64,8 +64,13 @@ def run_component(component_cls, location, daemon, pidfile_name, to_file, port = tuple(up[1].split(':')) port = (port[0], int(port[1])) try: - server = XMLRPCServer(port, keyfile=keyfile, certfile=certfile, - register=register, timeout=time_out, ca=ca, + server = XMLRPCServer(listen_all, + port, + keyfile=keyfile, + certfile=certfile, + register=register, + timeout=time_out, + ca=ca, protocol=protocol) except: logger.error("Server startup failed") diff --git a/src/lib/Options.py b/src/lib/Options.py index e6eebb808..9980566fb 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -173,6 +173,18 @@ def colon_split(c_string): return c_string.split(':') return [] +def get_bool(s): + # these values copied from ConfigParser.RawConfigParser.getboolean + # with the addition of True and False + truelist = ["1", "yes", "True", "true", "on"] + falselist = ["0", "no", "False", "false", "off"] + if s in truelist: + return True + elif s in falselist: + return False + else: + raise ValueError + # General options CFILE = Option('Specify configuration file', DEFAULT_CONFIG_LOCATION, cmd='-C', odesc='') @@ -249,6 +261,13 @@ SERVER_MCONNECT = Option('Server Metadata Connector list', cook=list_split, cf=('server', 'connectors'), default=['Probes'], ) SERVER_FILEMONITOR = Option('Server file monitor', cf=('server', 'filemonitor'), default='default', odesc='File monitoring driver') +SERVER_LISTEN_ALL = Option('Listen on all interfaces', + cf=('server', 'listen_all'), + cmd='--listen-all', + default=False, + long_arg=True, + cook=get_bool, + odesc='True|False') SERVER_LOCATION = Option('Server Location', cf=('components', 'bcfg2'), default='https://localhost:6789', cmd='-S', odesc='https://server:port') diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index a89beabbb..8cac8a53f 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -79,9 +79,9 @@ class SSLServer (SocketServer.TCPServer, object): allow_reuse_address = True logger = logging.getLogger("Cobalt.Server.TCPServer") - def __init__(self, server_address, RequestHandlerClass, keyfile=None, - certfile=None, reqCert=False, ca=None, timeout=None, - protocol='xmlrpc/ssl'): + def __init__(self, listen_all, server_address, RequestHandlerClass, + keyfile=None, certfile=None, reqCert=False, ca=None, + timeout=None, protocol='xmlrpc/ssl'): """Initialize the SSL-TCP server. @@ -97,9 +97,12 @@ class SSLServer (SocketServer.TCPServer, object): """ - all_iface_address = ('', server_address[1]) + if listen_all: + listen_address = ('', server_address[1]) + else: + listen_address = (server_address[0], server_address[1]) try: - SocketServer.TCPServer.__init__(self, all_iface_address, + SocketServer.TCPServer.__init__(self, listen_address, RequestHandlerClass) except socket.error: self.logger.error("Failed to bind to socket") @@ -310,7 +313,7 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer, """ - def __init__(self, server_address, RequestHandlerClass=None, + def __init__(self, listen_all, server_address, RequestHandlerClass=None, keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', timeout=10, logRequests=False, @@ -339,8 +342,14 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer, """A subclassed request handler to prevent class-attribute conflicts.""" SSLServer.__init__(self, - server_address, RequestHandlerClass, ca=ca, - timeout=timeout, keyfile=keyfile, certfile=certfile, protocol=protocol) + listen_all, + server_address, + RequestHandlerClass, + ca=ca, + timeout=timeout, + keyfile=keyfile, + certfile=certfile, + protocol=protocol) self.logRequests = logRequests self.serve = False self.register = register diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index f4bd5e5b7..546d5a249 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -35,6 +35,7 @@ if __name__ == '__main__': OPTINFO.update({'key' : Bcfg2.Options.SERVER_KEY, 'cert' : Bcfg2.Options.SERVER_CERT, 'ca' : Bcfg2.Options.SERVER_CA, + 'listen_all' : Bcfg2.Options.SERVER_LISTEN_ALL, 'location' : Bcfg2.Options.SERVER_LOCATION, 'passwd' : Bcfg2.Options.SERVER_PASSWORD, 'static' : Bcfg2.Options.SERVER_STATIC, @@ -51,6 +52,7 @@ if __name__ == '__main__': print("Could not read %s" % setup['configfile']) sys.exit(1) Bcfg2.Component.run_component(Bcfg2.Server.Core.Core, + listen_all=setup['listen_all'], location=setup['location'], daemon = setup['daemon'], pidfile_name = setup['daemon'], -- cgit v1.2.3-1-g7c22 From 37960c3d76a04017120a91da604bb911f3e9823d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Fri, 3 Jun 2011 23:22:40 +0200 Subject: Allow Bound entries and Client tags in Base Schema Bound entries and Client tags may also be used in Base/*.xml files, so the XML Schema for Base files must permit these entries. --- schemas/base.xsd | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/schemas/base.xsd b/schemas/base.xsd index 8eff0c69f..e8d677737 100644 --- a/schemas/base.xsd +++ b/schemas/base.xsd @@ -10,26 +10,30 @@ + - - + + + + - + + + + + + + - - - - - - + -- cgit v1.2.3-1-g7c22 From 368b34bcdd509d24bcc82ccbdbd056455bba7c37 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 3 Jun 2011 19:10:18 -0500 Subject: Base: Deprecate Base in favor of Bundler Signed-off-by: Sol Jerome --- doc/server/plugins/structures/base.txt | 7 +++++++ src/lib/Server/Plugins/Base.py | 1 + 2 files changed, 8 insertions(+) diff --git a/doc/server/plugins/structures/base.txt b/doc/server/plugins/structures/base.txt index 66d8062e1..03eae0573 100644 --- a/doc/server/plugins/structures/base.txt +++ b/doc/server/plugins/structures/base.txt @@ -6,6 +6,13 @@ Base ==== +.. deprecated:: 1.2.0 + +.. warning:: + + The Base plugin no longer receives new features/functionality. + Please use :ref:`server-plugins-structures-bundler-index` instead. + The Base plugin is a structure plugin that provides the ability to add lists of unrelated entries into client configuration entry inventories. diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 5e7d89727..5de57a87c 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = Bcfg2.Server.Plugin.StructFile + deprecated = True """Base creates independent clauses based on client metadata.""" def __init__(self, core, datastore): -- cgit v1.2.3-1-g7c22 From 80fb796b9a02403e5e90e7bde73fb2e44b3f5222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Sat, 4 Jun 2011 23:52:02 +0200 Subject: Document the "decision" option in bcfg2.conf(5) Add documentation for the "decision" option to the bcfg2.conf(5) man page. Also, note that it can be overridden using "-l none" on the bcfg2(1) command line. --- man/bcfg2.1 | 10 ++++++---- man/bcfg2.conf.5 | 4 ++++ src/lib/Options.py | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/man/bcfg2.1 b/man/bcfg2.1 index 7f108ed8f..49fd1e208 100644 --- a/man/bcfg2.1 +++ b/man/bcfg2.1 @@ -85,10 +85,12 @@ debian toolset; it calls apt\-get update and clean and dpkg \-\-configure \-\-pending. .TP -.BR "\-l " -Run the client in the server decision list mode. This approach is needed -when particular changes are deemed "high risk". It gives the ability to -centrally specify these changes, but only install them on clients when +.BR "\-l " +Run the client in the server decision list mode (unless "none" is +specified, which can be done in order to override the decision list mode +specified in bcfg2.conf). This approach is needed when particular +changes are deemed "high risk". It gives the ability to centrally +specify these changes, but only install them on clients when administrator supervision is available. Because collaborative configuration is one of the remaining hard issues in configuration management, these issues typically crop up in environments with several diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index b1acce7c3..44d8beb50 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -290,6 +290,10 @@ Global paranoid settings for Paths (defaults to false) These options only affect client functionality, specified in the [client] section. +.TP +.B decision +Specify the server decision list mode (whitelist or blacklist). + .TP .B drivers Specify tool driver set to use. This option can be used to explicitly diff --git a/src/lib/Options.py b/src/lib/Options.py index 9980566fb..5c0829df7 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -325,9 +325,9 @@ CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, cmd='-z') CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) -CLIENT_DLIST = Option('Run client in server decision list mode', default=False, +CLIENT_DLIST = Option('Run client in server decision list mode', default='none', cf=('client', 'decision'), - cmd='-l', odesc='') + cmd='-l', odesc='') CLIENT_FILE = Option('Configure from a file rather than querying the server', default=False, cmd='-f', odesc='') CLIENT_QUICK = Option('Disable some checksum verification', default=False, -- cgit v1.2.3-1-g7c22 From 260cd8caafdd26ccdcf1f999eaaa8ff2efde25a6 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 4 Jun 2011 17:49:01 -0500 Subject: Proxy: Catch traceback when name resolution fails (#1012) Signed-off-by: Sol Jerome --- src/lib/Proxy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index 9656b166c..e4a0f6a3d 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -193,7 +193,13 @@ class SSLHTTPConnection(httplib.HTTPConnection): ca_certs=self.ca, suppress_ragged_eofs=True, keyfile=self.key, certfile=self.cert, ssl_version=ssl_protocol_ver) - self.sock.connect((self.host, self.port)) + try: + self.sock.connect((self.host, self.port)) + except socket.gaierror: + e = sys.exc_info()[1] + self.logger.error("Unable to connect to %s:%s\n%s" % + (self.host, self.port, e.strerror)) + sys.exit(1) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] -- cgit v1.2.3-1-g7c22 From 70d480f2a77902347601596a7ab8f606a4b4290d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sun, 5 Jun 2011 14:00:56 -0500 Subject: Validate: Unsuppress errors for invalid schema paths (#1007) Signed-off-by: Sol Jerome --- src/lib/Server/Lint/Validate.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c9b5688e1..a561232b6 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -1,10 +1,12 @@ +import fnmatch import glob import lxml.etree import os -import fnmatch +from subprocess import Popen, PIPE, STDOUT +import sys + import Bcfg2.Options import Bcfg2.Server.Lint -from subprocess import Popen, PIPE, STDOUT class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that the repo validates """ @@ -47,6 +49,10 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname % schemadir)) + except IOError: + e = sys.exc_info()[1] + self.LintError("input/output error", e.message) + continue except: self.LintError("schema-failed-to-parse", "Failed to process schema %s" % -- cgit v1.2.3-1-g7c22 From ce4a4aa66d30d3c37098b37a8e5fd3fdbabed832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 00:54:55 +0200 Subject: All Genshi XML tags should be namespace-qualified Our XML Schema for Genshi templates expected "choose" blocks to be declared like this: However, we prefer to namespace-qualify not only the "choose" element, but also the "when" and "otherwise" tags (for clarity, and because that's how it's done in the Genshi documentation): This commit tells XML Schema validators to expect the latter style. --- schemas/genshi.xsd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/genshi.xsd b/schemas/genshi.xsd index 853b69c0d..d87c4a6e3 100644 --- a/schemas/genshi.xsd +++ b/schemas/genshi.xsd @@ -2,7 +2,8 @@ + targetNamespace="http://genshi.edgewall.org/" + elementFormDefault="qualified"> Genshi schema -- cgit v1.2.3-1-g7c22 From b2e9d11b0dca1af6e0ce9a9f21558b35b35f5777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 16:32:53 +0200 Subject: Accept non-ASCII diffs Currently, client reports don't include diffs of files which aren't US-ASCII encoded. The client transmits such files as Base64 blobs. As we'd like to change that, this commit teaches the server to properly handle non-ASCII diffs. --- src/lib/Server/Admin/Reports.py | 6 ++++++ src/lib/Server/Admin/__init__.py | 5 +++-- src/lib/Server/Plugins/Cfg.py | 7 ++++++- src/lib/Server/Plugins/DBStats.py | 1 + src/lib/Server/Reports/importscript.py | 18 +++++++++++++++--- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 942477a49..c9f3d3f58 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode): except (IOError, XMLSyntaxError): self.errExit("StatReports: Failed to parse %s" % (stats_file)) + try: + encoding = self.cfp.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientspath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: load_stats(clientsdata, statsdata, + encoding, verb, self.log, quick=quick, diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index 8915492a3..b34d7108c 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -113,7 +113,8 @@ class MetadataCore(Mode): def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None): Mode.__init__(self, configfile) options = {'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'configfile': Bcfg2.Options.CFILE} + 'configfile': Bcfg2.Options.CFILE, + 'encoding': Bcfg2.Options.ENCODING} setup = Bcfg2.Options.OptionParser(options) setup.hm = usage setup.parse(sys.argv[1:]) @@ -126,7 +127,7 @@ class MetadataCore(Mode): try: self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), setup['plugins'], - 'foo', 'UTF-8') + 'foo', setup['encoding']) except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed because %s" % msg) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 5e3cca847..c08e8c4b6 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -197,7 +197,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(new_entry['text']) + try: + etext = new_entry['text'].encode(self.encoding) + except: + logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) + raise Bcfg2.Server.Plugin.PluginExecutionError + open(name, 'w').write(etext) if log: logger.info("Wrote file %s" % name) badattr = [attr for attr in ['owner', 'group', 'perms'] diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py index 5ef1920e1..103fb7353 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Server/Plugins/DBStats.py @@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, try: Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, container, + self.core.encoding, 0, logger, True, diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index b6a3c2599..68774cec6 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -38,7 +38,7 @@ import platform from Bcfg2.Bcfg2Py3k import ConfigParser -def build_reason_kwargs(r_ent): +def build_reason_kwargs(r_ent, encoding, logger): binary_file = False if r_ent.get('current_bfile', False): binary_file = True @@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent): rc_diff = r_ent.get('current_diff') else: rc_diff = '' + if not binary_file: + try: + rc_diff = rc_diff.decode(encoding) + except: + logger.error("Reason isn't %s encoded, cannot decode it" % encoding) + rc_diff = '' return dict(owner=r_ent.get('owner', default=""), current_owner=r_ent.get('current_owner', default=""), group=r_ent.get('group', default=""), @@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent): is_binary=binary_file) -def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): +def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): clients = {} [clients.__setitem__(c.name, c) \ for c in Client.objects.all()] @@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): for (xpath, type) in pattern: for x in statistics.findall(xpath): counter_fields[type] = counter_fields[type] + 1 - kargs = build_reason_kwargs(x) + kargs = build_reason_kwargs(x, encoding, logger) try: rr = None @@ -270,6 +276,11 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s" % (statpath)) raise SystemExit(1) + try: + encoding = cf.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -288,6 +299,7 @@ if __name__ == '__main__': update_database() load_stats(clientsdata, statsdata, + encoding, verb, logger, quick=q, -- cgit v1.2.3-1-g7c22 From c55c7f92d4971eedd2e7eafa18e0cac424b637db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 17:06:16 +0200 Subject: Create non-ASCII diffs The client now also transmits diffs of files which include non-US-ASCII data (using the encoding setting from Options.py), unless they look like binary files. In the past, non-ASCII files were transmitted as Base64 blobs. In addition, "bcfg2 -I" no longer refuses to display non-ASCII diffs. Resolves ticket #999. --- src/lib/Client/Frame.py | 3 ++- src/lib/Client/Tools/POSIX.py | 28 +++++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index 60d158eb1..57844ab19 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -5,6 +5,7 @@ installs entries, and generates statistics. __revision__ = '$Revision$' import logging +import sys import time import Bcfg2.Client.Tools @@ -29,7 +30,7 @@ def promptFilter(prompt, entries): try: # py3k compatibility try: - ans = raw_input(iprompt) + ans = raw_input(iprompt.encode(sys.stdout.encoding, 'replace')) except NameError: ans = input(iprompt) if ans in ['y', 'Y']: diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 875db5ea7..862e0bc04 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -76,6 +76,21 @@ def normUid(entry): return False +def isString(strng, encoding): + """ + Returns true if the string contains no ASCII control characters + and can be decoded from the specified encoding. + """ + for char in strng: + if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: + return False + try: + strng.decode(encoding) + return True + except: + return False + + class POSIX(Bcfg2.Client.Tools.Tool): """POSIX File support code.""" name = 'POSIX' @@ -458,12 +473,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): # md5sum so it would be faster for big binary files contentStatus = content == tempdata if not contentStatus: - try: - content.decode('ascii') - isstring = True - except: - isstring = False - if tbin or not isstring: + if tbin or not isString(content, self.setup['encoding']): entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') nqtext += '\nBinary file, no printable diff' @@ -493,15 +503,15 @@ class POSIX(Bcfg2.Client.Tools.Tool): difflib.unified_diff(content.split('\n'), \ tempdata.split('\n'))]) try: - eudiff = udiff.encode('ascii') + dudiff = udiff.decode(self.setup['encoding']) except: - eudiff = "Binary file: no diff printed" + dudiff = "Binary file: no diff printed" nqtext = entry.get('qtext', '') if nqtext: nqtext += '\n' - nqtext += eudiff + nqtext += dudiff else: entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') -- cgit v1.2.3-1-g7c22 From 129aa6d584765fba5b0ff686d5d8da62b5ae405c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 6 Jun 2011 12:36:56 -0500 Subject: Validate: Create and use new error type for missing schemas Signed-off-by: Sol Jerome --- doc/help/troubleshooting.txt | 110 +++++++++++++++++++++++----------------- src/lib/Server/Lint/Validate.py | 2 +- src/lib/Server/Lint/__init__.py | 3 +- 3 files changed, 67 insertions(+), 48 deletions(-) diff --git a/doc/help/troubleshooting.txt b/doc/help/troubleshooting.txt index ecf53527c..5fe1500f4 100644 --- a/doc/help/troubleshooting.txt +++ b/doc/help/troubleshooting.txt @@ -81,121 +81,138 @@ Type `help` in bcfg2-info for more information. Error Messages ============== -This page describes error messages produced by Bcfg2 and steps that can +The tables below describe error messages produced by Bcfg2 and steps that can be taken to remedy them. +Client Errors +------------- + +------------------------------+----------+---------------------+--------------+ | Error | Location | Meaning | Repair | +==============================+==========+=====================+==============+ -| Incomplete information for | Client | The described entry | [1]_ | +| Incomplete information for | Client | The described entry | [c1]_ | | entry : | | is not fully | | | cannot verify | | specified by the | | | | | server, so no | | | | | verification can be | | | | | performed. | | +------------------------------+----------+---------------------+--------------+ -| Incomplete information for | Client | The described entry | [1]_ | +| Incomplete information for | Client | The described entry | [c1]_ | | entry : | | is not fully | | | cannot install | | specified by the | | | | | server, so no | | | | | verification can be | | | | | performed. | | +------------------------------+----------+---------------------+--------------+ -| The following entries are | Client | The client cannot | [2]_ | +| The following entries are | Client | The client cannot | [c2]_ | | not handled by any tool: | | figure out how to | | | : | | handle this entry. | | +------------------------------+----------+---------------------+--------------+ -| No ca is specified. Cannot | Client | The client is | [3]_ | +| No ca is specified. Cannot | Client | The client is | [c3]_ | | authenticate the server with | | unable to verify | | | SSL. | | the server | | +------------------------------+----------+---------------------+--------------+ -| GID normalization failed for | Client | The client is | [4]_ | +| GID normalization failed for | Client | The client is | [c4]_ | | FILENAME. Does group GROUP | | unable to convert | | | exist? | | the group GROUP to | | | | | a usable GID | | +------------------------------+----------+---------------------+--------------+ -| UID normalization failed for | Client | The client is | [5]_ | +| UID normalization failed for | Client | The client is | [c5]_ | | FILENAME. Does owner OWNER | | unable to convert | | | exist? | | the owner OWNER to | | | | | a usable UID | | +------------------------------+----------+---------------------+--------------+ -| Failed to bind entry: | Server | The server was | [6]_ | + + +.. [c1] This entry is not being bound. Ensure that a version of this + entry applies to this client. +.. [c2] It is possible that the type attribute for this generator entry + is undefined. You may need to add the appropriate type attribute + in the literal entry specification. +.. [c3] Copy the Bcfg2 server's CA certificate to the client and specify it + using the **ca** option in the [communication] section of + ``bcfg2.conf`` +.. [c4] If the group doesn't exist, you need to specify the correct one + in an :ref:`info.xml ` file or set the default group + appropriately. +.. [c5] If the owner doesn't exist, you need to specify the correct one + in an :ref:`info.xml ` file or set the default owner + appropriately. + +Server Errors +------------- + ++------------------------------+----------+---------------------+--------------+ +| Error | Location | Meaning | Repair | ++==============================+==========+=====================+==============+ +| Failed to bind entry: | Server | The server was | [s1]_ | | | | unable to find a | | | | | suitable version of | | | | | entry for client. | | +------------------------------+----------+---------------------+--------------+ -| Failed to bind to socket | Server | The server was | [7]_ | +| Failed to bind to socket | Server | The server was | [s2]_ | | | | unable to bind to | | | | | the tcp server | | | | | socket. | | +------------------------------+----------+---------------------+--------------+ -| Failed to load | Server | The server was | [8]_ | +| Failed to load | Server | The server was | [s3]_ | | ssl key | | unable to read and | | | | | process the ssl key.| | +------------------------------+----------+---------------------+--------------+ -| Failed to read file | Server | The server failed | [9]_ | +| Failed to read file | Server | The server failed | [s4]_ | | | | to read the | | | | | specified file | | +------------------------------+----------+---------------------+--------------+ -| Failed to parse file | Server | The server failed | [10]_ | +| Failed to parse file | Server | The server failed | [s5]_ | | | | to parse the | | | | | specified XML file | | +------------------------------+----------+---------------------+--------------+ -| Client metadata resolution | Server | The server cannot | [11]_ | +| Client metadata resolution | Server | The server cannot | [s6]_ | | error for | | resolve the client | | | | | hostname or the | | | | | client is | | | | | associated with a | | | | | non-profile group. | | +------------------------------+----------+---------------------+--------------+ -| Failed to decode | Server | The encoding being | [12]_ | +| Failed to decode | Server | The encoding being | [s7]_ | | Please verify you are using | | used is unable to | | | the proper encoding | | decode the | | | | | character present | | | | | in this file. | | +------------------------------+----------+---------------------+--------------+ -| Got unknown entries | Server | The Packages plugin | [13]_ | +| Got unknown entries | Server | The Packages plugin | [s8]_ | | [list of unknown entries] | | has no knowledge of | | | | | the listed entries | | +------------------------------+----------+---------------------+--------------+ -| Failed to import lxml | Server | The server cannot | [14]_ | +| Failed to import lxml | Server | The server cannot | [s9]_ | | dependency. Shutting | | import lxml | | | down server. | | | | +------------------------------+----------+---------------------+--------------+ -| You need to specify base64 | Server | The server cannot | [15]_ | +| You need to specify base64 | Server | The server cannot | [s10]_ | | encoding for | | send the file as | | | | | ascii text | | +------------------------------+----------+---------------------+--------------+ +| ERROR: Error reading file | Server | The server cannot | [s11]_ | +| '/path/to/schema': failed to | | find the schema | | +| load external entity | | file | | +| "/path/to/schema" | | | | ++------------------------------+----------+---------------------+--------------+ -.. [1] This entry is not being bound. Ensure that a version of this - entry applies to this client. -.. [2] It is possible that the type attribute for this generator entry - is undefined. You may need to add the appropriate type attribute - in the literal entry specification. -.. [3] Copy the Bcfg2 server's CA certificate to the client and specify it - using the **ca** option in the [communication] section of - ``bcfg2.conf`` -.. [4] If the group doesn't exist, you need to specify the correct one - in an :ref:`info.xml ` file or set the default group - appropriately. -.. [5] If the owner doesn't exist, you need to specify the correct one - in an :ref:`info.xml ` file or set the default owner - appropriately. -.. [6] This entry is not being bound. Ensure that a version of this - entry applies to this client. -.. [7] Ensure that another instance of the daemon (or any other process) - is not listening on the same port. -.. [8] Ensure that the key is readable by the user running the daemon - and that it is well-formed. -.. [9] Ensure that this file still exists; a frequent cause is the - deletion of a temp file. -.. [10] Ensure that the file is properly formed XML. -.. [11] Fix hostname resolution for the client or ensure that the profile +.. [s1] This entry is not being bound. Ensure that a version of this + entry applies to this client. +.. [s2] Ensure that another instance of the daemon (or any other process) + is not listening on the same port. +.. [s3] Ensure that the key is readable by the user running the daemon + and that it is well-formed. +.. [s4] Ensure that this file still exists; a frequent cause is the + deletion of a temp file. +.. [s5] Ensure that the file is properly formed XML. +.. [s6] Fix hostname resolution for the client or ensure that the profile group is properly setup. -.. [12] Ensure the correct encoding is specified in the [components] +.. [s7] Ensure the correct encoding is specified in the [components] section of ``bcfg2.conf``. -.. [13] For packages listed other than **gpg-pubkey**, this error means +.. [s8] For packages listed other than **gpg-pubkey**, this error means that the Packages plugin is unable to find the package in any of the sources listed in ``Packages/config.xml``. The issue often arises when the client is not in one of the groups necessary for @@ -203,10 +220,11 @@ be taken to remedy them. ignore the message as the Packages plugin has no knowledge of these packages (however, note that this package is most often specified as a BoundPackage entry). -.. [14] Ensure that you have installed all the necessary +.. [s9] Ensure that you have installed all the necessary :ref:`installation-prerequisites`. -.. [15] You likely need to specify a base64 encoding using an - :ref:`server-info` file for this entry. +.. [s10] You likely need to specify a base64 encoding using an + :ref:`server-info` file for this entry. +.. [s11] Verify that you have the proper prefix set in bcfg2.conf. FAQs ==== diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index a561232b6..8a8406e73 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -51,7 +51,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): schemadir)) except IOError: e = sys.exc_info()[1] - self.LintError("input/output error", e.message) + self.LintError("input-output-error", e.message) continue except: self.LintError("schema-failed-to-parse", diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 013cbf2ba..63cb2463b 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -88,7 +88,8 @@ class ErrorHandler (object): "xml-failed-to-read":"error", "xml-failed-to-verify":"error", "merge-cfg":"warning", - "merge-probes":"warning",} + "merge-probes":"warning", + "input-output-error": "error"} def __init__(self, config=None): self.errors = 0 -- cgit v1.2.3-1-g7c22 From 00888897e4a07f9f83b1c890f70382609c856199 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 6 Jun 2011 18:30:46 -0500 Subject: =?UTF-8?q?Options:=20Set=20default=20encoding=20to=20UTF-8=20(as?= =?UTF-8?q?=20per=20Holger=20Wei=C3=9F's=20suggestion)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sol Jerome --- man/bcfg2.conf.5 | 4 +--- src/lib/Options.py | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 44d8beb50..786f69f9a 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -410,9 +410,7 @@ eg: bcfg2 = https://10.3.1.6:6789 .TP .B encoding -Text encoding of configuration files. Defaults to the system default -encoding. - +Text encoding of configuration files. Defaults to UTF-8. .SH LOGGING OPTIONS Specified in the [logging] section. These options control the server diff --git a/src/lib/Options.py b/src/lib/Options.py index 5c0829df7..0d978c519 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -203,8 +203,10 @@ SENDMAIL_PATH = Option('Path to sendmail', cf=('reports', 'sendmailpath'), default='/usr/lib/sendmail') INTERACTIVE = Option('Prompt the user for each change', default=False, cmd='-I', ) -ENCODING = Option('Encoding of cfg files', default=sys.getdefaultencoding(), - cmd='-E', odesc='', +ENCODING = Option('Encoding of cfg files', + default='UTF-8', + cmd='-E', + odesc='', cf=('components', 'encoding')) PARANOID_PATH = Option('Specify path for paranoid file backups', default='/var/cache/bcfg2', cf=('paranoid', 'path'), -- cgit v1.2.3-1-g7c22 From fdda835c1c6181589f1eb5fa889056ac26a09347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Tue, 7 Jun 2011 13:22:00 +0200 Subject: bcfg2-reports: Show modified entries bcfg2-reports now shows modified entries if it's called with --modifiedentry, -m, or -s. --- doc/reports/dynamic.txt | 52 +++++++++++++++------------ man/bcfg2-reports.8 | 16 +++++++-- src/sbin/bcfg2-reports | 95 +++++++++++++++++++++++++++++++++++-------------- 3 files changed, 112 insertions(+), 51 deletions(-) diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 6e0ea8050..ecab980c6 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -188,29 +188,35 @@ displays:: Usage: python bcfg2-reports [option] ... Options and arguments (and corresponding environment variables): - -a : shows all hosts, including expired hosts - -b NAME : single-host mode - shows bad entries from the - current interaction of NAME - -c : shows only clean hosts - -d : shows only dirty hosts - -e NAME : single-host mode - shows extra entries from the - current interaction of NAME - -h : shows help and usage info about bcfg2-reports - -s NAME : single-host mode - shows bad and extra entries from - the current interaction of NAME - -x NAME : toggles expired/unexpired state of NAME - --badentry=KIND,NAME : shows only hosts whose current interaction has bad - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 - --extraentry=KIND,NAME : shows only hosts whose current interaction has extra - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 - --fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... - (name,time,state) - --sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) - --stale : shows hosts which haven't run in the last 24 hours + -a : shows all hosts, including expired hosts + -b NAME : single-host mode - shows bad entries from the + current interaction of NAME + -c : shows only clean hosts + -d : shows only dirty hosts + -e NAME : single-host mode - shows extra entries from the + current interaction of NAME + -h : shows help and usage info about bcfg2-reports + -m NAME : single-host mode - shows modified entries from the + current interaction of NAME + -s NAME : single-host mode - shows bad, modified, and extra + entries from the current interaction of NAME + -x NAME : toggles expired/unexpired state of NAME + --badentry=KIND,NAME : shows only hosts whose current interaction has bad + entries in of KIND kind and NAME name; if a single + argument ARG1 is given, then KIND,NAME pairs will be + read from a file of name ARG1 + --modifiedentry=KIND,NAME : shows only hosts whose current interaction has + modified entries in of KIND kind and NAME name; if a + single argument ARG1 is given, then KIND,NAME pairs + will be read from a file of name ARG1 + --extraentry=KIND,NAME : shows only hosts whose current interaction has extra + entries in of KIND kind and NAME name; if a single + argument ARG1 is given, then KIND,NAME pairs will be + read from a file of name ARG1 + --fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... + (name,time,state) + --sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) + --stale : shows hosts which haven't run in the last 24 hours Screenshots =========== diff --git a/man/bcfg2-reports.8 b/man/bcfg2-reports.8 index bc4c9344b..284298a69 100644 --- a/man/bcfg2-reports.8 +++ b/man/bcfg2-reports.8 @@ -39,10 +39,15 @@ of NAME. NAME is the name of the entry. .RS Shows help and usage info about bcfg2-reports. .RE +.B "\-m NAME" +.RS +Single-host mode \- shows modified entries from the current interaction +of NAME. NAME is the name of the entry. +.RE .B "\-s NAME" .RS -Single host mode \- shows bad and extra entries from the current -interaction of NAME. NAME is the name of the entry. +Single host mode \- shows bad, modified, and extra entries from the +current interaction of NAME. NAME is the name of the entry. .RE .B "\-x NAME" .RS @@ -66,6 +71,13 @@ pairs will be read from a file of name ARG1. KIND is the type of entry .RS Only displays the fields ARG1,ARG2,... (name, time, state) .RE +.B "\-\-modifiedentry=KIND,NAME" +.RS +Shows only hosts whose current interaction has modified entries in of +KIND kind and NAME name; if a single argument ARG1 is given, then +KIND,NAME pairs will be read from a file of name ARG1. KIND is the type +of entry (Package, Path, Service, etc). NAME is the name of the entry. +.RE .B "\-\-sort=ARG1,ARG2,..." .RS Sorts output on ARG1,ARG2,... (name, time, state) diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 33a291395..b5609db9f 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -104,6 +104,7 @@ def print_entry(item, max_name): fields = "" sort = "" badentry = "" +modifiedentry = "" extraentry = "" expire = "" singlehost = "" @@ -114,8 +115,8 @@ result = list() entrydict = dict() args = sys.argv[1:] -opts, pargs = getopt(args, 'ab:cde:hs:x:', - ['stale', 'sort=', 'fields=', 'badentry=', 'extraentry=']) +opts, pargs = getopt(args, 'ab:cde:hm:s:x:', + ['stale', 'sort=', 'fields=', 'badentry=', 'modifiedentry=', 'extraentry=']) for option in opts: if len(option) > 0: @@ -125,11 +126,13 @@ for option in opts: sort = option[1] if option[0] == '--badentry': badentry = option[1] + if option[0] == '--modifiedentry': + modifiedentry = option[1] if option[0] == '--extraentry': extraentry = option[1] if option[0] == '-x': expire = option[1] - if option[0] == '-s' or option[0] == '-b' or option[0] == '-e': + if option[0] == '-s' or option[0] == '-b' or option[0] == '-m' or option[0] == '-e': singlehost = option[1] if expire != "": @@ -147,29 +150,35 @@ elif '-h' in args: print("""Usage: bcfg2-reports [option] ... Options and arguments (and corresponding environment variables): --a : shows all hosts, including expired hosts --b NAME : single-host mode - shows bad entries from the - current interaction of NAME --c : shows only clean hosts --d : shows only dirty hosts --e NAME : single-host mode - shows extra entries from the - current interaction of NAME --h : shows help and usage info about bcfg2-reports --s NAME : single-host mode - shows bad and extra entries from - the current interaction of NAME --x NAME : toggles expired/unexpired state of NAME ---badentry=KIND,NAME : shows only hosts whose current interaction has bad - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 ---extraentry=KIND,NAME : shows only hosts whose current interaction has extra - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 ---fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... - (name,time,state) ---sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) ---stale : shows hosts which haven't run in the last 24 hours +-a : shows all hosts, including expired hosts +-b NAME : single-host mode - shows bad entries from the + current interaction of NAME +-c : shows only clean hosts +-d : shows only dirty hosts +-e NAME : single-host mode - shows extra entries from the + current interaction of NAME +-h : shows help and usage info about bcfg2-reports +-m NAME : single-host mode - shows modified entries from the + current interaction of NAME +-s NAME : single-host mode - shows bad, modified, and extra + entries from the current interaction of NAME +-x NAME : toggles expired/unexpired state of NAME +--badentry=KIND,NAME : shows only hosts whose current interaction has bad + entries in of KIND kind and NAME name; if a single + argument ARG1 is given, then KIND,NAME pairs will be + read from a file of name ARG1 +--modifiedentry=KIND,NAME : shows only hosts whose current interaction has + modified entries in of KIND kind and NAME name; if a + single argument ARG1 is given, then KIND,NAME pairs + will be read from a file of name ARG1 +--extraentry=KIND,NAME : shows only hosts whose current interaction has extra + entries in of KIND kind and NAME name; if a single + argument ARG1 is given, then KIND,NAME pairs will be + read from a file of name ARG1 +--fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... + (name,time,state) +--sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) +--stale : shows hosts which haven't run in the last 24 hours """) elif singlehost != "": for c_inst in c_list: @@ -183,6 +192,15 @@ elif singlehost != "": max_name = len(item.entry.name) for item in baditems: print_entry(item, max_name) + modifieditems = c_inst.current_interaction.modified() + if len(modifieditems) > 0 and ('-m' in args or '-s' in args): + print "Modified Entries:" + max_name = -1 + for item in modifieditems: + if len(item.entry.name) > max_name: + max_name = len(item.entry.name) + for item in modifieditems: + print_entry(item, max_name) extraitems = c_inst.current_interaction.extra() if len(extraitems) > 0 and ('-e' in args or '-s' in args): print("Extra Entries:") @@ -206,6 +224,9 @@ else: if badentry != "": badentry = badentry.split(',') + if modifiedentry != "": + modifiedentry = modifiedentry.split(',') + if extraentry != "": extraentry = extraentry.split(',') @@ -247,6 +268,28 @@ else: if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: result.append(c_inst) break + elif modifiedentry != "": + if len(modifiedentry) == 1: + fileread = fileinput.input(modifiedentry[0]) + for line in fileread: + modifiedentry = line.strip().split(',') + for c_inst in c_list: + modifieditems = c_inst.current_interaction.modified() + for item in modifieditems: + if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]: + result.append(c_inst) + if c_inst in entrydict: + entrydict.get(c_inst).append(modifiedentry[1]) + else: + entrydict[c_inst] = [modifiedentry[1]] + break + else: + for c_inst in c_list: + modifieditems = c_inst.current_interaction.modified() + for item in modifieditems: + if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]: + result.append(c_inst) + break elif extraentry != "": if len(extraentry) == 1: fileread = fileinput.input(extraentry[0]) -- cgit v1.2.3-1-g7c22 From 80929a6e73eeece44997b84f2beeb38eb3d7645d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Wed, 8 Jun 2011 17:16:44 +0200 Subject: bcfg2-reports: Show total numbers of entries Add a "-t NAME" option which reports the total (and good) number of managed entries on the host NAME. Also, allow for specifying "total", "good", and "bad" fields via --fields and --sort. --- doc/reports/dynamic.txt | 7 +++++-- man/bcfg2-reports.8 | 5 +++-- src/sbin/bcfg2-reports | 46 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index ecab980c6..4c75cce32 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -200,6 +200,8 @@ displays:: current interaction of NAME -s NAME : single-host mode - shows bad, modified, and extra entries from the current interaction of NAME + -t NAME : single-host mode - shows total number of managed and + good entries from the current interaction of NAME -x NAME : toggles expired/unexpired state of NAME --badentry=KIND,NAME : shows only hosts whose current interaction has bad entries in of KIND kind and NAME name; if a single @@ -214,8 +216,9 @@ displays:: argument ARG1 is given, then KIND,NAME pairs will be read from a file of name ARG1 --fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... - (name,time,state) - --sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) + (name,time,state,total,good,bad) + --sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... + (name,time,state,total,good,bad) --stale : shows hosts which haven't run in the last 24 hours Screenshots diff --git a/man/bcfg2-reports.8 b/man/bcfg2-reports.8 index 284298a69..51399e1c9 100644 --- a/man/bcfg2-reports.8 +++ b/man/bcfg2-reports.8 @@ -69,7 +69,8 @@ pairs will be read from a file of name ARG1. KIND is the type of entry .RE .B "\-\-fields=ARG1,ARG2,..." .RS -Only displays the fields ARG1,ARG2,... (name, time, state) +Only displays the fields ARG1,ARG2,... (name, time, state, total, good, +bad) .RE .B "\-\-modifiedentry=KIND,NAME" .RS @@ -80,7 +81,7 @@ of entry (Package, Path, Service, etc). NAME is the name of the entry. .RE .B "\-\-sort=ARG1,ARG2,..." .RS -Sorts output on ARG1,ARG2,... (name, time, state) +Sorts output on ARG1,ARG2,... (name, time, state, total, good, bad) .RE .B "\-\-stale" .RS diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index b5609db9f..36283f87d 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#!/usr/bin/python """Query reporting system for client status.""" __revision__ = '$Revision$' @@ -47,6 +47,23 @@ def statecompare(client1, client2): else: return 0 +def totalcompare(client1, client2): + """Compares two clients by their total entry counts.""" + return cmp(client2.current_interaction.totalcount, \ + client1.current_interaction.totalcount) + +def goodcompare(client1, client2): + """Compares two clients by their good entry counts.""" + return cmp(client2.current_interaction.goodcount, \ + client1.current_interaction.goodcount) + +def badcompare(client1, client2): + """Compares two clients by their bad entry counts.""" + return cmp(client2.current_interaction.totalcount - \ + client2.current_interaction.goodcount, \ + client1.current_interaction.totalcount - \ + client1.current_interaction.goodcount) + def crit_compare(criterion, client1, client2): """Compares two clients by the criteria provided in criterion.""" for crit in criterion: @@ -57,6 +74,12 @@ def crit_compare(criterion, client1, client2): comp = statecompare(client1, client2) elif crit == 'time': comp = timecompare(client1, client2) + elif crit == 'total': + comp = totalcompare(client1, client2) + elif crit == 'good': + comp = goodcompare(client1, client2) + elif crit == 'bad': + comp = badcompare(client1, client2) if comp != 0: return comp @@ -83,6 +106,13 @@ def print_fields(fields, cli, max_name, entrydict): fdata.append("clean") else: fdata.append("dirty") + elif field == 'total': + fdata.append("%5d" % cli.current_interaction.totalcount) + elif field == 'good': + fdata.append("%5d" % cli.current_interaction.goodcount) + elif field == 'bad': + fdata.append("%5d" % cli.current_interaction.totalcount \ + - cli.current_interaction.goodcount) else: try: fdata.append(getattr(cli, field)) @@ -115,7 +145,7 @@ result = list() entrydict = dict() args = sys.argv[1:] -opts, pargs = getopt(args, 'ab:cde:hm:s:x:', +opts, pargs = getopt(args, 'ab:cde:hm:s:t:x:', ['stale', 'sort=', 'fields=', 'badentry=', 'modifiedentry=', 'extraentry=']) for option in opts: @@ -132,7 +162,11 @@ for option in opts: extraentry = option[1] if option[0] == '-x': expire = option[1] - if option[0] == '-s' or option[0] == '-b' or option[0] == '-m' or option[0] == '-e': + if option[0] == '-s' or \ + option[0] == '-t' or \ + option[0] == '-b' or \ + option[0] == '-m' or \ + option[0] == '-e': singlehost = option[1] if expire != "": @@ -162,6 +196,8 @@ Options and arguments (and corresponding environment variables): current interaction of NAME -s NAME : single-host mode - shows bad, modified, and extra entries from the current interaction of NAME +-t NAME : single-host mode - shows total number of managed and + good entries from the current interaction of NAME -x NAME : toggles expired/unexpired state of NAME --badentry=KIND,NAME : shows only hosts whose current interaction has bad entries in of KIND kind and NAME name; if a single @@ -183,6 +219,10 @@ Options and arguments (and corresponding environment variables): elif singlehost != "": for c_inst in c_list: if singlehost == c_inst.name: + if '-t' in args: + managed = c_inst.current_interaction.totalcount + good = c_inst.current_interaction.goodcount + print("Total managed entries: %d (good: %d)" % (managed, good)) baditems = c_inst.current_interaction.bad() if len(baditems) > 0 and ('-b' in args or '-s' in args): print("Bad Entries:") -- cgit v1.2.3-1-g7c22 From 0133621823bdc7623875d926f235607c7f71b5ff Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 8 Jun 2011 19:40:54 -0500 Subject: bcfg2-reports: Revert shebang line modification Signed-off-by: Sol Jerome --- src/sbin/bcfg2-reports | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 36283f87d..9a4c6e60d 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """Query reporting system for client status.""" __revision__ = '$Revision$' -- cgit v1.2.3-1-g7c22 From bacc03705454a8e3c5f8c670b4823927587f1963 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 09:57:50 -0500 Subject: Frame: Fix 'important' entry installation in decision mode (Reported by m4z on irc) Signed-off-by: Sol Jerome --- src/lib/Client/Frame.py | 86 ++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index 57844ab19..e33e1a7df 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -123,22 +123,7 @@ class Frame: self.logger.info("Loaded tool drivers:") self.logger.info([tool.name for tool in self.tools]) - if not self.dryrun and not self.setup['bundle']: - for cfile in [cfl for cfl in config.findall(".//Path") \ - if cfl.get('name') in self.__important__ and \ - cfl.get('type') == 'file']: - tl = [t for t in self.tools if t.handlesEntry(cfile) \ - and t.canVerify(cfile)] - if tl: - if not tl[0].VerifyPath(cfile, []): - if self.setup['interactive'] and not \ - promptFilter("Install %s: %s? (y/N):", [cfile]): - continue - try: - self.states[cfile] = tl[0].InstallPath(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) + # find entries not handled by any tools problems = [entry for struct in config for \ entry in struct if entry not in self.handled] @@ -177,6 +162,54 @@ class Frame: return self.__dict__[name] raise AttributeError(name) + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode works with it + self.whitelist = [entry for entry in self.states \ + if not self.states[entry]] + if self.setup['decision'] == 'whitelist': + dwl = self.setup['decision_list'] + w_to_rem = [e for e in self.whitelist \ + if not matches_white_list(e, dwl)] + if w_to_rem: + self.logger.info("In whitelist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) + self.whitelist = [x for x in self.whitelist \ + if x not in w_to_rem] + + elif self.setup['decision'] == 'blacklist': + b_to_rem = [e for e in self.whitelist \ + if not passes_black_list(e, self.setup['decision_list'])] + if b_to_rem: + self.logger.info("In blacklist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist if x not in b_to_rem] + + # take care of important entries first + if not self.dryrun and not self.setup['bundle']: + for cfile in [cfl for cfl in self.config.findall(".//Path") \ + if cfl.get('name') in self.__important__ and \ + cfl.get('type') == 'file']: + if cfile not in self.whitelist: + continue + tl = [t for t in self.tools if t.handlesEntry(cfile) \ + and t.canVerify(cfile)] + if tl: + if not tl[0].VerifyPath(cfile, []): + if self.setup['interactive'] and not \ + promptFilter("Install %s: %s? (y/N):", [cfile]): + continue + try: + self.states[cfile] = tl[0].InstallPath(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + def Inventory(self): """ Verify all entries, @@ -210,26 +243,6 @@ class Frame: candidates = [entry for entry in self.states \ if not self.states[entry]] - self.whitelist = [entry for entry in self.states \ - if not self.states[entry]] - # Need to process decision stuff early so that dryrun mode works with it - if self.setup['decision'] == 'whitelist': - dwl = self.setup['decision_list'] - w_to_rem = [e for e in self.whitelist \ - if not matches_white_list(e, dwl)] - if w_to_rem: - self.logger.info("In whitelist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) - self.whitelist = [x for x in self.whitelist \ - if x not in w_to_rem] - - elif self.setup['decision'] == 'blacklist': - b_to_rem = [e for e in self.whitelist \ - if not passes_black_list(e, self.setup['decision_list'])] - if b_to_rem: - self.logger.info("In blacklist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) - self.whitelist = [x for x in self.whitelist if x not in b_to_rem] if self.dryrun: if self.whitelist: @@ -389,6 +402,7 @@ class Frame: self.Inventory() self.times['inventory'] = time.time() self.CondDisplayState('initial') + self.InstallImportant() self.Decide() self.Install() self.times['install'] = time.time() -- cgit v1.2.3-1-g7c22 From 0b074d1d8dff851081a892e6a6755c36f51c189f Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 10:34:13 -0500 Subject: Init: Remove Base from default plugin list Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index fff8bcd1c..a9170530e 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,7 +77,6 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', - 'Base', 'Bundler', 'Bzr', 'Cfg', -- cgit v1.2.3-1-g7c22 From 733f3e51f2e1986005b37caeedca8edf4c464471 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 10:41:58 -0500 Subject: YUM: Make YUMng the preferred client tool on RPM systems Signed-off-by: Sol Jerome --- src/lib/Client/Tools/YUM24.py | 2 -- src/lib/Client/Tools/YUMng.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Client/Tools/YUM24.py index 04d9f5c07..5387fdd6a 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Client/Tools/YUM24.py @@ -76,8 +76,6 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): __new_gpg_ireq__ = {'Package': ['name'], 'Instance': ['version', 'release']} - conflicts = ['YUMng', 'RPMng'] - def __init__(self, logger, setup, config): Bcfg2.Client.Tools.RPMng.RPMng.__init__(self, logger, setup, config) self.__important__ = self.__important__ + \ diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index c9e7aa15e..7299ab4c0 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -141,7 +141,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): 'Path': ['type']} __ireq__ = {'Package': ['name']} - conflicts = ['RPMng'] + conflicts = ['YUMng', 'RPMng'] def __init__(self, logger, setup, config): self.yb = yum.YumBase() -- cgit v1.2.3-1-g7c22 From fa45b31482f86cc75ca58123096d08dde015f48b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 9 Jun 2011 10:47:14 -0500 Subject: YUMng: Fix typo in last commit Signed-off-by: Sol Jerome --- src/lib/Client/Tools/YUMng.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 7299ab4c0..34d463941 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -141,7 +141,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): 'Path': ['type']} __ireq__ = {'Package': ['name']} - conflicts = ['YUMng', 'RPMng'] + conflicts = ['YUM24', 'RPMng'] def __init__(self, logger, setup, config): self.yb = yum.YumBase() -- cgit v1.2.3-1-g7c22 From e4f2ac90cb173a75d7e39d643b3e8147085ecca1 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 10 Jun 2011 10:47:46 -0500 Subject: POSIX: Detect missing cache directory failures Signed-off-by: Sol Jerome --- src/lib/Client/Tools/POSIX.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 862e0bc04..64821f621 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -14,6 +14,7 @@ import os import pwd import shutil import stat +import sys import time import Bcfg2.Client.Tools import Bcfg2.Options @@ -559,8 +560,14 @@ class POSIX(Bcfg2.Client.Tools.Tool): (entry.get('current_exists', 'true') == 'false'): bkupnam = entry.get('name').replace('/', '_') # current list of backups for this file - bkuplist = [f for f in os.listdir(self.ppath) if - f.startswith(bkupnam)] + try: + bkuplist = [f for f in os.listdir(self.ppath) if + f.startswith(bkupnam)] + except OSError: + e = sys.exc_info()[1] + self.logger.error("Failed to create backup list in %s: %s" % + (self.ppath, e.strerror)) + return False bkuplist.sort() while len(bkuplist) >= int(self.max_copies): # remove the oldest backup available -- cgit v1.2.3-1-g7c22 From 36aadd161281334c53b39ae028afbb477e0228ec Mon Sep 17 00:00:00 2001 From: Jack Neely Date: Fri, 10 Jun 2011 13:40:41 -0400 Subject: Make -q turn off package verification in YUMng This make the YUMng verification behavior like the APT tool. Before hand we just turned off package checksums which only worked with very new versions of Yum. --- src/lib/Client/Tools/YUMng.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 34d463941..324307f7f 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -485,6 +485,9 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): qtext_versions.append("U(%s)" % str(POs[0])) continue + if self.setup.get('quick', False): + # Passed -q on the command line + continue if not (pkg_verify and \ inst.get('pkg_verify', 'true').lower() == 'true'): continue -- cgit v1.2.3-1-g7c22 From da0fb10e340f3fd8e97ef29810fd0f59173492e7 Mon Sep 17 00:00:00 2001 From: Jack Neely Date: Fri, 10 Jun 2011 16:45:30 -0400 Subject: Addition Yum error handling Address ticket #955 and attempt to capture the exceptions that may possibly be generated from Yum or Yum's plugins and return some sort of error message. --- src/lib/Client/Tools/YUMng.py | 50 +++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 324307f7f..0435ca0d7 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -170,7 +170,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): except yum.Errors.RepoError, e: self.logger.error("YUMng Repository error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError - except yum.Errors.YumBaseError, e: + except Exception, e: self.logger.error("YUMng error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError @@ -671,38 +671,56 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): return True def _runYumTransaction(self): + def cleanup(): + self.yb.closeRpmDB() + self.RefreshPackages() + rDisplay = RPMDisplay(self.logger) yDisplay = YumDisplay(self.logger) # Run the Yum Transaction - rescode, restring = self.yb.buildTransaction() + try: + rescode, restring = self.yb.buildTransaction() + except yum.Errors.YumBaseError, e: + self.logger.error("Yum transaction error: %s" % str(e)) + cleanup() + return + self.logger.debug("Initial Yum buildTransaction() run said:") self.logger.debug(" resultcode: %s, msgs: %s" \ % (rescode, restring)) if rescode != 1: # Transaction built successfully, run it - self.yb.processTransaction(callback=yDisplay, - rpmDisplay=rDisplay) - self.logger.info("Single Pass for Install Succeeded") + try: + self.yb.processTransaction(callback=yDisplay, + rpmDisplay=rDisplay) + self.logger.info("Single Pass for Install Succeeded") + except yum.Errors.YumBaseError, e: + self.logger.error("Yum transaction error: %s" % str(e)) + cleanup() + return else: # The yum command failed. No packages installed. # Try installing instances individually. self.logger.error("Single Pass Install of Packages Failed") skipBroken = self.yb.conf.skip_broken self.yb.conf.skip_broken = True - rescode, restring = self.yb.buildTransaction() - if rescode != 1: - self.yb.processTransaction(callback=yDisplay, - rpmDisplay=rDisplay) - self.logger.debug( - "Second pass install did not install all packages") - else: - self.logger.error("Second pass yum install failed.") - self.logger.debug(" %s" % restring) + try: + rescode, restring = self.yb.buildTransaction() + if rescode != 1: + self.yb.processTransaction(callback=yDisplay, + rpmDisplay=rDisplay) + self.logger.debug( + "Second pass install did not install all packages") + else: + self.logger.error("Second pass yum install failed.") + self.logger.debug(" %s" % restring) + except yum.Errors.YumBaseError, e: + self.logger.error("Yum transaction error: %s" % str(e)) + self.yb.conf.skip_broken = skipBroken - self.yb.closeRpmDB() - self.RefreshPackages() + cleanup() def Install(self, packages, states): """ -- cgit v1.2.3-1-g7c22 From 99a4c8ed71fd15523fc9c6120e0a2fdac38ca092 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 08:11:16 -0500 Subject: bcfg2-info: Fix print formatting Signed-off-by: Sol Jerome --- src/sbin/bcfg2-info | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index c36e1af42..07953ae69 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -375,21 +375,21 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): except: print("Client %s not defined" % client) continue - print("Hostname:\t", client_meta.hostname) - print("Profile:\t", client_meta.profile) - print("Groups:\t\t", list(client_meta.groups)[0]) + print("Hostname:\t%s" % client_meta.hostname) + print("Profile:\t%s" % client_meta.profile) + print("Groups:\t\t%s" % list(client_meta.groups)[0]) for grp in list(client_meta.groups)[1:]: - print('\t\t%s' % grp) + print("\t\t%s" % grp) if client_meta.bundles: - print("Bundles:\t", list(client_meta.bundles)[0]) + print("Bundles:\t%s" % list(client_meta.bundles)[0]) for bnd in list(client_meta.bundles)[1:]: - print('\t\t%s' % bnd) + print("\t\t%s" % bnd) if client_meta.connectors: print("Connector data") print("=" * 80) for conn in client_meta.connectors: if getattr(client_meta, conn): - print("%s:\t" % (conn), getattr(client_meta, conn)) + print("%s:\t%s" % (conn, getattr(client_meta, conn))) print("=" * 80) def do_mappings(self, args): -- cgit v1.2.3-1-g7c22 From c1dfee4c7ed9eb1d8ca62d95182552cc847ec52e Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 11:24:42 -0500 Subject: Revert "Init: Remove Base from default plugin list" This reverts commit 611ce16c8cef81a6fc754c46dcb5cbe618b20b67. --- src/lib/Server/Admin/Init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index a9170530e..fff8bcd1c 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,6 +77,7 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', + 'Base', 'Bundler', 'Bzr', 'Cfg', -- cgit v1.2.3-1-g7c22 From 2c9a76cdfcce02f8d89129405f1f477753c47d3c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 13 Jun 2011 11:25:13 -0500 Subject: Init: Remove Base from default plugins list Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index fff8bcd1c..eab030cf8 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -103,8 +103,7 @@ plugin_list = ['Account', 'TGenshi'] # Default list of plugins to use -default_plugins = ['Base', - 'Bundler', +default_plugins = ['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', -- cgit v1.2.3-1-g7c22 From b1ab3e2bab9f07c13daf5dcfd4a9502eb84dcf0d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 14 Jun 2011 15:23:31 -0500 Subject: PY3K: Finish server-side code fixes Signed-off-by: Sol Jerome --- src/lib/SSLServer.py | 18 ++++++++++++++---- src/lib/Server/Core.py | 4 ++-- src/lib/Server/Plugin.py | 3 +++ src/lib/Server/Plugins/Cfg.py | 5 +++-- src/lib/Server/Plugins/Metadata.py | 2 +- src/lib/Server/Plugins/Probes.py | 5 +++-- src/lib/Server/Plugins/SSHbase.py | 21 +++++++++++++-------- 7 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index 8cac8a53f..bb0f61198 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -46,7 +46,11 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher): if '.' not in method: params = (address, ) + params response = self.instance._dispatch(method, params, self.funcs) - response = (response, ) + # py3k compatibility + if isinstance(response, bool) or isinstance(response, str): + response = (response, ) + else: + response = (response.decode('utf-8'), ) raw_response = xmlrpclib.dumps(response, methodresponse=1, allow_none=self.allow_none, encoding=self.encoding) @@ -191,9 +195,14 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): self.logger.error("No authentication data presented") return False auth_type, auth_content = header.split() - auth_content = base64.standard_b64decode(auth_content) + auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) try: - username, password = auth_content.split(":") + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(b":") + password = pw.decode('utf-8') except ValueError: username = auth_content password = "" @@ -234,11 +243,12 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): print("got select timeout") raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size)) + L.append(self.rfile.read(chunk_size).decode('utf-8')) size_remaining -= len(L[-1]) data = ''.join(L) response = self.server._marshaled_dispatch(self.client_address, data) + response = response.encode('utf-8') except: try: self.send_response(500) diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 8f9d3e746..5adfa5381 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -384,7 +384,7 @@ class Core(Component): # clear dynamic groups self.metadata.cgroups[meta.hostname] = [] try: - xpdata = lxml.etree.XML(probedata) + xpdata = lxml.etree.XML(probedata.encode('utf-8')) except: self.logger.error("Failed to parse probe data from client %s" % \ (address[0])) @@ -433,7 +433,7 @@ class Core(Component): @exposed def RecvStats(self, address, stats): """Act on statistics upload.""" - sdata = lxml.etree.XML(stats) + sdata = lxml.etree.XML(stats.encode('utf-8')) client = self.metadata.resolve_client(address) self.process_statistics(client, sdata) return "" diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3b331b300..3a36baf8e 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -693,6 +693,9 @@ class Specificity: self.prio = prio self.delta = delta + def __lt__(self, other): + return self.__cmp__(other) < 0 + def matches(self, metadata): return self.all or \ self.hostname == metadata.hostname or \ diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c08e8c4b6..4a0fd2dfe 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import binascii import logging import lxml +import operator import os import os.path import re @@ -25,7 +26,7 @@ logger = logging.getLogger('Bcfg2.Plugins.Cfg') def u_str(string, encoding): if sys.hexversion >= 0x03000000: - return str(string, encoding) + return string.encode(encoding) else: return unicode(string, encoding) @@ -105,7 +106,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] - matching.sort(self.sort_by_specific) + matching.sort(key=operator.attrgetter('specific')) non_delta = [matching.index(m) for m in matching if not m.specific.delta] if not non_delta: diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ca6e43851..6570f2912 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (address[0])) return False # populate the session cache - if user != 'root': + if user.decode('utf-8') != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index ea2e79ccc..3599f2af1 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -1,4 +1,5 @@ import lxml.etree +import operator import re import Bcfg2.Server.Plugin @@ -27,7 +28,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret = [] build = dict() candidates = self.get_matching(metadata) - candidates.sort(lambda x, y: cmp(x.specific, y.specific)) + candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: rem = specific_probe_matcher.match(entry.name) if not rem: @@ -90,7 +91,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data) + datafile.write(data.decode('utf-8')) def load_data(self): try: diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cf0998aaa..4a33c0cb0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", - "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"] + "ssh_host_rsa_key.pub.H_%s", + "ssh_host_key.pub.H_%s"] hostkeys = ["ssh_host_dsa_key.H_%s", - "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"] - keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key', - 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub', - 'ssh_host_key.pub'] + "ssh_host_rsa_key.H_%s", + "ssh_host_key.H_%s"] + keypatterns = ["ssh_host_dsa_key", + "ssh_host_rsa_key", + "ssh_host_key", + "ssh_host_dsa_key.pub", + "ssh_host_rsa_key.pub", + "ssh_host_key.pub"] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([str(value.data) for key, value in \ + self.__skn = "\n".join([value.data.decode() for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() @@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname) continue self.__skn += "%s %s" % (','.join(names[hostname]), - self.entries[pubkey].data) + self.entries[pubkey].data.decode()) return self.__skn def set_skn(self, value): @@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, hostkeys.sort() for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - self.entries[hostkey].data) + self.entries[hostkey].data.decode()) permdata = {'owner': 'root', 'group': 'root', 'type': 'file', -- cgit v1.2.3-1-g7c22