diff options
author | Jack Neely <jjneely@ncsu.edu> | 2011-06-22 13:37:58 -0400 |
---|---|---|
committer | Jack Neely <jjneely@ncsu.edu> | 2011-06-22 13:37:58 -0400 |
commit | b848b5303c21c2dcb3d3c52e260944676864e332 (patch) | |
tree | d1969c51a20a227b42ffb4c077ba1df79d784240 /src/lib | |
parent | 9d0c43a54e4172061af2779336b821bb93fefc71 (diff) | |
parent | 1186eae0c62ba6810ea597083bd8e56d487811c0 (diff) | |
download | bcfg2-b848b5303c21c2dcb3d3c52e260944676864e332.tar.gz bcfg2-b848b5303c21c2dcb3d3c52e260944676864e332.tar.bz2 bcfg2-b848b5303c21c2dcb3d3c52e260944676864e332.zip |
Merge branch 'master' into yumng
Conflicts:
src/lib/Client/Tools/YUMng.py
Diffstat (limited to 'src/lib')
27 files changed, 345 insertions, 165 deletions
diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index e33e1a7df..d8e308ee0 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -200,15 +200,18 @@ class Frame: tl = [t for t in self.tools if t.handlesEntry(cfile) \ and t.canVerify(cfile)] if tl: - if not tl[0].VerifyPath(cfile, []): - if self.setup['interactive'] and not \ - promptFilter("Install %s: %s? (y/N):", [cfile]): - continue - try: - self.states[cfile] = tl[0].InstallPath(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) + if self.setup['interactive'] and not \ + promptFilter("Install %s: %s? (y/N):", [cfile]): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tl[0].InstallPath(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tl[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) def Inventory(self): """ diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Client/Tools/APT.py index d7be44dc0..2b8cc3304 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Client/Tools/APT.py @@ -73,7 +73,8 @@ class APT(Bcfg2.Client.Tools.Tool): self.cmd.run("%s clean" % APTGET) try: self.pkg_cache = apt.cache.Cache() - except SystemError, e: + except SystemError: + e = sys.exc_info()[1] self.logger.info("Failed to initialize APT cache: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError self.pkg_cache.update() diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 64821f621..faec2e251 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -16,6 +16,10 @@ import shutil import stat import sys import time +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str + import Bcfg2.Client.Tools import Bcfg2.Options from Bcfg2.Client import XML @@ -145,7 +149,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: content = open(entry.get('name')).read() entry.set('current_bfile', binascii.b2a_base64(content)) - except IOError, error: + except IOError: + error = sys.exc_info()[1] self.logger.error("Failed to read %s: %s" % (error.filename, error.strerror)) @@ -457,12 +462,14 @@ class POSIX(Bcfg2.Client.Tools.Tool): if type(tempdata) == unicode: try: tempdata = tempdata.encode(self.setup['encoding']) - except UnicodeEncodeError, e: + except UnicodeEncodeError: + e = sys.exc_info()[1] self.logger.error("Error encoding file %s:\n %s" % \ (entry.get('name'), e)) try: content = open(entry.get('name')).read() - except IOError, error: + except IOError: + error = sys.exc_info()[1] if error.strerror == "No such file or directory": # print diff for files that don't exist (yet) content = '' @@ -586,7 +593,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): datetime.isoformat(datetime.now()))) self.logger.info("Backup of %s saved to %s" % (entry.get('name'), self.ppath)) - except IOError, e: + except IOError: + e = sys.exc_info()[1] self.logger.error("Failed to create backup file for %s" % \ (entry.get('name'))) self.logger.error(e) @@ -622,7 +630,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): % (entry.get('name'))) return False return True - except (OSError, IOError), err: + except (OSError, IOError): + err = sys.exc_info()[1] if err.errno == errno.EACCES: self.logger.info("Failed to open %s for writing" % (entry.get('name'))) else: @@ -702,7 +711,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): return False try: shutil.rmtree(ename) - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) else: @@ -710,20 +720,63 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: os.rmdir(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False try: os.remove(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False def Verifypermissions(self, entry, _): """Verify Path type='permissions' entry""" + if entry.get('perms') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % (entry.get('name'))) + return False + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid: + # owner mismatch for path + entry.set('current_owner', str(pstat.st_uid)) + self.logger.debug("%s %s ownership wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Owner for path %s is incorrect. " + "Current owner is %s but should be %s\n" % \ + (path, pstat.st_uid, entry.get('owner'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False + if group != pstat.st_gid: + # group mismatch for path + entry.set('current_group', str(pstat.st_gid)) + self.logger.debug("%s %s group wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Group for path %s is incorrect. " + "Current group is %s but should be %s\n" % \ + (path, pstat.st_gid, entry.get('group'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False return self.Verifydirectory(entry, _) def Installpermissions(self, entry): @@ -734,9 +787,23 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.error('Entry %s not completely specified. ' 'Try running bcfg2-lint.' % (entry.get('name'))) return False + plist = [entry.get('name')] + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid or group != pstat.st_gid: + # owner mismatch for path + plist.append(path) try: - os.chown(entry.get('name'), normUid(entry), normGid(entry)) - os.chmod(entry.get('name'), calcPerms(S_IFDIR, entry.get('perms'))) + for p in plist: + os.chown(p, normUid(entry), normGid(entry)) + os.chmod(p, calcPerms(S_IFDIR, entry.get('perms'))) return True except (OSError, KeyError): self.logger.error('Permission fixup failed for %s' % \ diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Client/Tools/Pacman.py index 082897934..c8c05061c 100644 --- a/src/lib/Client/Tools/Pacman.py +++ b/src/lib/Client/Tools/Pacman.py @@ -78,5 +78,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): try: self.logger.debug("Running : %s -S %s" % (self.pkgtool, pkgline)) self.cmd.run("%s -S %s" % (self.pkgtool, pkgline)) - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("Error occurred during installation: %s" % e) diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Client/Tools/RPMng.py index a1e14b3a6..5376118c2 100644 --- a/src/lib/Client/Tools/RPMng.py +++ b/src/lib/Client/Tools/RPMng.py @@ -2,11 +2,12 @@ __revision__ = '$Revision$' -import ConfigParser import os.path import rpm import rpmtools import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Client/Tools/YUM24.py index 5387fdd6a..66768fb34 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Client/Tools/YUM24.py @@ -1,13 +1,14 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision: $' -import ConfigParser import copy import os.path import sys import yum import Bcfg2.Client.XML import Bcfg2.Client.Tools.RPMng +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -151,7 +152,8 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): try: pkgDict = dict([(i.name, i) for i in \ self.yb.returnPackagesByDep(entry.get('name'))]) - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error('Yum Error Depsolving for %s: %s' % \ (entry.get('name'), str(e))) pkgDict = {} diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index 0435ca0d7..6aab9817c 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -1,9 +1,9 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision$' -import ConfigParser import copy import os.path +import sys import yum import yum.packages import yum.rpmtrans @@ -13,6 +13,8 @@ import yum.misc import rpmUtils.arch import Bcfg2.Client.XML import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -167,10 +169,12 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): self.yb.doConfigSetup() self.yb.doTsSetup() self.yb.doRpmDBSetup() - except yum.Errors.RepoError, e: + except yum.Errors.RepoError: + e = sys.exc_info()[1] self.logger.error("YUMng Repository error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("YUMng error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError @@ -505,7 +509,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): try: vResult = self._verifyHelper(_POs[0]) - except Exception, e: + except Exception: + e = sys.exc_info()[1] # Unknown Yum exception self.logger.warning(" Verify Exception: %s" % str(e)) package_fail = True @@ -680,7 +685,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): # Run the Yum Transaction try: rescode, restring = self.yb.buildTransaction() - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) cleanup() return @@ -695,7 +701,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): self.yb.processTransaction(callback=yDisplay, rpmDisplay=rDisplay) self.logger.info("Single Pass for Install Succeeded") - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) cleanup() return @@ -822,7 +829,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.install(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error installing some packages: %s" % yume) if len(upgrade_pkgs) > 0: @@ -832,7 +840,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.update(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" % yume) if len(reinstall_pkgs) > 0: @@ -841,7 +850,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.reinstall(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" \ % yume) diff --git a/src/lib/Options.py b/src/lib/Options.py index 0d978c519..619b16787 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -323,7 +323,7 @@ CLIENT_BUNDLE = Option('Only configure the given bundle(s)', default=[], cmd='-b', odesc='<bundle:bundle>', cook=colon_split) CLIENT_BUNDLEQUICK = Option('only verify/configure the given bundle(s)', default=False, cmd='-Q') -CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, +CLIENT_INDEP = Option('Only configure independent entries, ignore bundles', default=False, cmd='-z') CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index e4a0f6a3d..4cb0bbe80 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -47,6 +47,9 @@ __all__ = ["ComponentProxy", class CertificateError(Exception): def __init__(self, commonName): self.commonName = commonName + def __str__(self): + return ("Got unallowed commonName %s from server" + % self.commonName) class RetryMethod(xmlrpclib._Method): @@ -193,13 +196,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): ca_certs=self.ca, suppress_ragged_eofs=True, keyfile=self.key, certfile=self.cert, ssl_version=ssl_protocol_ver) - try: - self.sock.connect((self.host, self.port)) - except socket.gaierror: - e = sys.exc_info()[1] - self.logger.error("Unable to connect to %s:%s\n%s" % - (self.host, self.port, e.strerror)) - sys.exit(1) + self.sock.connect((self.host, self.port)) peer_cert = self.sock.getpeercert() if peer_cert and self.scns: scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index 8cac8a53f..21bf48d3e 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -46,7 +46,12 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher): if '.' not in method: params = (address, ) + params response = self.instance._dispatch(method, params, self.funcs) - response = (response, ) + # py3k compatibility + if isinstance(response, bool) or isinstance(response, str) \ + or isinstance(response, list): + response = (response, ) + else: + response = (response.decode('utf-8'), ) raw_response = xmlrpclib.dumps(response, methodresponse=1, allow_none=self.allow_none, encoding=self.encoding) @@ -191,9 +196,18 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): self.logger.error("No authentication data presented") return False auth_type, auth_content = header.split() - auth_content = base64.standard_b64decode(auth_content) try: - username, password = auth_content.split(":") + # py3k compatibility + auth_content = base64.standard_b64decode(auth_content) + except TypeError: + auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) + try: + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(bytes(":", encoding='utf-8')) + password = pw.decode('utf-8') except ValueError: username = auth_content password = "" @@ -234,11 +248,13 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): print("got select timeout") raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size)) + L.append(self.rfile.read(chunk_size).decode('utf-8')) size_remaining -= len(L[-1]) data = ''.join(L) response = self.server._marshaled_dispatch(self.client_address, data) + if sys.hexversion >= 0x03000000: + response = response.encode('utf-8') except: try: self.send_response(500) diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py index 3af25b15a..81fc4a1b1 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Server/Admin/Client.py @@ -27,7 +27,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val @@ -41,7 +42,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index a9170530e..eab030cf8 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -77,6 +77,7 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), # Complete list of plugins plugin_list = ['Account', + 'Base', 'Bundler', 'Bzr', 'Cfg', @@ -102,8 +103,7 @@ plugin_list = ['Account', 'TGenshi'] # Default list of plugins to use -default_plugins = ['Base', - 'Bundler', +default_plugins = ['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 8f9d3e746..5adfa5381 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -384,7 +384,7 @@ class Core(Component): # clear dynamic groups self.metadata.cgroups[meta.hostname] = [] try: - xpdata = lxml.etree.XML(probedata) + xpdata = lxml.etree.XML(probedata.encode('utf-8')) except: self.logger.error("Failed to parse probe data from client %s" % \ (address[0])) @@ -433,7 +433,7 @@ class Core(Component): @exposed def RecvStats(self, address, stats): """Act on statistics upload.""" - sdata = lxml.etree.XML(stats) + sdata = lxml.etree.XML(stats.encode('utf-8')) client = self.metadata.resolve_client(address) self.process_statistics(client, sdata) return "<ok/>" diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py index aa822409c..bf774f695 100644 --- a/src/lib/Server/Hostbase/backends.py +++ b/src/lib/Server/Hostbase/backends.py @@ -57,12 +57,14 @@ class NISBackend(object): return user - except NISAUTHError, e: + except NISAUTHError: + e = sys.exc_info()[1] return None def get_user(self, user_id): try: return User.objects.get(pk=user_id) - except User.DoesNotExist, e: + except User.DoesNotExist: + e = sys.exc_info()[1] return None diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py index 21b462c86..f3db26f67 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Server/Hostbase/ldapauth.py @@ -69,7 +69,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -86,7 +87,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -108,7 +110,8 @@ class ldapauth(object): raw_obj = result_data[0][1] distinguishedName = raw_obj['distinguishedName'] return ('success', distinguishedName[0],) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -134,7 +137,8 @@ class ldapauth(object): self.is_superuser = False return - except KeyError, e: + except KeyError: + e = sys.exc_info()[1] raise LDAPAUTHError("Portions of the LDAP User profile not present") def member_of(self): diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py index c44c7bf16..4e641f13c 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Server/Hostbase/settings.py @@ -1,9 +1,10 @@ -from ConfigParser import ConfigParser, NoSectionError, NoOptionError import os.path +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) -c = ConfigParser() +c = ConfigParser.ConfigParser() #This needs to be configurable one day somehow c.read(['./bcfg2.conf']) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8e86cc564..09443d4c0 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): props = self.core.plugins['Properties'] for propfile, pdata in props.store.entries.items(): if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.data, 'properties') + self.check_xml(pdata.name, pdata.xdata, 'properties') def check_metadata(self): """ check metadata files for required headers """ diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 3b331b300..30c4f9686 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked): def Index(self): """Build local data structures.""" try: - xdata = XML(self.data) + self.xdata = XML(self.data) except XMLSyntaxError: logger.error("Failed to parse %s" % (self.name)) return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() + self.entries = self.xdata.getchildren() + if self.__identifier__ is not None: + self.label = self.xdata.attrib[self.__identifier__] def __iter__(self): return iter(self.entries) @@ -455,53 +456,51 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" + __identifier__ = None + def __init__(self, name): XMLFileBacked.__init__(self, name) - self.fragments = {} - - def Index(self): - """Build internal data structures.""" - try: - xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - logger.error("Failed to parse file %s" % self.name) - return - self.fragments = {} - work = {lambda x: True: xdata.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + self.matches = {} + + def _match(self, item, metadata): + """ recursive helper for Match() """ + if isinstance(item, lxml.etree._Comment): + return [] + elif item.tag == 'Group': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') not in metadata.groups) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') in metadata.groups)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + elif item.tag == 'Client': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') != metadata.hostname) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') == metadata.hostname)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + else: + rv = copy.deepcopy(item) + for child in rv.iterchildren(): + rv.remove(child) + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return [rv] + def Match(self, metadata): """Return matching fragments of independent.""" - matching = [frag for (pred, frag) in list(self.fragments.items()) - if pred(metadata)] - if matching: - return reduce(lambda x, y: x + y, matching) - logger.error("File %s got null match" % (self.name)) - return [] + if metadata.hostname not in self.matches: + rv = [] + for child in self.entries: + rv.extend(self._match(child, metadata)) + logger.debug("File %s got %d match(es)" % (self.name, len(rv))) + self.matches[metadata.hostname] = rv + return self.matches[metadata.hostname] class INode: @@ -693,6 +692,9 @@ class Specificity: self.prio = prio self.delta = delta + def __lt__(self, other): + return self.__cmp__(other) < 0 + def matches(self, metadata): return self.all or \ self.hostname == metadata.hostname or \ diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index c08e8c4b6..23ba0a745 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -4,9 +4,11 @@ __revision__ = '$Revision$' import binascii import logging import lxml +import operator import os import os.path import re +import stat import sys import tempfile @@ -23,9 +25,10 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') +# py3k compatibility def u_str(string, encoding): if sys.hexversion >= 0x03000000: - return str(string, encoding) + return string.encode(encoding) else: return unicode(string, encoding) @@ -95,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) self.specific = CfgMatcher(path.split('/')[-1]) + path = path def sort_by_specific(self, one, other): return cmp(one.specific, other.specific) @@ -105,7 +109,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] - matching.sort(self.sort_by_specific) + matching.sort(key=operator.attrgetter('specific')) non_delta = [matching.index(m) for m in matching if not m.specific.delta] if not non_delta: @@ -119,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): self.bind_info_to_entry(entry, metadata) used = self.get_pertinent_entries(metadata) basefile = used.pop(0) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = "%s/%s" % (self.path, entry.get('name')) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ca6e43851..6570f2912 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (address[0])) return False # populate the session cache - if user != 'root': + if user.decode('utf-8') != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index ea2e79ccc..ec0f294dd 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -1,12 +1,100 @@ import lxml.etree +import operator import re +try: + import json + has_json = True +except ImportError: + try: + import simplejson as json + has_json = True + except ImportError: + has_json = False + +try: + import syck + has_syck = True +except ImportError: + has_syck = False + try: + import yaml + has_yaml = True + except ImportError: + has_yaml = False + import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P<basename>\S+)") +class ProbeData (object): + """ a ProbeData object emulates a str object, but also has .xdata + and .json properties to provide convenient ways to use ProbeData + objects as XML or JSON data """ + def __init__(self, data): + self.data = data + self._xdata = None + self._json = None + self._yaml = None + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __getattr__(self, name): + """ make ProbeData act like a str object """ + return getattr(self.data, name) + + def __complex__(self): + return complex(self.data) + + def __int__(self): + return int(self.data) + + def __long__(self): + return long(self.data) + + def __float__(self): + return float(self.data) + + @property + def xdata(self): + if self._xdata is None: + try: + self._xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + pass + return self._xdata + + @property + def json(self): + if self._json is None and has_json: + try: + self._json = json.loads(self.data) + except ValueError: + pass + return self._json + + @property + def yaml(self): + if self._yaml is None: + if has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml.YAMLError: + pass + elif has_syck: + try: + self._yaml = syck.load(self.data) + except syck.error: + pass + return self._yaml + + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") @@ -27,7 +115,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret = [] build = dict() candidates = self.get_matching(metadata) - candidates.sort(lambda x, y: cmp(x.specific, y.specific)) + candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: rem = specific_probe_matcher.match(entry.name) if not rem: @@ -80,7 +168,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, cx = lxml.etree.SubElement(top, 'Client', name=client) for probe in sorted(probed): lxml.etree.SubElement(cx, 'Probe', name=probe, - value=self.probedata[client][probe]) + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) data = lxml.etree.tostring(top, encoding='UTF-8', @@ -90,7 +178,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data) + datafile.write(data.decode('utf-8')) def load_data(self): try: @@ -105,7 +193,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') + self.probedata[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get('value')) elif (pdata.tag == 'Group'): self.cgroups[client.get('name')].append(pdata.get('name')) @@ -128,9 +217,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.logger.error("Got null response to probe %s from %s" % \ (data.get('name'), client.hostname)) try: - self.probedata[client.hostname].update({data.get('name'): ''}) + self.probedata[client.hostname].update({data.get('name'): + ProbeData('')}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): ''} + self.probedata[client.hostname] = \ + {data.get('name'): ProbeData('')} return dlines = data.text.split('\n') self.logger.debug("%s:probe:%s:%s" % (client.hostname, @@ -141,11 +232,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, if newgroup not in self.cgroups[client.hostname]: self.cgroups[client.hostname].append(newgroup) dlines.remove(line) - dtext = "\n".join(dlines) + dobj = ProbeData("\n".join(dlines)) try: - self.probedata[client.hostname].update({data.get('name'): dtext}) + self.probedata[client.hostname].update({data.get('name'): dobj}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): dtext} + self.probedata[client.hostname] = {data.get('name'): dobj} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index dea797a10..54c5def57 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,44 +6,7 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - def Index(self): - """Build internal data structures.""" - if type(self.data) is not lxml.etree._Element: - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % - self.name) - - self.fragments = {} - work = {lambda x: True: self.data.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - - + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cf0998aaa..4a33c0cb0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", - "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"] + "ssh_host_rsa_key.pub.H_%s", + "ssh_host_key.pub.H_%s"] hostkeys = ["ssh_host_dsa_key.H_%s", - "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"] - keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key', - 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub', - 'ssh_host_key.pub'] + "ssh_host_rsa_key.H_%s", + "ssh_host_key.H_%s"] + keypatterns = ["ssh_host_dsa_key", + "ssh_host_rsa_key", + "ssh_host_key", + "ssh_host_dsa_key.pub", + "ssh_host_rsa_key.pub", + "ssh_host_key.pub"] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([str(value.data) for key, value in \ + self.__skn = "\n".join([value.data.decode() for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() @@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname) continue self.__skn += "%s %s" % (','.join(names[hostname]), - self.entries[pubkey].data) + self.entries[pubkey].data.decode()) return self.__skn def set_skn(self, value): @@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, hostkeys.sort() for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - self.entries[hostkey].data) + self.entries[hostkey].data.decode()) permdata = {'owner': 'root', 'group': 'root', 'type': 'file', diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index 8b6bad574..aeb3b9f74 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -28,6 +28,7 @@ datafields = { } +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html index 6ef4c9aff..ec9a17468 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Server/Reports/reports/templates/base.html @@ -87,7 +87,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.2.0pre2</span> + <span>Bcfg2 Version 1.2.0pre3</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py index 291528e2e..2e30125f9 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py @@ -15,6 +15,7 @@ try: except: colorize = False +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 2aa35f1ec..f30c38a05 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string |