diff options
40 files changed, 345 insertions, 259 deletions
diff --git a/src/lib/Bcfg2/Bcfg2Py3k.py b/src/lib/Bcfg2/Bcfg2Py3k.py index 640efff95..2c179a5ef 100644 --- a/src/lib/Bcfg2/Bcfg2Py3k.py +++ b/src/lib/Bcfg2/Bcfg2Py3k.py @@ -82,6 +82,11 @@ try: except: unicode = unicode +# base64 compat +from base64 import b64encode as _b64encode, b64decode as _b64decode +b64encode = lambda s: _b64encode(s.encode('ascii')).decode('ascii') +b64decode = lambda s: _b64decode(s.encode('ascii')).decode('ascii') + try: input = raw_input except: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py index b8fb0f4d0..3ecef2cd7 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py @@ -1,6 +1,6 @@ import os import sys -from base import POSIXTool, device_map +from .base import POSIXTool, device_map class POSIXDevice(POSIXTool): __req__ = ['name', 'dev_type', 'perms', 'owner', 'group'] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py index 4b0ad93ef..27834a16d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py @@ -3,7 +3,7 @@ import sys import stat import shutil import Bcfg2.Client.XML -from base import POSIXTool +from .base import POSIXTool class POSIXDirectory(POSIXTool): __req__ = ['name', 'perms', 'owner', 'group'] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 1b5535d28..bba72791d 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -3,10 +3,9 @@ import sys import stat import time import difflib -import binascii import tempfile -from base import POSIXTool -from Bcfg2.Bcfg2Py3k import unicode +from .base import POSIXTool +from Bcfg2.Bcfg2Py3k import unicode, b64encode, b64decode class POSIXFile(POSIXTool): __req__ = ['name', 'perms', 'owner', 'group'] @@ -20,6 +19,9 @@ class POSIXFile(POSIXTool): for char in strng: if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: return False + if not hasattr(strng, "decode"): + # py3k + return True try: strng.decode(encoding) return True @@ -29,13 +31,14 @@ class POSIXFile(POSIXTool): def _get_data(self, entry): is_binary = False if entry.get('encoding', 'ascii') == 'base64': - tempdata = binascii.a2b_base64(entry.text) + tempdata = b64decode(entry.text) is_binary = True + elif entry.get('empty', 'false') == 'true': tempdata = '' else: tempdata = entry.text - if isinstance(tempdata, unicode): + if isinstance(tempdata, unicode) and unicode != str: try: tempdata = tempdata.encode(self.setup['encoding']) except UnicodeEncodeError: @@ -163,7 +166,7 @@ class POSIXFile(POSIXTool): if is_binary: # don't compute diffs if the file is binary prompt.append('Binary file, no printable diff') - attrs['current_bfile'] = binascii.b2a_base64(content) + attrs['current_bfile'] = b64encode(content) else: if interactive: diff = self._diff(content, self._get_data(entry)[0], @@ -171,8 +174,10 @@ class POSIXFile(POSIXTool): filename=entry.get("name")) if diff: udiff = ''.join(diff) + if hasattr(udiff, "decode"): + udiff = udiff.decode(self.setup['encoding']) try: - prompt.append(udiff.decode(self.setup['encoding'])) + prompt.append(udiff) except UnicodeEncodeError: prompt.append("Could not encode diff") else: @@ -182,10 +187,9 @@ class POSIXFile(POSIXTool): diff = self._diff(content, self._get_data(entry)[0], difflib.ndiff, filename=entry.get("name")) if diff: - attrs["current_bdiff"] = \ - binascii.b2a_base64("\n".join(diff)) + attrs["current_bdiff"] = b64encode("\n".join(diff)) else: - attrs['current_bfile'] = binascii.b2a_base64(content) + attrs['current_bfile'] = b64encode(content) if interactive: entry.set("qtext", "\n".join(prompt)) if not sensitive: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py index 569ca3445..060c9499e 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Hardlink.py @@ -1,6 +1,6 @@ import os import sys -from base import POSIXTool +from .base import POSIXTool class POSIXHardlink(POSIXTool): __req__ = ['name', 'to'] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py index 64a36cce4..42d4a4cb4 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py @@ -1,7 +1,7 @@ import os import sys import shutil -from base import POSIXTool +from .base import POSIXTool class POSIXNonexistent(POSIXTool): __req__ = ['name'] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py b/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py index c041b9ade..de2bee120 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Permissions.py @@ -1,6 +1,6 @@ import os import sys -from base import POSIXTool +from .base import POSIXTool class POSIXPermissions(POSIXTool): __req__ = ['name', 'perms', 'owner', 'group'] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py b/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py index d5222513e..1a5abc652 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Symlink.py @@ -1,6 +1,6 @@ import os import sys -from base import POSIXTool +from .base import POSIXTool class POSIXSymlink(POSIXTool): __req__ = ['name', 'to'] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 7e649a2c1..2e4a54475 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -7,7 +7,7 @@ import shutil import pkgutil from datetime import datetime import Bcfg2.Client.Tools -from base import POSIXTool +from .base import POSIXTool class POSIX(Bcfg2.Client.Tools.Tool): """POSIX File support code.""" diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 43f2ce68c..af28dd0a6 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -553,8 +553,7 @@ class BaseCore(object): for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): for probe in plugin.GetProbes(metadata): resp.append(probe) - return lxml.etree.tostring(resp, encoding='UTF-8', - xml_declaration=False) + return lxml.etree.tostring(resp, encoding='unicode') except: err = sys.exc_info()[1] self.critical_error("Error determining probes for %s: %s" % @@ -608,8 +607,7 @@ class BaseCore(object): client = self.resolve_client(address)[0] try: config = self.BuildConfiguration(client) - return lxml.etree.tostring(config, encoding='UTF-8', - xml_declaration=False) + return lxml.etree.tostring(config, encoding='unicode') except Bcfg2.Server.Plugin.MetadataConsistencyError: self.critical_error("Metadata consistency failure for %s" % client) diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 42ea6ba8c..7ee18f924 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -90,9 +90,9 @@ class Plugin (object): if el.text and not keep_text: el.text = '...' [el.remove(c) for c in el.iterchildren()] - xml = lxml.etree.tostring(el).strip() + xml = lxml.etree.tostring(el, encoding='unicode').strip() else: - xml = lxml.etree.tostring(element).strip() + xml = lxml.etree.tostring(element, encoding='unicode').strip() return " line %s: %s" % (element.sourceline, xml) diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index 82162aa4a..fa2e16c10 100644 --- a/src/lib/Bcfg2/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -265,7 +265,8 @@ class ThreadedStatistics(Statistics, threading.Thread): (metadata, data) = self.work_queue.get_nowait() try: pending_data.append((metadata.hostname, - lxml.etree.tostring(data))) + lxml.etree.tostring(data, + encoding='unicode'))) except: err = sys.exc_info()[1] self.logger.warning("Dropping interaction for %s: %s" % @@ -560,7 +561,7 @@ class DirectoryBacked(object): event.filename).lstrip('/') if action == 'deleted': - for key in self.entries.keys(): + for key in list(self.entries.keys()): if key.startswith(relpath): del self.entries[key] # We remove values from self.entries, but not diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index f7577d60e..aef239fce 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -6,11 +6,10 @@ import sys import stat import pkgutil import logging -import binascii import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import u_str +from Bcfg2.Bcfg2Py3k import u_str, b64encode import Bcfg2.Server.Lint logger = logging.getLogger(__name__) @@ -290,7 +289,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if entry.get('encoding') == 'base64': - data = binascii.b2a_base64(data) + data = b64encode(data) else: try: data = u_str(data, self.encoding) @@ -371,26 +370,26 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if attr in new_entry] if badattr: # check for info files and inform user of their removal - if os.path.exists(self.path + "/:info"): - logger.info("Removing :info file and replacing with " - "info.xml") - os.remove(self.path + "/:info") - if os.path.exists(self.path + "/info"): - logger.info("Removing info file and replacing with " - "info.xml") - os.remove(self.path + "/info") + for ifile in ['info', ':info']: + info = os.path.join(self.path, ifile) + if os.path.exists(info): + logger.info("Removing %s and replacing with info.xml" % + info) + os.remove(info) metadata_updates = {} metadata_updates.update(self.metadata) for attr in badattr: metadata_updates[attr] = new_entry.get(attr) infoxml = lxml.etree.Element('FileInfo') infotag = lxml.etree.SubElement(infoxml, 'Info') - [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ - for attr in metadata_updates] + [infotag.attrib.__setitem__(attr, metadata_updates[attr]) + for attr in metadata_updates] ofile = open(self.path + "/info.xml", "w") - ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) + ofile.write(lxml.etree.tostring(infoxml, encoding='unicode', + pretty_print=True)) ofile.close() - self.debug_log("Wrote file %s" % (self.path + "/info.xml"), + self.debug_log("Wrote file %s" % os.path.join(self.path, + "info.xml"), flag=log) diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index b8750f850..63c590f0f 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -1,4 +1,3 @@ -import binascii import difflib import logging import lxml.etree @@ -14,6 +13,8 @@ except ImportError: import Bcfg2.Server.Plugin from Bcfg2.Server.Reports.importscript import load_stat from Bcfg2.Server.Reports.reports.models import Client +from Bcfg2.Bcfg2Py3k import b64decode + # for debugging output only logger = logging.getLogger('Bcfg2.Plugins.DBStats') @@ -88,7 +89,7 @@ class DBStats(Bcfg2.Server.Plugin.ThreadedStatistics, ret.append('\n'.join(entry.reason.unpruned)) elif entry.reason.current_diff != '': if entry.reason.is_binary: - ret.append(binascii.a2b_base64(entry.reason.current_diff)) + ret.append(b64decode(entry.reason.current_diff)) else: ret.append('\n'.join(difflib.restore(\ entry.reason.current_diff.split('\n'), 1))) diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index b4a20b0eb..a48524ac9 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -7,19 +7,19 @@ the client """ import os import sys import errno -import binascii import lxml.etree import Bcfg2.Options import Bcfg2.Server import Bcfg2.Server.Plugin +from Bcfg2.Bcfg2Py3k import b64decode probecode = """#!/usr/bin/env python import os import pwd import grp -import binascii import lxml.etree +from Bcfg2.Bcfg2Py3k import b64encode path = "%s" @@ -33,8 +33,8 @@ data = lxml.etree.Element("ProbedFileData", owner=pwd.getpwuid(stat[4])[0], group=grp.getgrgid(stat[5])[0], perms=oct(stat[0] & 07777)) -data.text = binascii.b2a_base64(open(path).read()) -print(lxml.etree.tostring(data)) +data.text = b64encode(open(path).read()) +print(lxml.etree.tostring(data, encoding="unicode")) """ class FileProbes(Bcfg2.Server.Plugin.Plugin, @@ -103,7 +103,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, def write_data(self, data, metadata): """Write the probed file data to the bcfg2 specification.""" filename = data.get("name") - contents = binascii.a2b_base64(data.text) + contents = b64decode(data.text) entry = self.entries[metadata.hostname][filename] cfg = self.core.plugins['Cfg'] specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) @@ -120,7 +120,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, # get current entry data if entry.text and entry.get("encoding") == "base64": - entrydata = binascii.a2b_base64(entry.text) + entrydata = b64decode(entry.text) else: entrydata = entry.text @@ -187,7 +187,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, # get current entry data if entry.get("encoding") == "base64": - entrydata = binascii.a2b_base64(entry.text) + entrydata = b64decode(entry.text) else: entrydata = entry.text if entrydata == contents: @@ -199,8 +199,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, if os.path.exists(infoxml): return - self.logger.info("Writing info.xml at %s for %s" % - (infoxml, data.get("name"))) + self.logger.info("Writing %s for %s" % (infoxml, data.get("name"))) info = \ lxml.etree.Element("Info", owner=data.get("owner", @@ -216,6 +215,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, root.append(info) try: open(infoxml, "w").write(lxml.etree.tostring(root, + encoding='unicode', pretty_print=True)) except IOError: err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 3225b6c85..8f4f42c96 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -151,8 +151,8 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) # prep data dataroot = xmltree.getroot() - newcontents = str(lxml.etree.tostring(dataroot, pretty_print=True, - encoding='unicode')) + newcontents = lxml.etree.tostring(dataroot, pretty_print=True, + encoding='unicode') fd = datafile.fileno() while locked(fd) == True: @@ -807,7 +807,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # _any_ port numbers - perhaps a priority queue could # be faster? curtime = time.time() - for addrpair in self.session_cache.keys(): + for addrpair in list(self.session_cache.keys()): if addresspair[0] == addrpair[0]: (stamp, _) = self.session_cache[addrpair] if curtime - stamp > cache_ttl: @@ -1104,7 +1104,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, "secure mode" % address[0]) return False # populate the session cache - if user.decode('utf-8') != 'root': + if user != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 702ff64d2..71ae6a038 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -177,7 +177,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, to_remove.append(pkg) else: self.logger.error("Packages: Malformed Package: %s" % - lxml.etree.tostring(pkg)) + lxml.etree.tostring(pkg, + encoding='unicode')) gpkgs = collection.get_groups(groups) for group, pkgs in gpkgs.items(): diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index e08f52a28..34b5ab187 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -52,7 +52,8 @@ if has_django: class ClientProbeDataSet(dict): - """ dict of probe => [probe data] that records a for each host """ + """ dict of probe => [probe data] that records a timestamp for + each host """ def __init__(self, *args, **kwargs): if "timestamp" in kwargs and kwargs['timestamp'] is not None: self.timestamp = kwargs.pop("timestamp") @@ -191,12 +192,10 @@ class Probes(Bcfg2.Server.Plugin.Probing, value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) - data = lxml.etree.tostring(top, encoding='UTF-8', - xml_declaration=True, - pretty_print='true') try: datafile = open(os.path.join(self.data, 'probed.xml'), 'w') - datafile.write(data.decode('utf-8')) + datafile.write(lxml.etree.tostring(top, encoding='unicode', + pretty_print='true')) except IOError: err = sys.exc_info()[1] self.logger.error("Failed to write probed.xml: %s" % err) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index a879b064f..3405ad50c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -43,6 +43,7 @@ class PropertyFile(Bcfg2.Server.Plugin.StructFile): try: open(self.name, "wb").write(lxml.etree.tostring(self.xdata, + encoding='unicode', pretty_print=True)) return True except IOError: diff --git a/src/lib/Bcfg2/Server/Plugins/SEModules.py b/src/lib/Bcfg2/Server/Plugins/SEModules.py index a99e54bb2..0d75bb48f 100644 --- a/src/lib/Bcfg2/Server/Plugins/SEModules.py +++ b/src/lib/Bcfg2/Server/Plugins/SEModules.py @@ -1,15 +1,14 @@ import os import logging -import binascii -import posixpath - import Bcfg2.Server.Plugin +from Bcfg2.Bcfg2Py3k import b64encode + logger = logging.getLogger(__name__) class SEModuleData(Bcfg2.Server.Plugin.SpecificData): def bind_entry(self, entry, _): entry.set('encoding', 'base64') - entry.text = binascii.b2a_base64(self.data) + entry.text = b64encode(self.data) class SEModules(Bcfg2.Server.Plugin.GroupSpool): diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index a1a29727f..cbe8d0d9b 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -1,20 +1,16 @@ """This module manages ssh key files for bcfg2""" -import binascii import re import os +import sys import socket import shutil -import sys +import logging import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import u_str +from Bcfg2.Bcfg2Py3k import u_str, reduce, b64encode -if sys.hexversion >= 0x03000000: - from functools import reduce - -import logging logger = logging.getLogger(__name__) class KeyData(Bcfg2.Server.Plugin.SpecificData): @@ -31,7 +27,7 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): def bind_entry(self, entry, metadata): entry.set('type', 'file') if entry.get('encoding') == 'base64': - entry.text = binascii.b2a_base64(self.data) + entry.text = b64encode(self.data) else: try: entry.text = u_str(self.data, self.encoding) diff --git a/src/lib/Bcfg2/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py index 666beef21..e62638b4f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Snapshots.py +++ b/src/lib/Bcfg2/Server/Plugins/Snapshots.py @@ -1,9 +1,5 @@ -#import lxml.etree import logging -import binascii import difflib -#import sqlalchemy -#import sqlalchemy.orm import Bcfg2.Server.Plugin import Bcfg2.Server.Snapshots import Bcfg2.Logger @@ -13,8 +9,7 @@ import time import threading # Compatibility import -from Bcfg2.Bcfg2Py3k import Queue -from Bcfg2.Bcfg2Py3k import u_str +from Bcfg2.Bcfg2Py3k import Queue, u_str, b64decode logger = logging.getLogger('Snapshots') @@ -45,13 +40,12 @@ def build_snap_ent(entry): if entry.get('encoding', 'ascii') == 'ascii': desired['contents'] = u_str(entry.text) else: - desired['contents'] = u_str(binascii.a2b_base64(entry.text)) + desired['contents'] = u_str(b64decode(entry.text)) if 'current_bfile' in entry.attrib: - state['contents'] = u_str(binascii.a2b_base64( \ - entry.get('current_bfile'))) + state['contents'] = u_str(b64decode(entry.get('current_bfile'))) elif 'current_bdiff' in entry.attrib: - diff = binascii.a2b_base64(entry.get('current_bdiff')) + diff = b64decode(entry.get('current_bdiff')) state['contents'] = u_str( \ '\n'.join(difflib.restore(diff.split('\n'), 1))) diff --git a/src/lib/Bcfg2/Server/Plugins/Statistics.py b/src/lib/Bcfg2/Server/Plugins/Statistics.py index ce8d085cc..33b78047a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Statistics.py +++ b/src/lib/Bcfg2/Server/Plugins/Statistics.py @@ -1,16 +1,14 @@ '''This file manages the statistics collected by the BCFG2 Server''' -import binascii import copy import difflib import logging -from lxml.etree import XML, SubElement, Element, XMLSyntaxError import lxml.etree import os import sys from time import asctime, localtime, time, strptime, mktime import threading - +from Bcfg2.Bcfg2Py3k import b64decode import Bcfg2.Server.Plugin @@ -20,7 +18,7 @@ class StatisticsStore(object): def __init__(self, filename): self.filename = filename - self.element = Element('Dummy') + self.element = lxml.etree.Element('Dummy') self.dirty = 0 self.lastwrite = 0 self.logger = logging.getLogger('Bcfg2.Server.Statistics') @@ -36,7 +34,8 @@ class StatisticsStore(object): ioerr = sys.exc_info()[1] self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr)) else: - fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True)) + fout.write(lxml.etree.tostring(self.element, + encoding='unicode')) fout.close() os.rename(self.filename + '.new', self.filename) self.dirty = 0 @@ -48,11 +47,11 @@ class StatisticsStore(object): fin = open(self.filename, 'r') data = fin.read() fin.close() - self.element = XML(data) + self.element = lxml.etree.XML(data) self.dirty = 0 - except (IOError, XMLSyntaxError): + except (IOError, lxml.etree.XMLSyntaxError): self.logger.error("Creating new statistics file %s"%(self.filename)) - self.element = Element('ConfigStatistics') + self.element = lxml.etree.Element('ConfigStatistics') self.WriteBack() self.dirty = 0 @@ -78,7 +77,7 @@ class StatisticsStore(object): nummatch = len(nodes) if nummatch == 0: # Create an entry for this node - node = SubElement(self.element, 'Node', name=client) + node = lxml.etree.SubElement(self.element, 'Node', name=client) elif nummatch == 1 and not node_dirty: # Delete old instance node = nodes[0] @@ -150,9 +149,9 @@ class Statistics(Bcfg2.Server.Plugin.ThreadedStatistics, if cfentry.get('sensitive') in ['true', 'True']: raise Bcfg2.Server.Plugin.PluginExecutionError elif 'current_bfile' in cfentry.attrib: - contents = binascii.a2b_base64(cfentry.get('current_bfile')) + contents = b64decode(cfentry.get('current_bfile')) elif 'current_bdiff' in cfentry.attrib: - diff = binascii.a2b_base64(cfentry.get('current_bdiff')) + diff = b64decode(cfentry.get('current_bdiff')) contents = '\n'.join(difflib.restore(diff.split('\n'), 1)) else: contents = None diff --git a/src/lib/Bcfg2/Server/Plugins/TCheetah.py b/src/lib/Bcfg2/Server/Plugins/TCheetah.py index d99baa512..2bf475363 100644 --- a/src/lib/Bcfg2/Server/Plugins/TCheetah.py +++ b/src/lib/Bcfg2/Server/Plugins/TCheetah.py @@ -1,13 +1,11 @@ '''This module implements a templating generator based on Cheetah''' -import binascii import logging import sys import traceback import Bcfg2.Server.Plugin -# py3k compatibility -if sys.hexversion >= 0x03000000: - unicode = str + +from Bcfg2.Bcfg2Py3k import unicode, b64encode logger = logging.getLogger('Bcfg2.Plugins.TCheetah') @@ -60,7 +58,7 @@ class TemplateFile: else: if entry.get('encoding') == 'base64': # take care of case where file needs base64 encoding - entry.text = binascii.b2a_base64(self.template) + entry.text = b64encode(self.template) else: entry.text = unicode(str(self.template), self.encoding) except: diff --git a/src/lib/Bcfg2/Server/Plugins/TGenshi.py b/src/lib/Bcfg2/Server/Plugins/TGenshi.py index 846c2a464..c84e0a506 100644 --- a/src/lib/Bcfg2/Server/Plugins/TGenshi.py +++ b/src/lib/Bcfg2/Server/Plugins/TGenshi.py @@ -1,12 +1,10 @@ """This module implements a templating generator based on Genshi.""" -import binascii import logging import sys import Bcfg2.Server.Plugin -# py3k compatibility -if sys.hexversion >= 0x03000000: - unicode = str + +from Bcfg2.Bcfg2Py3k import unicode, b64encode logger = logging.getLogger('Bcfg2.Plugins.TGenshi') @@ -99,7 +97,7 @@ class TemplateFile: else: if entry.get('encoding') == 'base64': # take care of case where file needs base64 encoding - entry.text = binascii.b2a_base64(textdata) + entry.text = b64encode(textdata) else: entry.text = unicode(textdata, self.encoding) else: diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py index 61f0c103c..4eced8340 100755 --- a/src/lib/Bcfg2/Server/Reports/importscript.py +++ b/src/lib/Bcfg2/Server/Reports/importscript.py @@ -4,7 +4,6 @@ Imports statistics.xml and clients.xml files in to database backend for new statistics engine """ -import binascii import os import sys import traceback @@ -35,7 +34,7 @@ import Bcfg2.Logger import platform # Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser +from Bcfg2.Bcfg2Py3k import ConfigParser, b64decode def build_reason_kwargs(r_ent, encoding, logger): @@ -54,7 +53,7 @@ def build_reason_kwargs(r_ent, encoding, logger): # No point in flagging binary if we have no data binary_file = False elif r_ent.get('current_bdiff', False): - rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) + rc_diff = b64decode(r_ent.get('current_bdiff')) elif r_ent.get('current_diff', False): rc_diff = r_ent.get('current_diff') else: diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports index 7fa08110a..e49446385 100755 --- a/src/sbin/bcfg2-build-reports +++ b/src/sbin/bcfg2-build-reports @@ -110,7 +110,7 @@ def rss(reportxml, delivery, report): for item in items: channel.append(item) - tree = tostring(rssdata, encoding='UTF-8', xml_declaration=True) + tree = tostring(rssdata, encoding='unicode') fil.write(tree) fil.close() @@ -260,7 +260,7 @@ if __name__ == '__main__': # Apply XSLT, different ones based on report type, and options if deliverymechanism == 'null-operator': # Special Cases - fileout(tostring(ElementTree(procnodereport).getroot(), encoding='UTF-8', xml_declaration=True), deliv) + fileout(tostring(ElementTree(procnodereport).getroot(), encoding='unicode'), deliv) break transform = delivtype + '-' + deliverymechanism + '.xsl' @@ -312,7 +312,7 @@ if __name__ == '__main__': (toastring, socket.getfqdn(), outputstring) mail(outputstring, c) #call function to send else: - outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot(), encoding='UTF-8', xml_declaration=True) + outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot(), encoding='unicode') if deliverymechanism == 'rss': rss(outputstring, deliv, reprt) else: # Must be deliverymechanism == 'www': diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 79f9e26a2..9da4a25d1 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -255,7 +255,7 @@ class PropertiesEncryptor(Encryptor): while xdata.getparent() != None: xdata = xdata.getparent() xdata.set("encryption", "true") - return lxml.etree.tostring(xdata) + return lxml.etree.tostring(xdata, encoding='unicode') def _get_passphrase(self, chunk): pname = chunk.get("encrypted") or chunk.get("encryption") diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 7e8558855..3f7f33344 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -395,8 +395,7 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): try: metadata = self.build_metadata(client) self.Bind(entry, metadata) - data = lxml.etree.tostring(entry, encoding="UTF-8", - xml_declaration=True) + data = lxml.etree.tostring(entry, encoding="unicode") if outfile: open(outfile, 'w').write(data) else: diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDevice.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDevice.py index f32da9c37..b737cfe1e 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDevice.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDevice.py @@ -4,8 +4,8 @@ import unittest import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Client.Tools.POSIX.Device import * -from Test__init import get_posix_object -from Testbase import TestPOSIXTool +from .Test__init import get_posix_object +from .Testbase import TestPOSIXTool from .....common import * class TestPOSIXDevice(TestPOSIXTool): @@ -97,8 +97,8 @@ class TestPOSIXDevice(TestPOSIXTool): self.assertTrue(ptool.install(entry)) mock_exists.assert_called_with(entry, remove=True) mock_makedev.assert_called_with(0, 10) - mock_mknod.assert_called_with(entry.get("name"), - device_map[entry.get("dev_type")] | 0644, + mock_mknod.assert_called_with(entry.get("name"), # 0o644 + device_map[entry.get("dev_type")] | 420, mock_makedev.return_value) mock_install.assert_called_with(ptool, entry) @@ -127,6 +127,6 @@ class TestPOSIXDevice(TestPOSIXTool): self.assertTrue(ptool.install(entry)) mock_exists.assert_called_with(entry, remove=True) - mock_mknod.assert_called_with(entry.get("name"), - device_map[entry.get("dev_type")] | 0644) + mock_mknod.assert_called_with(entry.get("name"), # 0o644 + device_map[entry.get("dev_type")] | 420) mock_install.assert_called_with(ptool, entry) diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py index 34c98fa9c..79879afed 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestDirectory.py @@ -5,8 +5,8 @@ import unittest import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Client.Tools.POSIX.Directory import * -from Test__init import get_posix_object -from Testbase import TestPOSIXTool +from .Test__init import get_posix_object +from .Testbase import TestPOSIXTool from .....common import * class TestPOSIXDirectory(TestPOSIXTool): @@ -25,14 +25,14 @@ class TestPOSIXDirectory(TestPOSIXTool): mock_exists.reset_mock() exists_rv = MagicMock() - exists_rv.__getitem__.return_value = stat.S_IFREG | 0644 + exists_rv.__getitem__.return_value = stat.S_IFREG | 420 # 0o644 mock_exists.return_value = exists_rv self.assertFalse(ptool.verify(entry, [])) mock_exists.assert_called_with(entry) mock_exists.reset_mock() mock_verify.return_value = False - exists_rv.__getitem__.return_value = stat.S_IFDIR | 0644 + exists_rv.__getitem__.return_value = stat.S_IFDIR | 420 # 0o644 self.assertFalse(ptool.verify(entry, [])) mock_exists.assert_called_with(entry) mock_verify.assert_called_with(ptool, entry, []) @@ -105,7 +105,7 @@ class TestPOSIXDirectory(TestPOSIXTool): reset() exists_rv = MagicMock() - exists_rv.__getitem__.return_value = stat.S_IFREG | 0644 + exists_rv.__getitem__.return_value = stat.S_IFREG | 420 # 0o644 mock_exists.return_value = exists_rv self.assertTrue(ptool.install(entry)) mock_unlink.assert_called_with(entry.get("name")) @@ -114,7 +114,7 @@ class TestPOSIXDirectory(TestPOSIXTool): mock_install.assert_called_with(ptool, entry) reset() - exists_rv.__getitem__.return_value = stat.S_IFDIR | 0644 + exists_rv.__getitem__.return_value = stat.S_IFDIR | 420 # 0o644 mock_install.return_value = True self.assertTrue(ptool.install(entry)) mock_exists.assert_called_with(entry) diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py index 06912c19c..084268959 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py @@ -5,10 +5,11 @@ import difflib import binascii import unittest import lxml.etree +from Bcfg2.Bcfg2Py3k import b64encode, b64decode from mock import Mock, MagicMock, patch from Bcfg2.Client.Tools.POSIX.File import * -from Test__init import get_posix_object -from Testbase import TestPOSIXTool +from .Test__init import get_posix_object +from .Testbase import TestPOSIXTool from .....common import * def get_file_object(posix=None): @@ -33,17 +34,18 @@ class TestPOSIXFile(TestPOSIXTool): def test_is_string(self): ptool = self.get_obj() - for char in range(8) + range(14, 32): + for char in list(range(8)) + list(range(14, 32)): self.assertFalse(ptool._is_string("foo" + chr(char) + "bar", - 'utf_8')) - for char in range(9, 14) + range(33, 128): + 'UTF-8')) + for char in list(range(9, 14)) + list(range(33, 128)): self.assertTrue(ptool._is_string("foo" + chr(char) + "bar", - 'utf_8')) - self.assertFalse(ptool._is_string("foo" + chr(128) + "bar", - 'ascii')) + 'UTF-8')) ustr = 'é' - self.assertTrue(ptool._is_string(ustr, 'utf_8')) - self.assertFalse(ptool._is_string(ustr, 'ascii')) + self.assertTrue(ptool._is_string(ustr, 'UTF-8')) + if not inPy3k: + self.assertFalse(ptool._is_string("foo" + chr(128) + "bar", + 'ascii')) + self.assertFalse(ptool._is_string(ustr, 'ascii')) def test_get_data(self): orig_entry = lxml.etree.Element("Path", name="/test", type="file") @@ -51,7 +53,7 @@ class TestPOSIXFile(TestPOSIXTool): ptool = self.get_obj(posix=get_posix_object(setup=setup)) entry = copy.deepcopy(orig_entry) - entry.text = binascii.b2a_base64("test") + entry.text = b64encode("test") entry.set("encoding", "base64") self.assertEqual(ptool._get_data(entry), ("test", True)) @@ -63,7 +65,7 @@ class TestPOSIXFile(TestPOSIXTool): entry.text = "test" self.assertEqual(ptool._get_data(entry), ("test", False)) - ustr = u'é' + ustr = u('é') entry = copy.deepcopy(orig_entry) entry.text = ustr self.assertEqual(ptool._get_data(entry), (ustr, False)) @@ -207,7 +209,7 @@ class TestPOSIXFile(TestPOSIXTool): mock_rename.assert_called_with(newfile, entry.get("name")) mock_unlink.assert_called_with(newfile) - @patch("%.open" % builtins) + @patch("%s.open" % builtins) @patch("Bcfg2.Client.Tools.POSIX.File.%s._diff" % test_obj.__name__) @patch("Bcfg2.Client.Tools.POSIX.File.%s._get_data" % test_obj.__name__) @patch("Bcfg2.Client.Tools.POSIX.File.%s._is_string" % test_obj.__name__) @@ -239,8 +241,7 @@ class TestPOSIXFile(TestPOSIXTool): mock_open.assert_called_with(entry.get("name")) mock_open.return_value.read.assert_any_call() self.assertFalse(mock_diff.called) - self.assertEqual(entry.get("current_bfile"), - binascii.b2a_base64(ondisk)) + self.assertEqual(entry.get("current_bfile"), b64encode(ondisk)) # binary data on disk entry = reset() @@ -248,8 +249,7 @@ class TestPOSIXFile(TestPOSIXTool): ptool._get_diffs(entry, content=ondisk) self.assertFalse(mock_open.called) self.assertFalse(mock_diff.called) - self.assertEqual(entry.get("current_bfile"), - binascii.b2a_base64(ondisk)) + self.assertEqual(entry.get("current_bfile"), b64encode(ondisk)) # sensitive, non-interactive -- do nothing entry = reset() @@ -278,7 +278,7 @@ class TestPOSIXFile(TestPOSIXTool): filename=entry.get("name")) self.assertIsNone(entry.get("qtext")) self.assertEqual(entry.get("current_bdiff"), - binascii.b2a_base64("\n".join(mock_diff.return_value))) + b64encode("\n".join(mock_diff.return_value))) del entry.attrib["current_bdiff"] self.assertItemsEqual(orig_entry.attrib, entry.attrib) @@ -297,14 +297,14 @@ class TestPOSIXFile(TestPOSIXTool): self.assertIsNotNone(entry.get("qtext")) self.assertTrue(entry.get("qtext").startswith("test\n")) self.assertEqual(entry.get("current_bdiff"), - binascii.b2a_base64("\n".join(mock_diff.return_value))) + b64encode("\n".join(mock_diff.return_value))) del entry.attrib['qtext'] del entry.attrib["current_bdiff"] self.assertItemsEqual(orig_entry.attrib, entry.attrib) # non-sensitive, interactive with unicode data entry = reset() - entry.text = u"tëst" + entry.text = u("tëst") encoded = entry.text.encode(setup['encoding']) mock_get_data.return_value = (encoded, False) ptool._get_diffs(entry, interactive=True) @@ -317,7 +317,7 @@ class TestPOSIXFile(TestPOSIXTool): filename=entry.get("name"))]) self.assertIsNotNone(entry.get("qtext")) self.assertEqual(entry.get("current_bdiff"), - binascii.b2a_base64("\n".join(mock_diff.return_value))) + b64encode("\n".join(mock_diff.return_value))) del entry.attrib['qtext'] del entry.attrib["current_bdiff"] self.assertItemsEqual(orig_entry.attrib, entry.attrib) diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestHardlink.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestHardlink.py index 663c98af2..221da9ea5 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestHardlink.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestHardlink.py @@ -4,8 +4,8 @@ import unittest import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Client.Tools.POSIX.Hardlink import * -from Test__init import get_posix_object -from Testbase import TestPOSIXTool +from .Test__init import get_posix_object +from .Testbase import TestPOSIXTool from .....common import * class TestPOSIXHardlink(TestPOSIXTool): diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py index 8d959a15f..2b46bd9e6 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestNonexistent.py @@ -4,8 +4,8 @@ import unittest import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Client.Tools.POSIX.Nonexistent import * -from Test__init import get_config, get_posix_object -from Testbase import TestPOSIXTool +from .Test__init import get_config, get_posix_object +from .Testbase import TestPOSIXTool from .....common import * class TestPOSIXNonexistent(TestPOSIXTool): diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestPermissions.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestPermissions.py index 9d8130658..008f0c839 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestPermissions.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestPermissions.py @@ -3,8 +3,8 @@ import unittest import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Client.Tools.POSIX.Permissions import * -from Test__init import get_posix_object -from Testbase import TestPOSIXTool +from .Test__init import get_posix_object +from .Testbase import TestPOSIXTool from .....common import * class TestPOSIXPermissions(TestPOSIXTool): diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestSymlink.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestSymlink.py index 03504ca61..7ce8a4437 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestSymlink.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/TestSymlink.py @@ -4,8 +4,8 @@ import unittest import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Client.Tools.POSIX.Symlink import * -from Test__init import get_posix_object -from Testbase import TestPOSIXTool +from .Test__init import get_posix_object +from .Testbase import TestPOSIXTool from .....common import * class TestPOSIXSymlink(TestPOSIXTool): diff --git a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index 442641c0b..e8e5dbe23 100644 --- a/testsuite/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -6,7 +6,7 @@ import lxml.etree from mock import Mock, MagicMock, patch import Bcfg2.Client.Tools from Bcfg2.Client.Tools.POSIX.base import * -from Test__init import get_posix_object +from .Test__init import get_posix_object from .....common import * try: @@ -240,13 +240,13 @@ class TestPOSIXTool(Bcfg2TestCase): raise KeyError os.chown.side_effect = chown_rv entry.set("type", "device") - entry.set("dev_type", device_map.keys()[0]) + entry.set("dev_type", list(device_map.keys())[0]) self.assertFalse(ptool._set_perms(entry)) mock_norm_uid.assert_called_with(entry) mock_norm_gid.assert_called_with(entry) mock_chown.assert_called_with(entry.get("name"), 0, 0) mock_chmod.assert_called_with(entry.get("name"), - int(entry.get("perms"), 8) | device_map.values()[0]) + int(entry.get("perms"), 8) | list(device_map.values())[0]) mock_utime.assert_called_with(entry.get("name"), (mtime, mtime)) mock_set_secontext.assert_called_with(entry, path=entry.get("name")) mock_set_acls.assert_called_with(entry, path=entry.get("name")) diff --git a/testsuite/Testlib/TestServer/TestPlugin.py b/testsuite/Testlib/TestServer/TestPlugin.py index d1904a2e6..c00a7d91d 100644 --- a/testsuite/Testlib/TestServer/TestPlugin.py +++ b/testsuite/Testlib/TestServer/TestPlugin.py @@ -4,9 +4,10 @@ import copy import logging import unittest import lxml.etree +import Bcfg2.Server +from Bcfg2.Bcfg2Py3k import reduce from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugin import * -import Bcfg2.Server from ...common import * class FakeElementTree(lxml.etree._ElementTree): @@ -228,8 +229,9 @@ class TestThreadedStatistics(TestStatistics): mock_start.assert_any_call() @patch("%s.open" % builtins) + @patch("%s.dump" % cPickle.__name__) @patch("Bcfg2.Server.Plugin.ThreadedStatistics.run", Mock()) - def test_save(self, mock_open): + def test_save(self, mock_dump, mock_open): core = Mock() ts = self.get_obj(core) queue = Mock() @@ -255,78 +257,76 @@ class TestThreadedStatistics(TestStatistics): queue.get_nowait = Mock(side_effect=lambda: queue.data.pop()) mock_open.side_effect = None - # oh, the joy of working around different package names in - # py3k... - with patch("%s.dump" % cPickle.__name__) as mock_dump: - ts.save() - queue.empty.assert_any_call() - queue.get_nowait.assert_any_call() - mock_open.assert_called_with(ts.pending_file, 'w') - mock_open.return_value.close.assert_any_call() - # the order of the queue data gets changed, so we have to - # verify this call in an ugly way - self.assertItemsEqual(mock_dump.call_args[0][0], self.data) - self.assertEqual(mock_dump.call_args[0][1], mock_open.return_value) + ts.save() + queue.empty.assert_any_call() + queue.get_nowait.assert_any_call() + mock_open.assert_called_with(ts.pending_file, 'w') + mock_open.return_value.close.assert_any_call() + # the order of the queue data gets changed, so we have to + # verify this call in an ugly way + self.assertItemsEqual(mock_dump.call_args[0][0], self.data) + self.assertEqual(mock_dump.call_args[0][1], mock_open.return_value) @patch("os.unlink") @patch("os.path.exists") @patch("%s.open" % builtins) @patch("lxml.etree.XML") + @patch("%s.load" % cPickle.__name__) @patch("Bcfg2.Server.Plugin.ThreadedStatistics.run", Mock()) - def test_load(self, mock_XML, mock_open, mock_exists, mock_unlink): + def test_load(self, mock_load, mock_XML, mock_open, mock_exists, + mock_unlink): core = Mock() core.terminate.isSet.return_value = False ts = self.get_obj(core) - with patch("%s.load" % cPickle.__name__) as mock_load: - ts.work_queue = Mock() + ts.work_queue = Mock() + ts.work_queue.data = [] + def reset(): + core.reset_mock() + mock_open.reset_mock() + mock_exists.reset_mock() + mock_unlink.reset_mock() + mock_load.reset_mock() + mock_XML.reset_mock() + ts.work_queue.reset_mock() ts.work_queue.data = [] - def reset(): - core.reset_mock() - mock_open.reset_mock() - mock_exists.reset_mock() - mock_unlink.reset_mock() - mock_load.reset_mock() - mock_XML.reset_mock() - ts.work_queue.reset_mock() - ts.work_queue.data = [] - - mock_exists.return_value = False - self.assertTrue(ts.load()) - mock_exists.assert_called_with(ts.pending_file) - reset() - mock_exists.return_value = True - mock_open.side_effect = OSError - self.assertFalse(ts.load()) - mock_exists.assert_called_with(ts.pending_file) - mock_open.assert_called_with(ts.pending_file, 'r') + mock_exists.return_value = False + self.assertTrue(ts.load()) + mock_exists.assert_called_with(ts.pending_file) - reset() - mock_open.side_effect = None - mock_load.return_value = self.data - ts.work_queue.put_nowait.side_effect = Full - self.assertTrue(ts.load()) - mock_exists.assert_called_with(ts.pending_file) - mock_open.assert_called_with(ts.pending_file, 'r') - mock_open.return_value.close.assert_any_call() - mock_load.assert_called_with(mock_open.return_value) + reset() + mock_exists.return_value = True + mock_open.side_effect = OSError + self.assertFalse(ts.load()) + mock_exists.assert_called_with(ts.pending_file) + mock_open.assert_called_with(ts.pending_file, 'r') - reset() - core.build_metadata.side_effect = lambda x: x - mock_XML.side_effect = lambda x, parser=None: x - ts.work_queue.put_nowait.side_effect = None - self.assertTrue(ts.load()) - mock_exists.assert_called_with(ts.pending_file) - mock_open.assert_called_with(ts.pending_file, 'r') - mock_open.return_value.close.assert_any_call() - mock_load.assert_called_with(mock_open.return_value) - self.assertItemsEqual(mock_XML.call_args_list, - [call(x, parser=Bcfg2.Server.XMLParser) - for h, x in self.data]) - self.assertItemsEqual(ts.work_queue.put_nowait.call_args_list, - [call((h, x)) for h, x in self.data]) - mock_unlink.assert_called_with(ts.pending_file) + reset() + mock_open.side_effect = None + mock_load.return_value = self.data + ts.work_queue.put_nowait.side_effect = Full + self.assertTrue(ts.load()) + mock_exists.assert_called_with(ts.pending_file) + mock_open.assert_called_with(ts.pending_file, 'r') + mock_open.return_value.close.assert_any_call() + mock_load.assert_called_with(mock_open.return_value) + + reset() + core.build_metadata.side_effect = lambda x: x + mock_XML.side_effect = lambda x, parser=None: x + ts.work_queue.put_nowait.side_effect = None + self.assertTrue(ts.load()) + mock_exists.assert_called_with(ts.pending_file) + mock_open.assert_called_with(ts.pending_file, 'r') + mock_open.return_value.close.assert_any_call() + mock_load.assert_called_with(mock_open.return_value) + self.assertItemsEqual(mock_XML.call_args_list, + [call(x, parser=Bcfg2.Server.XMLParser) + for h, x in self.data]) + self.assertItemsEqual(ts.work_queue.put_nowait.call_args_list, + [call((h, x)) for h, x in self.data]) + mock_unlink.assert_called_with(ts.pending_file) @patch("threading.Thread.start", Mock()) @patch("Bcfg2.Server.Plugin.ThreadedStatistics.load") @@ -1018,7 +1018,7 @@ class TestStructFile(TestXMLFileBacked): mock_match.side_effect = match_rv actual = sf.Match(metadata) expected = reduce(lambda x, y: x + y, - children.values() + subgroups.values()) + list(children.values()) + list(subgroups.values())) self.assertEqual(len(actual), len(expected)) # easiest way to compare the values is actually to make # them into an XML document and let assertXMLEqual compare @@ -1047,7 +1047,7 @@ class TestStructFile(TestXMLFileBacked): expected = lxml.etree.Element(xdata.tag, **xdata.attrib) expected.text = xdata.text expected.extend(reduce(lambda x, y: x + y, - children.values() + subchildren.values())) + list(children.values()) + list(subchildren.values()))) expected.extend(standalone) self.assertXMLEqual(actual, expected) @@ -1064,7 +1064,7 @@ class TestStructFile(TestXMLFileBacked): for call in mock_xml_match.call_args_list: actual.append(call[0][0]) self.assertEqual(call[0][1], metadata) - expected = groups.values() + standalone + expected = list(groups.values()) + standalone # easiest way to compare the values is actually to make # them into an XML document and let assertXMLEqual compare # them diff --git a/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py index a779a7707..df9248e50 100644 --- a/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testlib/TestServer/TestPlugins/TestProbes.py @@ -53,7 +53,7 @@ class TestProbeData(Bcfg2TestCase): def test_xdata(self): xdata = lxml.etree.Element("test") lxml.etree.SubElement(xdata, "test2") - data = ProbeData(lxml.etree.tostring(xdata)) + data = ProbeData(lxml.etree.tostring(xdata, encoding='unicode')) self.assertIsNotNone(data.xdata) self.assertIsNotNone(data.xdata.find("test2")) @@ -193,7 +193,7 @@ class TestProbes(TestProbing, TestConnector, TestDatabaseBacked): rv = dict() rv["foo.example.com"] = ClientProbeDataSet(timestamp=time.time()) rv["foo.example.com"]["xml"] = \ - ProbeData(lxml.etree.tostring(test_xdata)) + ProbeData(lxml.etree.tostring(test_xdata, encoding='unicode')) rv["foo.example.com"]["text"] = ProbeData("freeform text") rv["foo.example.com"]["multiline"] = ProbeData("""multiple lines @@ -277,7 +277,8 @@ text mock_open.assert_called_with(os.path.join(datastore, probes.name, "probed.xml"), "w") - data = lxml.etree.XML(str(mock_open.return_value.write.call_args[0][0])) + print("data=%s" % mock_open.return_value.write.call_args[0][0]) + data = lxml.etree.XML(mock_open.return_value.write.call_args[0][0]) self.assertEqual(len(data.xpath("//Client")), 2) foodata = data.find("Client[@name='foo.example.com']") @@ -290,6 +291,7 @@ text xml = foodata.find("Probe[@name='xml']") self.assertIsNotNone(xml) self.assertIsNotNone(xml.get("value")) + print("value=%s" % xml.get("value")) xdata = lxml.etree.XML(xml.get("value")) self.assertIsNotNone(xdata) self.assertIsNotNone(xdata.find("test")) diff --git a/testsuite/common.py b/testsuite/common.py index cc24112e7..98ee0c6f5 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -37,45 +37,16 @@ except ImportError: if inPy3k: builtins = "builtins" + + def u(x): + return x else: builtins = "__builtin__" -if hasattr(unittest.TestCase, "assertItemsEqual"): - TestCase = unittest.TestCase -else: - def assertion(predicate, default_msg=None): - @wraps(predicate) - def inner(*args, **kwargs): - if 'msg' in kwargs: - msg = kwargs['msg'] - del kwargs['msg'] - else: - msg = default_msg % args - assert predicate(*args, **kwargs), msg - return inner + import codecs + def u(x): + return codecs.unicode_escape_decode(x)[0] - class TestCase(unittest.TestCase): - # versions of TestCase before python 2.7 lacked a lot of the - # really handy convenience methods, so we provide them -- at - # least the easy ones and the ones we use. - assertIs = assertion(lambda a, b: a is b, "%s is not %s") - assertIsNot = assertion(lambda a, b: a is not b, "%s is %s") - assertIsNone = assertion(lambda x: x is None, "%s is not None") - assertIsNotNone = assertion(lambda x: x is not None, "%s is None") - assertIn = assertion(lambda a, b: a in b, "%s is not in %s") - assertNotIn = assertion(lambda a, b: a not in b, "%s is in %s") - assertIsInstance = assertion(isinstance, "%s is not %s") - assertNotIsInstance = assertion(lambda a, b: not isinstance(a, b), - "%s is %s") - assertGreater = assertion(lambda a, b: a > b, - "%s is not greater than %s") - assertGreaterEqual = assertion(lambda a, b: a >= b, - "%s is not greater than or equal to %s") - assertLess = assertion(lambda a, b: a < b, "%s is not less than %s") - assertLessEqual = assertion(lambda a, b: a <= b, - "%s is not less than or equal to %s") - assertItemsEqual = assertion(lambda a, b: sorted(a) == sorted(b), - "Items do not match:\n%s\n%s") if hasattr(unittest, "skip"): can_skip = True @@ -117,7 +88,129 @@ else: return decorator -class Bcfg2TestCase(TestCase): +needs_assertItemsEqual = False +needs_others = False +if not hasattr(unittest.TestCase, "assertItemsEqual"): + # TestCase in Py3k lacks assertItemsEqual, but has the other + # convenience methods. this code is cribbed from the py2.7 + # unittest library + import collections + needs_assertItemsEqual = True + + def _count_diff_all_purpose(actual, expected): + '''Returns list of (cnt_act, cnt_exp, elem) triples where the + counts differ''' + # elements need not be hashable + s, t = list(actual), list(expected) + m, n = len(s), len(t) + NULL = object() + result = [] + for i, elem in enumerate(s): + if elem is NULL: + continue + cnt_s = cnt_t = 0 + for j in range(i, m): + if s[j] == elem: + cnt_s += 1 + s[j] = NULL + for j, other_elem in enumerate(t): + if other_elem == elem: + cnt_t += 1 + t[j] = NULL + if cnt_s != cnt_t: + diff = _Mismatch(cnt_s, cnt_t, elem) + result.append(diff) + + for i, elem in enumerate(t): + if elem is NULL: + continue + cnt_t = 0 + for j in range(i, n): + if t[j] == elem: + cnt_t += 1 + t[j] = NULL + diff = _Mismatch(0, cnt_t, elem) + result.append(diff) + return result + + def _count_diff_hashable(actual, expected): + '''Returns list of (cnt_act, cnt_exp, elem) triples where the + counts differ''' + # elements must be hashable + s, t = _ordered_count(actual), _ordered_count(expected) + result = [] + for elem, cnt_s in s.items(): + cnt_t = t.get(elem, 0) + if cnt_s != cnt_t: + diff = _Mismatch(cnt_s, cnt_t, elem) + result.append(diff) + for elem, cnt_t in t.items(): + if elem not in s: + diff = _Mismatch(0, cnt_t, elem) + result.append(diff) + return result + +if not hasattr(unittest.TestCase, "assertIn"): + # versions of TestCase before python 2.7 and python 3.1 lacked a + # lot of the really handy convenience methods, so we provide them + # -- at least the easy ones and the ones we use. + needs_others = True + + def _assertion(predicate, default_msg=None): + @wraps(predicate) + def inner(self, *args, **kwargs): + if 'msg' in kwargs: + msg = kwargs['msg'] + del kwargs['msg'] + else: + msg = default_msg % args + assert predicate(*args, **kwargs), msg + return inner + + +class Bcfg2TestCase(unittest.TestCase): + if needs_assertItemsEqual: + def assertItemsEqual(self, expected_seq, actual_seq, msg=None): + first_seq, second_seq = list(actual_seq), list(expected_seq) + try: + first = collections.Counter(first_seq) + second = collections.Counter(second_seq) + except TypeError: + # Handle case with unhashable elements + differences = _count_diff_all_purpose(first_seq, second_seq) + else: + if first == second: + return + differences = _count_diff_hashable(first_seq, second_seq) + + if differences: + standardMsg = 'Element counts were not equal:\n' + lines = ['First has %d, Second has %d: %r' % diff + for diff in differences] + diffMsg = '\n'.join(lines) + standardMsg = self._truncateMessage(standardMsg, diffMsg) + msg = self._formatMessage(msg, standardMsg) + self.fail(msg) + + if needs_others: + assertIs = _assertion(lambda a, b: a is b, "%s is not %s") + assertIsNot = _assertion(lambda a, b: a is not b, "%s is %s") + assertIsNone = _assertion(lambda x: x is None, "%s is not None") + assertIsNotNone = _assertion(lambda x: x is not None, "%s is None") + assertIn = _assertion(lambda a, b: a in b, "%s is not in %s") + assertNotIn = _assertion(lambda a, b: a not in b, "%s is in %s") + assertIsInstance = _assertion(isinstance, "%s is not instance of %s") + assertNotIsInstance = _assertion(lambda a, b: not isinstance(a, b), + "%s is instance of %s") + assertGreater = _assertion(lambda a, b: a > b, + "%s is not greater than %s") + assertGreaterEqual = _assertion(lambda a, b: a >= b, + "%s is not greater than or equal to %s") + assertLess = _assertion(lambda a, b: a < b, "%s is not less than %s") + assertLessEqual = _assertion(lambda a, b: a <= b, + "%s is not less than or equal to %s") + + def assertXMLEqual(self, el1, el2, msg=None): self.assertEqual(el1.tag, el2.tag, msg=msg) self.assertEqual(el1.text, el2.text, msg=msg) |