Apply changes from the magically working debian package

10 files changed, 79 insertions, 91 deletions

diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py
index 9ffae49d8..dd841dd08 100644
--- a/src/lib/Bcfg2/Client/Proxy.py
+++ b/src/lib/Bcfg2/Client/Proxy.py
@@ -21,7 +21,9 @@ except ImportError:
 version = sys.version_info[:2]
 has_py26 = version >= (2, 6)
 has_py32 = version >= (3, 2)
+has_py34 = version >= (3, 4)
 has_py36 = version >= (3, 6)
+has_py310 = version >= (3, 10)
 
 __all__ = ["ComponentProxy",
            "RetryMethod",
@@ -200,8 +202,12 @@ class SSLHTTPConnection(httplib.HTTPConnection):
         elif self.protocol == 'xmlrpc/tlsv1':
             ssl_protocol_ver = ssl.PROTOCOL_TLSv1
         elif self.protocol == 'xmlrpc/tls':
-            if has_py36:
+            if has_py310:
+                ssl_protocol_ver = ssl.PROTOCOL_TLS_SERVER
+            elif has_py36:
                 ssl_protocol_ver = ssl.PROTOCOL_TLS
+            elif has_py34:
+                ssl_protocol_ver = ssl.PROTOCOL_TLSv1_2
             else:
                 self.logger.warning("Cannot use PROTOCOL_TLS, due to "
                                     "python version. Switching to "
@@ -229,9 +235,9 @@ class SSLHTTPConnection(httplib.HTTPConnection):
 
         rawsock.settimeout(self.timeout)
         self.sock = ssl.wrap_socket(rawsock, cert_reqs=other_side_required,
-                                  ca_certs=self.ca, suppress_ragged_eofs=True,
-                                  keyfile=self.key, certfile=self.cert,
-                                  ssl_version=ssl_protocol_ver)
+                                    ca_certs=self.ca, suppress_ragged_eofs=True,
+                                    keyfile=self.key, certfile=self.cert,
+                                    ssl_version=ssl_protocol_ver)
         self.sock.connect((self.host, self.port))
         peer_cert = self.sock.getpeercert()
         if peer_cert and self.scns:
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py
index bcd695058..d6d1d43e4 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py
@@ -217,7 +217,7 @@ class POSIXAugeas(POSIXTool):
         return self._augeas[entry.get("name")]
 
     def fully_specified(self, entry):
-        return len(entry.getchildren()) != 0
+        return len(entry) != 0
 
     def get_commands(self, entry):
         """ Get a list of commands to verify or install.
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py
index 1f1772d46..c68c57995 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py
@@ -5,9 +5,10 @@ import sys
 import stat
 import difflib
 import tempfile
+from base64 import b64encode, b64decode
+
 import Bcfg2.Options
 from Bcfg2.Client.Tools.POSIX.base import POSIXTool
-from Bcfg2.Compat import unicode, b64encode, b64decode  # pylint: disable=W0622
 import Bcfg2.Utils
 
 
@@ -22,27 +23,23 @@ class POSIXFile(POSIXTool):
         """ Get a tuple of (<file data>, <is binary>) for the given entry """
         is_binary = entry.get('encoding', 'ascii') == 'base64'
         if entry.get('empty', 'false') == 'true' or not entry.text:
-            tempdata = ''
+            tempdata = b''
         elif is_binary:
-            tempdata = b64decode(entry.text)
+            tempdata = b64decode(entry.text.encode('ascii'))
         else:
             tempdata = entry.text
-            if isinstance(tempdata, unicode) and unicode != str:
-                try:
-                    tempdata = tempdata.encode(Bcfg2.Options.setup.encoding)
-                except UnicodeEncodeError:
-                    err = sys.exc_info()[1]
-                    self.logger.error("POSIX: Error encoding file %s: %s" %
-                                      (entry.get('name'), err))
+            try:
+                tempdata = tempdata.encode(Bcfg2.Options.setup.encoding)
+            except UnicodeEncodeError:
+                err = sys.exc_info()[1]
+                self.logger.error("POSIX: Error encoding file %s: %s" %
+                                  (entry.get('name'), err))
         return (tempdata, is_binary)
 
     def verify(self, entry, modlist):
         ondisk = self._exists(entry)
         tempdata, is_binary = self._get_data(entry)
-        if isinstance(tempdata, str) and str != unicode:
-            tempdatasize = len(tempdata)
-        else:
-            tempdatasize = len(tempdata.encode(Bcfg2.Options.setup.encoding))
+        tempdatasize = len(tempdata)
 
         different = False
         content = None
@@ -50,7 +47,7 @@ class POSIXFile(POSIXTool):
             # first, see if the target file exists at all; if not,
             # they're clearly different
             different = True
-            content = ""
+            content = b''
         elif tempdatasize != ondisk[stat.ST_SIZE]:
             # next, see if the size of the target file is different
             # from the size of the desired content
@@ -61,10 +58,7 @@ class POSIXFile(POSIXTool):
             # which might be faster for big binary files, but slower
             # for everything else
             try:
-                content = open(entry.get('name')).read()
-            except UnicodeDecodeError:
-                content = open(entry.get('name'),
-                               encoding=Bcfg2.Options.setup.encoding).read()
+                content = open(entry.get('name'), 'rb').read()
             except IOError:
                 self.logger.error("POSIX: Failed to read %s: %s" %
                                   (entry.get("name"), sys.exc_info()[1]))
@@ -98,11 +92,7 @@ class POSIXFile(POSIXTool):
                               (os.path.dirname(entry.get('name')), err))
             return False
         try:
-            if isinstance(filedata, str) and str != unicode:
-                os.fdopen(newfd, 'w').write(filedata)
-            else:
-                os.fdopen(newfd, 'wb').write(
-                    filedata.encode(Bcfg2.Options.setup.encoding))
+            os.fdopen(newfd, 'wb').write(filedata)
         except (OSError, IOError):
             err = sys.exc_info()[1]
             self.logger.error("POSIX: Failed to open temp file %s for writing "
@@ -159,33 +149,30 @@ class POSIXFile(POSIXTool):
             # binary, and either include that fact or the diff in our
             # prompts for -I and the reports
             try:
-                content = open(entry.get('name')).read()
-            except UnicodeDecodeError:
-                content = open(entry.get('name'), encoding='utf-8').read()
+                content = open(entry.get('name'), 'rb').read()
             except IOError:
                 self.logger.error("POSIX: Failed to read %s: %s" %
                                   (entry.get("name"), sys.exc_info()[1]))
                 return False
+
+        content_new = self._get_data(entry)[0]
         if not is_binary:
-            is_binary |= not Bcfg2.Utils.is_string(
-                content, Bcfg2.Options.setup.encoding)
+            try:
+                text_content = content.decode(Bcfg2.Options.setup.encoding)
+                text_content_new = content_new.decode(Bcfg2.Options.setup.encoding)
+            except UnicodeDecodeError:
+                is_binary = True
         if is_binary:
             # don't compute diffs if the file is binary
             prompt.append('Binary file, no printable diff')
-            attrs['current_bfile'] = b64encode(content)
+            attrs['current_bfile'] = b64encode(content).decode('ascii')
         else:
-            diff = self._diff(content, self._get_data(entry)[0],
+            diff = self._diff(text_content, text_content_new,
                               filename=entry.get("name"))
             if interactive:
                 if diff:
-                    udiff = '\n'.join(diff)
-                    if hasattr(udiff, "decode"):
-                        udiff = udiff.decode(Bcfg2.Options.setup.encoding)
-                    try:
-                        prompt.append(udiff)
-                    except UnicodeEncodeError:
-                        prompt.append("Could not encode diff")
-                elif entry.get("empty", "true"):
+                    prompt.append(diff)
+                elif entry.get("empty", 'false') == 'true':
                     # the file doesn't exist on disk, but there's no
                     # expected content
                     prompt.append("%s does not exist" % entry.get("name"))
@@ -194,9 +181,10 @@ class POSIXFile(POSIXTool):
                                   "printable diff")
             if not sensitive:
                 if diff:
-                    attrs["current_bdiff"] = b64encode("\n".join(diff))
+                    attrs["current_bdiff"] = b64encode(diff.encode(
+                        Bcfg2.Options.setup.encoding)).decode('ascii')
                 else:
-                    attrs['current_bfile'] = b64encode(content)
+                    attrs['current_bfile'] = b64encode(content).decode('ascii')
         if interactive:
             entry.set("qtext", "\n".join(prompt))
         if not sensitive:
@@ -212,7 +200,10 @@ class POSIXFile(POSIXTool):
         else:
             fromfile = ""
             tofile = ""
-        return difflib.unified_diff(content1.split('\n'),
-                                    content2.split('\n'),
-                                    fromfile=fromfile,
-                                    tofile=tofile)
+
+        diff = difflib.unified_diff(
+            content1.splitlines(True), content2.splitlines(True),
+            fromfile=fromfile, tofile=tofile)
+        if diff:
+            return ''.join(diff)
+        return None
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py
index d67a68c8b..4221cc615 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py
@@ -21,8 +21,8 @@ class POSIXNonexistent(POSIXTool):
         recursive = entry.get('recursive', '').lower() == 'true'
         if recursive:
             # ensure that configuration spec is consistent first
-            for struct in self.config.getchildren():
-                for el in struct.getchildren():
+            for struct in self.config:
+                for el in struct:
                     if (el.tag == 'Path' and
                             el.get('type') != 'nonexistent' and
                             el.get('name').startswith(ename)):
diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
index 224119a79..004f85682 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
@@ -121,7 +121,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
 
     def Inventory(self, structures=None):
         if not structures:
-            structures = self.config.getchildren()
+            structures = self.config
         # we calculate a list of all POSIXUser and POSIXGroup entries,
         # and then add POSIXGroup entries that are required to create
         # the primary group for each user to the structures.  this is
@@ -223,8 +223,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
         else:
             for attr, idx in self.attr_mapping[entry.tag].items():
                 val = str(self.existing[entry.tag][entry.get("name")][idx])
-                entry.set("current_%s" %
-                          attr, val.decode(Bcfg2.Options.setup.encoding))
+                entry.set("current_%s" % attr, val)
                 if attr in ["uid", "gid"]:
                     if entry.get(attr) is None:
                         # no uid/gid specified, so we let the tool
diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py
index 2e3421840..c84b6aac8 100644
--- a/src/lib/Bcfg2/Client/Tools/__init__.py
+++ b/src/lib/Bcfg2/Client/Tools/__init__.py
@@ -190,12 +190,12 @@ class Tool(object):
         :returns: dict - A dict of the state of entries suitable for
                   updating :attr:`Bcfg2.Client.Client.states`
         """
-        if not structures:
-            structures = self.config.getchildren()
+        if structures is None:
+            structures = self.config
         mods = self.buildModlist()
         states = dict()
         for struct in structures:
-            for entry in struct.getchildren():
+            for entry in struct:
                 if self.canVerify(entry):
                     try:
                         func = getattr(self, "Verify%s" % entry.tag)
@@ -261,8 +261,8 @@ class Tool(object):
 
         :returns: list of lxml.etree._Element """
         rv = []
-        for struct in self.config.getchildren():
-            rv.extend([entry for entry in struct.getchildren()
+        for struct in self.config:
+            rv.extend([entry for entry in struct
                        if self.handlesEntry(entry)])
         return rv
 
@@ -282,8 +282,8 @@ class Tool(object):
 
         :returns: list of lxml.etree._Element """
         rv = []
-        for struct in self.config.getchildren():
-            rv.extend([entry.get('name') for entry in struct.getchildren()
+        for struct in self.config:
+            rv.extend([entry.get('name') for entry in struct
                        if entry.tag == 'Path'])
         return rv
 
diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py
index a7e0dade5..14c14a224 100644
--- a/src/lib/Bcfg2/Client/__init__.py
+++ b/src/lib/Bcfg2/Client/__init__.py
@@ -399,7 +399,7 @@ class Client(object):
 
         if Bcfg2.Options.setup.bundle_quick:
             newconfig = XML.XML('<Configuration/>')
-            for bundle in self.config.getchildren():
+            for bundle in self.config:
                 name = bundle.get("name")
                 if (name and (name in Bcfg2.Options.setup.only_bundles or
                               name not in Bcfg2.Options.setup.except_bundles)):
@@ -612,8 +612,8 @@ class Client(object):
 
         """
         # initialize all states
-        for struct in self.config.getchildren():
-            for entry in struct.getchildren():
+        for struct in self.config:
+            for entry in struct:
                 self.states[entry] = False
         for tool in self.tools:
             try:
@@ -665,7 +665,7 @@ class Client(object):
         # first perform bundle filtering
         all_bundle_names = [b.get('name')
                             for b in self.config.findall('./Bundle')]
-        bundles = self.config.getchildren()
+        bundles = self.config
         if Bcfg2.Options.setup.only_bundles:
             # warn if non-existent bundle given
             for bundle in Bcfg2.Options.setup.only_bundles:
diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py
index 9a8d76c47..2e13301f8 100644
--- a/src/lib/Bcfg2/Compat.py
+++ b/src/lib/Bcfg2/Compat.py
@@ -88,6 +88,13 @@ def u_str(string, encoding=None):
         else:
             return unicode(string)
 
+
+def ensure_binary(string, encoding='utf-8'):
+    if type(string) == unicode:
+        return string.encode(encoding)
+    return string
+
+
 try:
     from functools import wraps
 except ImportError:
@@ -95,24 +102,6 @@ except ImportError:
         """ implementation of functools.wraps() for python 2.4 """
         return lambda f: f
 
-
-# base64 compat
-if sys.hexversion >= 0x03000000:
-    from base64 import b64encode as _b64encode, b64decode as _b64decode
-
-    @wraps(_b64encode)
-    def b64encode(val, **kwargs):  # pylint: disable=C0111
-        try:
-            return _b64encode(val, **kwargs)
-        except TypeError:
-            return _b64encode(val.encode('UTF-8'), **kwargs).decode('UTF-8')
-
-    @wraps(_b64decode)
-    def b64decode(val, **kwargs):  # pylint: disable=C0111
-        return _b64decode(val.encode('UTF-8'), **kwargs).decode('UTF-8')
-else:
-    from base64 import b64encode, b64decode
-
 try:
     input = raw_input
 except NameError:
diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py
index 172f88f24..9b493944b 100644
--- a/src/lib/Bcfg2/DBSettings.py
+++ b/src/lib/Bcfg2/DBSettings.py
@@ -64,14 +64,14 @@ settings = dict(
         }
     })
 
-if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 6:
+if HAS_DJANGO and django.VERSION >= (1, 6):
     settings['MIDDLEWARE_CLASSES'] += \
         ('django.contrib.admindocs.middleware.XViewMiddleware',)
 elif HAS_SOUTH:
     settings['MIDDLEWARE_CLASSES'] += \
         ('django.middleware.doc.XViewMiddleware',)
 
-if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 7:
+if HAS_DJANGO and django.VERSION >= (1, 7):
     settings['INSTALLED_APPS'] += ('Bcfg2.Reporting',)
 elif HAS_SOUTH:
     settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting')
@@ -79,7 +79,7 @@ elif HAS_SOUTH:
         'Reporting': 'Bcfg2.Reporting.south_migrations',
         'Server': 'Bcfg2.Server.south_migrations',
     }
-if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 8:
+if HAS_DJANGO and django.VERSION >= (1, 8):
     settings['TEMPLATES'] = [{
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         'DIRS': [
@@ -164,7 +164,7 @@ def finalize_django_config(opts=None, silent=False):
         setattr(module, name, value)
     try:
         django.conf.settings.configure(**settings)
-        if django.VERSION[0] == 1 and django.VERSION[1] >= 7:
+        if django.VERSION >= (1, 7):
             django.setup()  # pylint: disable=E1101
     except RuntimeError:
         if not silent:
@@ -174,7 +174,7 @@ def finalize_django_config(opts=None, silent=False):
 
 def sync_databases(**kwargs):
     """ Synchronize all databases that we know about.  """
-    if django.VERSION[0] == 1 and django.VERSION[1] >= 7:
+    if django.VERSION >= (1, 7):
         # Nothing needed here, it's all handled with migrate
         return
 
@@ -226,7 +226,7 @@ def migrate_databases(**kwargs):
     logger = logging.getLogger()
     for database in settings['DATABASES']:
         logger.debug("Migrating database %s" % (database))
-        if django.VERSION[0] == 1 and django.VERSION[1] >= 7:
+        if django.VERSION >= (1, 7):
             if initial_django_migration(database):
                 logger.warning(
                     "No applied django migrations found for database %s. "
@@ -270,6 +270,9 @@ class PerApplicationRouter(object):
         """ Called when Django wants to determine what relations to allow. Only
         allow relations within an app """
         # pylint: disable=W0212
+        if obj1._meta.app_label in ('sites', 'contenttypes', 'auth'):
+            return True
+
         return obj1._meta.app_label == obj2._meta.app_label
         # pylint: enable=W0212
 
diff --git a/src/lib/Bcfg2/manage.py b/src/lib/Bcfg2/manage.py
index 9675a3db1..66cf93084 100755..100644
--- a/src/lib/Bcfg2/manage.py
+++ b/src/lib/Bcfg2/manage.py
@@ -18,7 +18,7 @@ def main():
         Bcfg2.Options.PositionalArgument('django_command', nargs='*')])
     parser.parse()
 
-    if django.VERSION[0] == 1 and django.VERSION[1] >= 6:
+    if django.VERSION >= (1, 6):
         from django.core.management import execute_from_command_line
         execute_from_command_line(
             sys.argv[:1] + Bcfg2.Options.setup.django_command)