Merge branch 'non-ascii-diffs' of https://github.com/weiss/bcfg2

7 files changed, 52 insertions, 16 deletions

diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py
index 60d158eb1..57844ab19 100644
--- a/src/lib/Client/Frame.py
+++ b/src/lib/Client/Frame.py
@@ -5,6 +5,7 @@ installs entries, and generates statistics.
 __revision__ = '$Revision$'
 
 import logging
+import sys
 import time
 import Bcfg2.Client.Tools
 
@@ -29,7 +30,7 @@ def promptFilter(prompt, entries):
         try:
             # py3k compatibility
             try:
-                ans = raw_input(iprompt)
+                ans = raw_input(iprompt.encode(sys.stdout.encoding, 'replace'))
             except NameError:
                 ans = input(iprompt)
             if ans in ['y', 'Y']:
diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py
index 875db5ea7..862e0bc04 100644
--- a/src/lib/Client/Tools/POSIX.py
+++ b/src/lib/Client/Tools/POSIX.py
@@ -76,6 +76,21 @@ def normUid(entry):
         return False
 
 
+def isString(strng, encoding):
+    """
+       Returns true if the string contains no ASCII control characters
+       and can be decoded from the specified encoding.
+    """
+    for char in strng:
+        if ord(char) < 9 or ord(char) > 13 and ord(char) < 32:
+            return False
+    try:
+        strng.decode(encoding)
+        return True
+    except:
+        return False
+
+
 class POSIX(Bcfg2.Client.Tools.Tool):
     """POSIX File support code."""
     name = 'POSIX'
@@ -458,12 +473,7 @@ class POSIX(Bcfg2.Client.Tools.Tool):
         # md5sum so it would be faster for big binary files
         contentStatus = content == tempdata
         if not contentStatus:
-            try:
-                content.decode('ascii')
-                isstring = True
-            except:
-                isstring = False
-            if tbin or not isstring:
+            if tbin or not isString(content, self.setup['encoding']):
                 entry.set('current_bfile', binascii.b2a_base64(content))
                 nqtext = entry.get('qtext', '')
                 nqtext += '\nBinary file, no printable diff'
@@ -493,15 +503,15 @@ class POSIX(Bcfg2.Client.Tools.Tool):
                                        difflib.unified_diff(content.split('\n'), \
                                                             tempdata.split('\n'))])
                     try:
-                        eudiff = udiff.encode('ascii')
+                        dudiff = udiff.decode(self.setup['encoding'])
                     except:
-                        eudiff = "Binary file: no diff printed"
+                        dudiff = "Binary file: no diff printed"
 
                     nqtext = entry.get('qtext', '')
 
                     if nqtext:
                         nqtext += '\n'
-                    nqtext += eudiff
+                    nqtext += dudiff
                 else:
                     entry.set('current_bfile', binascii.b2a_base64(content))
                     nqtext = entry.get('qtext', '')
diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py
index 942477a49..c9f3d3f58 100644
--- a/src/lib/Server/Admin/Reports.py
+++ b/src/lib/Server/Admin/Reports.py
@@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode):
         except (IOError, XMLSyntaxError):
             self.errExit("StatReports: Failed to parse %s" % (stats_file))
 
+        try:
+            encoding = self.cfp.get('components', 'encoding')
+        except:
+            encoding = 'UTF-8'
+
         if not clientspath:
             try:
                 clientspath = "%s/Metadata/clients.xml" % \
@@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode):
         try:
             load_stats(clientsdata,
                        statsdata,
+                       encoding,
                        verb,
                        self.log,
                        quick=quick,
diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py
index 8915492a3..b34d7108c 100644
--- a/src/lib/Server/Admin/__init__.py
+++ b/src/lib/Server/Admin/__init__.py
@@ -113,7 +113,8 @@ class MetadataCore(Mode):
     def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None):
         Mode.__init__(self, configfile)
         options = {'plugins': Bcfg2.Options.SERVER_PLUGINS,
-                   'configfile': Bcfg2.Options.CFILE}
+                   'configfile': Bcfg2.Options.CFILE,
+                   'encoding': Bcfg2.Options.ENCODING}
         setup = Bcfg2.Options.OptionParser(options)
         setup.hm = usage
         setup.parse(sys.argv[1:])
@@ -126,7 +127,7 @@ class MetadataCore(Mode):
         try:
             self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(),
                                                 setup['plugins'],
-                                                'foo', 'UTF-8')
+                                                'foo', setup['encoding'])
         except Bcfg2.Server.Core.CoreInitError:
             msg = sys.exc_info()[1]
             self.errExit("Core load failed because %s" % msg)
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 5e3cca847..c08e8c4b6 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -197,7 +197,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
             if os.path.exists("%s.genshi" % name):
                 logger.error("Cfg: Unable to pull data for genshi types")
                 raise Bcfg2.Server.Plugin.PluginExecutionError
-            open(name, 'w').write(new_entry['text'])
+            try:
+                etext = new_entry['text'].encode(self.encoding)
+            except:
+                logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding))
+                raise Bcfg2.Server.Plugin.PluginExecutionError
+            open(name, 'w').write(etext)
             if log:
                 logger.info("Wrote file %s" % name)
         badattr = [attr for attr in ['owner', 'group', 'perms']
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index 5ef1920e1..103fb7353 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
             try:
                 Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata,
                                                              container,
+                                                             self.core.encoding,
                                                              0,
                                                              logger,
                                                              True,
diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
index b6a3c2599..68774cec6 100755
--- a/src/lib/Server/Reports/importscript.py
+++ b/src/lib/Server/Reports/importscript.py
@@ -38,7 +38,7 @@ import platform
 from Bcfg2.Bcfg2Py3k import ConfigParser
 
 
-def build_reason_kwargs(r_ent):
+def build_reason_kwargs(r_ent, encoding, logger):
     binary_file = False
     if r_ent.get('current_bfile', False):
         binary_file = True
@@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent):
         rc_diff = r_ent.get('current_diff')
     else:
         rc_diff = ''
+    if not binary_file:
+        try:
+            rc_diff = rc_diff.decode(encoding)
+        except:
+            logger.error("Reason isn't %s encoded, cannot decode it" % encoding)
+            rc_diff = ''
     return dict(owner=r_ent.get('owner', default=""),
                 current_owner=r_ent.get('current_owner', default=""),
                 group=r_ent.get('group', default=""),
@@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent):
                 is_binary=binary_file)
 
 
-def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
+def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''):
     clients = {}
     [clients.__setitem__(c.name, c) \
         for c in Client.objects.all()]
@@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
             for (xpath, type) in pattern:
                 for x in statistics.findall(xpath):
                     counter_fields[type] = counter_fields[type] + 1
-                    kargs = build_reason_kwargs(x)
+                    kargs = build_reason_kwargs(x, encoding, logger)
 
                     try:
                         rr = None
@@ -270,6 +276,11 @@ if __name__ == '__main__':
         print("StatReports: Failed to parse %s" % (statpath))
         raise SystemExit(1)
 
+    try:
+        encoding = cf.get('components', 'encoding')
+    except:
+        encoding = 'UTF-8'
+
     if not clientpath:
         try:
             clientspath = "%s/Metadata/clients.xml" % \
@@ -288,6 +299,7 @@ if __name__ == '__main__':
     update_database()
     load_stats(clientsdata,
                statsdata,
+               encoding,
                verb,
                logger,
                quick=q,