From 3914d14bec1cb7c0f6a600ea8d04ee0e6abc6550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 16:32:53 +0200 Subject: Accept non-ASCII diffs Currently, client reports don't include diffs of files which aren't US-ASCII encoded. The client transmits such files as Base64 blobs. As we'd like to change that, this commit teaches the server to properly handle non-ASCII diffs. --- src/lib/Server/Admin/Reports.py | 6 ++++++ src/lib/Server/Admin/__init__.py | 5 +++-- src/lib/Server/Plugins/Cfg.py | 7 ++++++- src/lib/Server/Plugins/DBStats.py | 1 + src/lib/Server/Reports/importscript.py | 18 +++++++++++++++--- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 942477a49..c9f3d3f58 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode): except (IOError, XMLSyntaxError): self.errExit("StatReports: Failed to parse %s" % (stats_file)) + try: + encoding = self.cfp.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientspath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: load_stats(clientsdata, statsdata, + encoding, verb, self.log, quick=quick, diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index 8915492a3..b34d7108c 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -113,7 +113,8 @@ class MetadataCore(Mode): def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None): Mode.__init__(self, configfile) options = {'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'configfile': Bcfg2.Options.CFILE} + 'configfile': Bcfg2.Options.CFILE, + 'encoding': Bcfg2.Options.ENCODING} setup = Bcfg2.Options.OptionParser(options) setup.hm = usage setup.parse(sys.argv[1:]) @@ -126,7 +127,7 @@ class MetadataCore(Mode): try: self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), setup['plugins'], - 'foo', 'UTF-8') + 'foo', setup['encoding']) except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed because %s" % msg) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 5e3cca847..c08e8c4b6 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -197,7 +197,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(new_entry['text']) + try: + etext = new_entry['text'].encode(self.encoding) + except: + logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) + raise Bcfg2.Server.Plugin.PluginExecutionError + open(name, 'w').write(etext) if log: logger.info("Wrote file %s" % name) badattr = [attr for attr in ['owner', 'group', 'perms'] diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py index 5ef1920e1..103fb7353 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Server/Plugins/DBStats.py @@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, try: Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, container, + self.core.encoding, 0, logger, True, diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index b6a3c2599..68774cec6 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -38,7 +38,7 @@ import platform from Bcfg2.Bcfg2Py3k import ConfigParser -def build_reason_kwargs(r_ent): +def build_reason_kwargs(r_ent, encoding, logger): binary_file = False if r_ent.get('current_bfile', False): binary_file = True @@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent): rc_diff = r_ent.get('current_diff') else: rc_diff = '' + if not binary_file: + try: + rc_diff = rc_diff.decode(encoding) + except: + logger.error("Reason isn't %s encoded, cannot decode it" % encoding) + rc_diff = '' return dict(owner=r_ent.get('owner', default=""), current_owner=r_ent.get('current_owner', default=""), group=r_ent.get('group', default=""), @@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent): is_binary=binary_file) -def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): +def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): clients = {} [clients.__setitem__(c.name, c) \ for c in Client.objects.all()] @@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): for (xpath, type) in pattern: for x in statistics.findall(xpath): counter_fields[type] = counter_fields[type] + 1 - kargs = build_reason_kwargs(x) + kargs = build_reason_kwargs(x, encoding, logger) try: rr = None @@ -270,6 +276,11 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s" % (statpath)) raise SystemExit(1) + try: + encoding = cf.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -288,6 +299,7 @@ if __name__ == '__main__': update_database() load_stats(clientsdata, statsdata, + encoding, verb, logger, quick=q, -- cgit v1.2.3-1-g7c22 From 8c497c815589c7c5878490df7661c6f5d3330829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Holger=20Wei=C3=9F?= Date: Mon, 6 Jun 2011 17:06:16 +0200 Subject: Create non-ASCII diffs The client now also transmits diffs of files which include non-US-ASCII data (using the encoding setting from Options.py), unless they look like binary files. In the past, non-ASCII files were transmitted as Base64 blobs. In addition, "bcfg2 -I" no longer refuses to display non-ASCII diffs. Resolves ticket #999. --- src/lib/Client/Frame.py | 3 ++- src/lib/Client/Tools/POSIX.py | 28 +++++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index 60d158eb1..57844ab19 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -5,6 +5,7 @@ installs entries, and generates statistics. __revision__ = '$Revision$' import logging +import sys import time import Bcfg2.Client.Tools @@ -29,7 +30,7 @@ def promptFilter(prompt, entries): try: # py3k compatibility try: - ans = raw_input(iprompt) + ans = raw_input(iprompt.encode(sys.stdout.encoding, 'replace')) except NameError: ans = input(iprompt) if ans in ['y', 'Y']: diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index 875db5ea7..862e0bc04 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -76,6 +76,21 @@ def normUid(entry): return False +def isString(strng, encoding): + """ + Returns true if the string contains no ASCII control characters + and can be decoded from the specified encoding. + """ + for char in strng: + if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: + return False + try: + strng.decode(encoding) + return True + except: + return False + + class POSIX(Bcfg2.Client.Tools.Tool): """POSIX File support code.""" name = 'POSIX' @@ -458,12 +473,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): # md5sum so it would be faster for big binary files contentStatus = content == tempdata if not contentStatus: - try: - content.decode('ascii') - isstring = True - except: - isstring = False - if tbin or not isstring: + if tbin or not isString(content, self.setup['encoding']): entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') nqtext += '\nBinary file, no printable diff' @@ -493,15 +503,15 @@ class POSIX(Bcfg2.Client.Tools.Tool): difflib.unified_diff(content.split('\n'), \ tempdata.split('\n'))]) try: - eudiff = udiff.encode('ascii') + dudiff = udiff.decode(self.setup['encoding']) except: - eudiff = "Binary file: no diff printed" + dudiff = "Binary file: no diff printed" nqtext = entry.get('qtext', '') if nqtext: nqtext += '\n' - nqtext += eudiff + nqtext += dudiff else: entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') -- cgit v1.2.3-1-g7c22