From e32bbfbca5233d4ad7a5bee74698d614ff0b1f24 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Mon, 30 Jun 2008 15:59:01 +0000 Subject: Unicode support (from stousignant) (Resolves Ticket #549) git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4731 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Client/Tools/POSIX.py | 13 +++++++++++-- src/lib/Options.py | 3 +++ src/lib/Server/Admin/Pull.py | 2 +- src/lib/Server/Core.py | 3 ++- src/lib/Server/Plugin.py | 11 +++++++---- src/lib/Server/Plugins/Cfg.py | 6 +++--- src/lib/Server/Plugins/Metadata.py | 2 +- src/lib/Server/Plugins/TCheetah.py | 9 +++++++-- src/lib/Server/Plugins/TGenshi.py | 17 ++++++++++++++--- src/lib/Server/Statistics.py | 2 +- src/sbin/bcfg2 | 5 +++-- src/sbin/bcfg2-build-reports | 6 +++--- src/sbin/bcfg2-info | 13 +++++++------ src/sbin/bcfg2-ping-sweep | 2 +- src/sbin/bcfg2-server | 8 +++++--- 15 files changed, 69 insertions(+), 33 deletions(-) diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index ec7d3383d..df8558e1d 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -320,11 +320,15 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.error("Cannot verify incomplete ConfigFile %s" % (entry.get('name'))) return False tempdata = entry.text + if type(tempdata) == unicode: + tempdata = tempdata.encode(self.setup['encoding']) try: content = open(entry.get('name')).read() except IOError, error: self.logger.error("Failed to read %s: %s" % (error.filename, error.strerror)) return False + # comparaison should be done with figerprints or md5sum so it would be faster + # for big binary files contentStatus = content == tempdata if not contentStatus: if tbin or not isString(content): @@ -348,7 +352,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): break if do_diff: diff = '\n'.join(rawdiff) - entry.set("current_bdiff", binascii.b2a_base64(diff)) +# entry.set("current_bdiff", binascii.b2a_base64(diff)) +# entry.set("current_diff", diff) udiff = '\n'.join([x for x in \ difflib.unified_diff(content.split('\n'), \ tempdata.split('\n'))]) @@ -356,6 +361,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): eudiff = udiff.encode('ascii') except: eudiff = "Binary file: no diff printed" + nqtext = entry.get('qtext', '') if nqtext: @@ -417,7 +423,10 @@ class POSIX(Bcfg2.Client.Tools.Tool): elif entry.get('empty', 'false') == 'true': filedata = '' else: - filedata = entry.text + if type(entry.text) == unicode: + filedata = entry.text.encode(self.setup['encoding']) + else: + filedata = entry.text newfile.write(filedata) newfile.close() try: diff --git a/src/lib/Options.py b/src/lib/Options.py index 6075d1ced..2d37d1cab 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -247,6 +247,9 @@ AGENT_PORT = Option('Agent port', default=6789, cmd='-p', odesc='', cf=('communication', 'agent-port')) AGENT_HOST = Option('Remote host', default=False, cmd='-H', odesc='') +ENCODING = Option('Encoding of cfg files', default=sys.getdefaultencoding(), cmd='-E', odesc='', + cf=('components', 'encoding')) + class OptionParser(OptionSet): '''OptionParser bootstraps option parsing, getting the value of the config file''' def __init__(self, args): diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py index ede4271fb..71bfdefc2 100644 --- a/src/lib/Server/Admin/Pull.py +++ b/src/lib/Server/Admin/Pull.py @@ -70,7 +70,7 @@ class Pull(Bcfg2.Server.Admin.Mode): new_entry['text'] = '\n'.join(difflib.restore(diff.split('\n'), 1)) else: print "found no data::" - print lxml.etree.tostring(cfentry) + print lxml.etree.tostring(cfentry, encoding='UTF-8', xml_declaration=True) raise SystemExit(1) return new_entry diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index e7e22f3c3..b3742e8d3 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -200,7 +200,7 @@ except ImportError: class Core(object): '''The Core object is the container for all Bcfg2 Server logic, and modules''' - def __init__(self, repo, structures, generators, password, svn): + def __init__(self, repo, structures, generators, password, svn, encoding): object.__init__(self) self.datastore = repo try: @@ -215,6 +215,7 @@ class Core(object): self.revision = '-1' self.password = password self.svn = svn + self.encoding = encoding try: if self.svn: self.read_svn_revision() diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 6b308d1f7..61e7021c6 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -444,7 +444,7 @@ class Specificity: return False class SpecificData(object): - def __init__(self, name, _, specific): + def __init__(self, name, _, specific, encoding): self.name = name self.specific = specific @@ -459,13 +459,14 @@ class SpecificData(object): class EntrySet: '''Entry sets deal with the host- and group-specific entries''' ignore = re.compile("^(.*~|\\..*\\.(tmp|sw[px]))$") - def __init__(self, basename, path, props, entry_type): + def __init__(self, basename, path, props, entry_type, encoding): self.path = path self.entry_type = entry_type self.entries = {} self.properties = props self.metadata = default_file_metadata.copy() self.infoxml = None + self.encoding = encoding pattern = '(.*/)?%s(\.((H_(?P\S+))|' % basename pattern += '(G(?P\d+)_(?P\S+))))?$' self.specific = re.compile(pattern) @@ -509,7 +510,7 @@ class EntrySet: return self.entries[event.filename] = self.entry_type(fpath, self.properties, - spec) + spec, self.encoding) self.entries[event.filename].handle_event(event) def specificity_from_filename(self, fname): @@ -631,6 +632,7 @@ class GroupSpool(GeneratorPlugin): self.entries = {} self.handles = {} self.AddDirectoryMonitor('') + self.encoding = core.encoding if self.use_props: try: self.properties = TemplateProperties( \ @@ -661,7 +663,8 @@ class GroupSpool(GeneratorPlugin): self.entries[ident] = self.es_cls(self.filename_pattern, dirpath, self.properties, - self.es_child_cls) + self.es_child_cls, + self.encoding) self.Entries['ConfigFile'][ident] = self.entries[ident].bind_entry if not posixpath.isdir(epath): # do not pass through directory events diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 80c9d7467..5f3f1688b 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -49,8 +49,8 @@ class CfgMatcher: return self.basefile_reg.match(fname) class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, basename, path, props, entry_type): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type) + def __init__(self, basename, path, props, entry_type, encoding): + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type, encoding) self.specific = CfgMatcher(path.split('/')[-1]) def sort_by_specific(self, one, other): @@ -79,7 +79,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if entry.get('encoding') == 'base64': entry.text = binascii.b2a_base64(data) else: - entry.text = data + entry.text = unicode(data, self.encoding) if entry.text in ['', None]: entry.set('empty', 'true') diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index 084873471..b78d05ada 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -296,7 +296,7 @@ class Metadata(Bcfg2.Server.Plugin.MetadataPlugin, value=self.probedata[client][probe]) for group in self.cgroups[client]: lxml.etree.SubElement(cx, "Group", name=group) - data = lxml.etree.tostring(top) + data = lxml.etree.tostring(top, encoding='UTF-8', xml_declaration=True) try: datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py index ec82f5753..e2b4d370d 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Server/Plugins/TCheetah.py @@ -9,10 +9,11 @@ logger = logging.getLogger('Bcfg2.Plugins.TCheetah') class TemplateFile: '''Template file creates Cheetah template structures for the loaded file''' - def __init__(self, name, properties, specific): + def __init__(self, name, properties, specific, encoding): self.name = name self.properties = properties self.specific = specific + self.encoding = encoding self.template = None def handle_event(self, event): @@ -34,7 +35,11 @@ class TemplateFile: self.template.path = entry.get('realname', entry.get('name')) try: - entry.text = str(self.template) + if type(self.template) == unicode: + entry.text = self.template + else : + logger.debug("Override encoding of template to %s" % self.encoding) + entry.text = unicode(str(self.template), self.encoding) except: (a, b, c) = sys.exc_info() msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index e58fb8017..16470a231 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -17,10 +17,11 @@ def removecomment(stream): class TemplateFile: '''Template file creates Genshi template structures for the loaded file''' - def __init__(self, name, properties, specific): + def __init__(self, name, properties, specific, encoding): self.name = name self.properties = properties self.specific = specific + self.encoding = encoding if self.specific.all: matchname = self.name elif self.specific.group: @@ -52,9 +53,19 @@ class TemplateFile: name=fname, metadata=metadata, path=self.name, properties=self.properties).filter(removecomment) if isinstance(self.template, TextTemplate): - entry.text = stream.render('text') + textdata = stream.render('text') + if type(textdata) == unicode: + entry.text = textdata + else: + logger.debug("Override encoding of template to %s" % self.encoding) + entry.text = unicode(textdata, self.encoding) else: - entry.text = stream.render('xml') + xmldata = stream.render('xml') + if type(xmldata) == unicode: + entry.text = xmldata + else: + logger.debug("Override encoding of template to %s" % self.encoding) + entry.text = unicode(xmldata, self.encoding) except TemplateError, terror: logger.error('Genshi template error: %s' % terror) raise Bcfg2.Server.Plugin.PluginExecutionError diff --git a/src/lib/Server/Statistics.py b/src/lib/Server/Statistics.py index bbfc684bd..7111113aa 100644 --- a/src/lib/Server/Statistics.py +++ b/src/lib/Server/Statistics.py @@ -28,7 +28,7 @@ class Statistics(object): except IOError, ioerr: self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr)) else: - fout.write(lxml.etree.tostring(self.element)) + fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True)) fout.close() os.rename(self.filename + '.new', self.filename) self.dirty = 0 diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 472eab795..5ac1e3aac 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -116,6 +116,7 @@ class Client: 'agent-background': Bcfg2.Options.CLIENT_BACKGROUND, 'key': Bcfg2.Options.SERVER_KEY, 'decision-list': DECISION_LIST, + 'encoding': Bcfg2.Options.ENCODING, } self.setup = Bcfg2.Options.OptionParser(optinfo) @@ -244,7 +245,7 @@ class Client: if len(probes.findall(".//probe")) > 0: try: # upload probe responses - proxy.RecvProbeData(Bcfg2.Client.XML.tostring(probedata)) + proxy.RecvProbeData(Bcfg2.Client.XML.tostring(probedata, encoding='UTF-8', xml_declaration=True)) except: self.logger.error("Failed to upload probe data", exc_info=1) raise SystemExit(1) @@ -293,7 +294,7 @@ class Client: feedback = self.tools.GenerateStats() try: - proxy.RecvStats(Bcfg2.Client.XML.tostring(feedback)) + proxy.RecvStats(Bcfg2.Client.XML.tostring(feedback, encoding='UTF-8', xml_declaration=True)) except xmlrpclib.Fault: self.logger.error("Failed to upload configuration statistics") raise SystemExit(2) diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports index 534cbdd91..8ae8475a3 100755 --- a/src/sbin/bcfg2-build-reports +++ b/src/sbin/bcfg2-build-reports @@ -100,7 +100,7 @@ def rss(reportxml, delivery, report): for item in items: channel.append(item) - tree = "" + tostring(rssdata) + tree = tostring(rssdata, encoding='UTF-8', xml_declaration=True) fil.write(tree) fil.close() @@ -249,7 +249,7 @@ if __name__ == '__main__': #apply XSLT, different ones based on report type, and options if deliverymechanism == 'null-operator': #Special Cases - fileout(tostring(ElementTree(procnodereport).getroot()), deliv) + fileout(tostring(ElementTree(procnodereport).getroot(), encoding='UTF-8', xml_declaration=True), deliv) break transform = delivtype + '-' + deliverymechanism + '.xsl' @@ -301,7 +301,7 @@ if __name__ == '__main__': (toastring, socket.getfqdn(), outputstring) mail(outputstring, c) #call function to send else: - outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot()) + outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot(), encoding='UTF-8', xml_declaration=True) if deliverymechanism == 'rss': rss(outputstring, deliv, reprt) else: # must be deliverymechanism == 'www': diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index dac7fa6bf..9cffc8719 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -23,10 +23,10 @@ def printTabular(rows): print fstring % row class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): - def __init__(self, repo, struct, gens, passwd, svn): + def __init__(self, repo, struct, gens, passwd, svn, encoding): cmd.Cmd.__init__(self) try: - Bcfg2.Server.Core.Core.__init__(self, repo, struct, gens, passwd, svn) + Bcfg2.Server.Core.Core.__init__(self, repo, struct, gens, passwd, svn, encoding) except Bcfg2.Server.Core.CoreInitError, msg: print "Core load failed because %s" % msg raise SystemExit(1) @@ -97,7 +97,7 @@ Usage: [quit|exit]""" if len(args.split()) == 2: client, ofile = args.split() output = open(ofile, 'w') - data = lxml.etree.tostring(self.BuildConfiguration(client)) + data = lxml.etree.tostring(self.BuildConfiguration(client), encoding='UTF-8', xml_declaration=True) output.write(data) output.close() else: @@ -121,7 +121,7 @@ Usage: [quit|exit]""" entry = lxml.etree.Element('ConfigFile', name=fname) metadata = self.metadata.get_metadata(client) self.Bind(entry, metadata) - print lxml.etree.tostring(entry) + print lxml.etree.tostring(entry, encoding="UTF-8", xml_declaration=True) else: print 'Usage: buildfile filename hostname' @@ -271,12 +271,13 @@ if __name__ == '__main__': 'svn': Bcfg2.Options.SERVER_SVN, 'structures': Bcfg2.Options.SERVER_STRUCTURES, 'generators': Bcfg2.Options.SERVER_GENERATORS, - 'password': Bcfg2.Options.SERVER_PASSWORD}) + 'password': Bcfg2.Options.SERVER_PASSWORD, + 'encoding': Bcfg2.Options.ENCODING}) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) loop = infoCore(setup['repo'], setup['structures'], setup['generators'], - setup['password'], setup['svn']) + setup['password'], setup['svn'], setup['encoding']) loop.plugins['Metadata'] if "args" in setup and setup['args']: loop.onecmd(" ".join(setup['args'])) diff --git a/src/sbin/bcfg2-ping-sweep b/src/sbin/bcfg2-ping-sweep index 54c645277..88d8bd782 100755 --- a/src/sbin/bcfg2-ping-sweep +++ b/src/sbin/bcfg2-ping-sweep @@ -63,6 +63,6 @@ if __name__ == '__main__': elm.set("pingable",'N') fout = open(clientdatapath, 'w') - fout.write(lxml.etree.tostring(clientElement.getroot())) + fout.write(lxml.etree.tostring(clientElement.getroot(), encoding='UTF-8', xml_declaration=True)) fout.close() diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index a5c5310ba..d26fce9bf 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -41,7 +41,8 @@ class Bcfg2Serv(Bcfg2.Component.Component): try: self.Core = Core(setup['repo'], setup['structures'], - setup['generators'], setup['password'], setup['svn']) + setup['generators'], setup['password'], + setup['svn'], setup['encoding']) except CoreInitError, msg: logger.critical("Fatal error: %s" % (msg)) raise SystemExit, 1 @@ -104,7 +105,7 @@ class Bcfg2Serv(Bcfg2.Component.Component): if isinstance(p, Bcfg2.Server.Plugin.ProbingPlugin)]: for probe in plugin.GetProbes(meta): resp.append(probe) - return tostring(resp) + return tostring(resp, encoding='UTF-8', xml_declaration=True) except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: warning = 'Client metadata resolution error for %s; check server log' % address[0] self.logger.warning(warning) @@ -158,7 +159,7 @@ class Bcfg2Serv(Bcfg2.Component.Component): '''Build config for a client''' try: client = self.Core.metadata.resolve_client(address) - return tostring(self.Core.BuildConfiguration(client)) + return tostring(self.Core.BuildConfiguration(client), encoding='UTF-8', xml_declaration=True) except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: self.logger.warning("Metadata consistency failure for %s" % (address)) raise Fault, (6, "Metadata consistency failure") @@ -201,6 +202,7 @@ if __name__ == '__main__': 'location' : Bcfg2.Options.SERVER_LOCATION, 'passwd' : Bcfg2.Options.SERVER_PASSWORD, 'static' : Bcfg2.Options.SERVER_STATIC, + 'encoding' : Bcfg2.Options.ENCODING, }) -- cgit v1.2.3-1-g7c22