Client side of improved support for binary files in statistics

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@3587 ce84e21b-d406-0410-9b95-82705330c041

6 files changed, 104 insertions, 37 deletions

diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py
index ecd4917a3..b19594402 100644
--- a/src/lib/Client/Frame.py
+++ b/src/lib/Client/Frame.py
@@ -1,7 +1,7 @@
 '''Frame is the Client Framework that verifies and installs entries, and generates statistics'''
 __revision__ = '$Revision$'
 
-import binascii, logging, time
+import logging, time
 import Bcfg2.Client.Tools
 
 def cmpent(ent1, ent2):
@@ -326,10 +326,7 @@ class Frame:
             for item in data:
                 item.set('qtext', '')
                 container.append(item)
-                if item.get('current_diff', False):
-                    item.set('current_bdiff',
-                             binascii.b2a_base64(item.get('current_diff')))
-                    item.set('current_diff', '')
+                item.text = None
 
         timeinfo = Bcfg2.Client.XML.Element("OpStamps")
         feedback.append(stats)
diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py
index 783e3192a..f4e3063a6 100644
--- a/src/lib/Client/Tools/POSIX.py
+++ b/src/lib/Client/Tools/POSIX.py
@@ -43,7 +43,20 @@ def normGid(entry):
     except (OSError, KeyError):
         self.logger.error('GID normalization failed for %s' % (entry.get('name')))
         return False
-    
+
+text_chars = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
+notrans = string.maketrans("", "")
+
+def isString(strng):
+    '''Returns true if a string contains no binary chars'''
+    if "\0" in strng:
+        return False
+
+    if not strng:
+        return True
+
+    return len(strng.translate(notrans, text_chars)) == 0
+
 class POSIX(Bcfg2.Client.Tools.Tool):
     '''POSIX File support code'''
     __name__ = 'POSIX'
@@ -251,22 +264,27 @@ class POSIX(Bcfg2.Client.Tools.Tool):
             return False
         contentStatus = content == tempdata
         if not contentStatus:
-            diff = '\n'.join([x for x in difflib.ndiff(content.split('\n'), tempdata.split('\n'))])
-            try:
-                entry.set("current_diff", xml.sax.saxutils.escape(diff))
-            except:
-                pass
-            udiff = '\n'.join([x for x in difflib.unified_diff(content.split('\n'), \
-                                                               tempdata.split('\n'))])
-            try:
-                eudiff = udiff.encode('ascii')
-            except:
-                eudiff = "Binary file: no diff printed"
-            nqtext = entry.get('qtext', '')
-            if nqtext:
-                nqtext += '\n'
-            nqtext += eudiff 
-            entry.set('qtext', nqtext)
+            if not isString(content) or not isString(tempdata):
+                entry.set('current_bfile', binascii.b2a_base64(content))
+                nqtext = entry.get('qtext', '')
+                nqtext += '\nBinary file, no printable diff'
+                entry.set('qtext', nqtest)
+            else:
+                diff = '\n'.join([x for x in difflib.ndiff(content.split('\n'),
+                                                           tempdata.split('\n'))])
+                entry.set("current_bdiff", binascii.b2a_base64(diff))
+                udiff = '\n'.join([x for x in \
+                                   difflib.unified_diff(content.split('\n'), \
+                                                        tempdata.split('\n'))])
+                try:
+                    eudiff = udiff.encode('ascii')
+                except:
+                    eudiff = "Binary file: no diff printed"
+                nqtext = entry.get('qtext', '')
+                if nqtext:
+                    nqtext += '\n'
+                    nqtext += eudiff 
+                entry.set('qtext', nqtext)
         qtxt = entry.get('qtext', '')
         qtxt += "\nInstall ConfigFile %s: (y/N): " % (entry.get('name'))
         entry.set('qtext', qtxt)
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 3c32dcad7..aa68a80df 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -275,6 +275,7 @@ class INode:
 class XMLSrc(XMLFileBacked):
     '''XMLSrc files contain a LNode hierarchy that returns matching entries'''
     __node__ = INode
+    __cacheobj__ = dict
 
     def __init__(self, filename):
         XMLFileBacked.__init__(self, filename)
@@ -307,7 +308,7 @@ class XMLSrc(XMLFileBacked):
     def Cache(self, metadata):
         '''Build a package dict for a given host'''
         if self.cache == None or self.cache[0] != metadata:
-            cache = (metadata, {})
+            cache = (metadata, self.__cacheobj__())
             if self.pnode == None:
                 logger.error("Cache method called early for %s; forcing data load" % (self.name))
                 self.HandleEvent()
diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py
index abab0220f..a11e237d1 100644
--- a/src/lib/Server/Plugins/Pkgmgr.py
+++ b/src/lib/Server/Plugins/Pkgmgr.py
@@ -5,6 +5,28 @@ import logging, re, Bcfg2.Server.Plugin
 
 logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr')
 
+class FuzzyDict(dict):
+    fuzzy = re.compile('(?P<name>.*):(?P<alist>\S+(,\S+)*)')
+    def __getitem__(self, key):
+        mdata = self.fuzzy.match(key)
+        if mdata:
+            return dict.__getitem__(self, mdata.groupdict()['name'])
+        return dict.__getitem__(self, key)
+
+    def has_key(self, key):
+        mdata = self.fuzzy.match(key)
+        if self.fuzzy.match(key):
+            return dict.has_key(self, mdata.groupdict()['name'])
+        return dict.has_key(self, key)
+
+    def get(self, key, default=None):
+        try:
+            return self.__getitem__(key)
+        except:
+            if default:
+                return default
+            raise
+
 class PNode(Bcfg2.Server.Plugin.INode):
     '''PNode has a list of packages available at a particular group intersection'''
     splitters = {'rpm':re.compile('^(.*/)?(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \
@@ -12,6 +34,18 @@ class PNode(Bcfg2.Server.Plugin.INode):
                  'encap':re.compile('^(?P<name>[\w-]+)-(?P<version>[\w\d\.+-]+).encap.*$')}
     ignore = ['Package']
 
+    def Match(self, metadata, data):
+        '''Return a dictionary of package mappings'''
+        if self.predicate(metadata):
+            for key in self.contents:
+                try:
+                    data[key].update(self.contents[key])
+                except:
+                    data[key] = FuzzyDict()
+                    data[key].update(self.contents[key])
+            for child in self.children:
+                child.Match(metadata, data)
+
     def __init__(self, data, pdict, parent=None):
         # copy local attributes to all child nodes if no local attribute exists
         if not pdict.has_key('Package'):
@@ -26,7 +60,7 @@ class PNode(Bcfg2.Server.Plugin.INode):
                     pass
         Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent)
         if not self.contents.has_key('Package'):
-            self.contents['Package'] = {}
+            self.contents['Package'] = FuzzyDict()
         for pkg in data.findall('./Package'):
             if pkg.attrib.has_key('name') and pkg.get('name') not in pdict['Package']:
                 pdict['Package'].append(pkg.get('name'))
@@ -74,6 +108,7 @@ class PNode(Bcfg2.Server.Plugin.INode):
 class PkgSrc(Bcfg2.Server.Plugin.XMLSrc):
     '''PkgSrc files contain a PNode hierarchy that returns matching package entries'''
     __node__ = PNode
+    __cacheobj__ = FuzzyDict
 
 class Pkgmgr(Bcfg2.Server.Plugin.PrioDir):
     '''This is a generator that handles package assignments'''
@@ -82,3 +117,15 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir):
     __author__ = 'bcfg-dev@mcs.anl.gov'
     __child__ = PkgSrc
     __element__ = 'Package'
+
+    def HandleEvent(self, event):
+        '''Handle events and update dispatch table'''
+        Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event)
+        for src in self.entries.values():
+            for itype, children in src.items.iteritems():
+                for child in children:
+                    try:
+                        self.Entries[itype][child] = self.BindEntry
+                    except KeyError:
+                        self.Entries[itype] = FuzzyDict([(child,
+                                                          self.BindEntry)])
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
index 4d3f18957..3e468d91f 100644
--- a/src/lib/Server/Plugins/SSHbase.py
+++ b/src/lib/Server/Plugins/SSHbase.py
@@ -6,7 +6,7 @@ import Bcfg2.Server.Plugin
 
 def update_file(path, diff):
     '''Update file at path using diff'''
-    newdata = '\n'.join(difflib.restore(xml.sax.saxutils.unescape(diff).split('\n'), 1))
+    newdata = '\n'.join(difflib.restore(diff.split('\n'), 1))
     print "writing file, %s" % path
     open(path, 'w').write(newdata)
 
diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin
index 5264bacf4..95aa2b68a 100755
--- a/src/sbin/bcfg2-admin
+++ b/src/sbin/bcfg2-admin
@@ -3,7 +3,7 @@
 
 import getopt, difflib, logging, lxml.etree, os, popen2, re, socket, sys, ConfigParser
 import Bcfg2.Server.Core, Bcfg2.Logging, Bcfg2.tlslite.api
-import xml.sax.saxutils, binascii, time
+import binascii, time
 
 log = logging.getLogger('bcfg-admin')
 
@@ -155,12 +155,6 @@ def initialize_repo(cfile):
     open("%s/Metadata/clients.xml"%repo, "w").write(clients%socket.getfqdn())
     print "Repository created successfuly in %s" % (repo)
 
-def update_file(path, diff):
-    '''Update file at path using diff'''
-    newdata = '\n'.join(difflib.restore(xml.sax.saxutils.unescape(diff).split('\n'), 1))
-    print "writing file, %s" % path
-    open(path, 'w').write(newdata)
-
 def get_repo_path(cfile='/etc/bcfg2.conf'):
     '''return repository path'''
     cfp = ConfigParser.ConfigParser()
@@ -319,11 +313,21 @@ def do_pull(cfile, repopath, client, etype, ename):
     entry = statblock[0].xpath('.//Bad/ConfigFile[@name="%s"]' % ename)
     if not entry:
         err_exit("Could not find state data for entry; rerun bcfg2 on client system")
-
-    if 'current_bdiff' in entry[0].attrib:
-        diff = binascii.a2b_base64(entry[-1].get('current_bdiff'))
+    cfentry = entry[-1]
+
+    mode = 'diff'
+    if 'current_bdiff' in cfentry.attrib:
+        diff = binascii.a2b_base64(cfentry.get('current_bdiff'))
+    elif 'current_diff' in cfentry.attrib:
+        diff = cfentry.get('current_diff')
+    elif 'current_bfile' in cfentry.attrib:
+        mode = 'full'
+        data = binascii.a2b_base64(cfentry.get('current_bfile'))
+
+    if mode == 'diff':
+        print "Located diff:\n %s" % diff
     else:
-        diff = entry[-1].get('current_diff')
+        print "Found full file data:\n", data
 
     try:
         bcore = Bcfg2.Server.Core.Core({}, cfile)
@@ -342,7 +346,7 @@ def do_pull(cfile, repopath, client, etype, ename):
                  + "%s" % ([g.__name__ for g in glist]))
     plugin = glist[0]
     if diff == None:
-        err_exit("Failed to locate diff\nStatistics entry was:\n%s" % lxml.etree.tostring(entry[-1]))
+        err_exit("Failed to locate diff\nStatistics entry was:\n%s" % lxml.etree.tostring(cfentry))
     try:
         plugin.AcceptEntry(m, 'ConfigFile', ename, diff)
     except Bcfg2.Server.Plugin.PluginExecutionError: