Merge branch 'master' of git.mcs.anl.gov:bcfg2

27 files changed, 373 insertions, 152 deletions

diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py
index 3af25b15a..81fc4a1b1 100644
--- a/src/lib/Server/Admin/Client.py
+++ b/src/lib/Server/Admin/Client.py
@@ -27,7 +27,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore):
             for i in args[2:]:
                 attr, val = i.split('=', 1)
                 if attr not in ['profile', 'uuid', 'password',
-                                'location', 'secure', 'address']:
+                                'location', 'secure', 'address',
+                                'auth']:
                     print("Attribute %s unknown" % attr)
                     raise SystemExit(1)
                 attr_d[attr] = val
@@ -41,7 +42,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore):
             for i in args[2:]:
                 attr, val = i.split('=', 1)
                 if attr not in ['profile', 'uuid', 'password',
-                                'location', 'secure', 'address']:
+                                'location', 'secure', 'address',
+                                'auth']:
                     print("Attribute %s unknown" % attr)
                     raise SystemExit(1)
                 attr_d[attr] = val
diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py
index fff8bcd1c..eab030cf8 100644
--- a/src/lib/Server/Admin/Init.py
+++ b/src/lib/Server/Admin/Init.py
@@ -103,8 +103,7 @@ plugin_list = ['Account',
                'TGenshi']
 
 # Default list of plugins to use
-default_plugins = ['Base',
-                   'Bundler',
+default_plugins = ['Bundler',
                    'Cfg',
                    'Metadata',
                    'Pkgmgr',
diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py
index af1c83072..d03b37d57 100644
--- a/src/lib/Server/Admin/Perf.py
+++ b/src/lib/Server/Admin/Perf.py
@@ -22,6 +22,7 @@ class Perf(Bcfg2.Server.Admin.Mode):
             'password': Bcfg2.Options.SERVER_PASSWORD,
             'server': Bcfg2.Options.SERVER_LOCATION,
             'user': Bcfg2.Options.CLIENT_USER,
+            'timeout': Bcfg2.Options.CLIENT_TIMEOUT,
             }
         setup = Bcfg2.Options.OptionParser(optinfo)
         setup.parse(sys.argv[2:])
@@ -30,7 +31,8 @@ class Perf(Bcfg2.Server.Admin.Mode):
                                            setup['password'],
                                            key=setup['key'],
                                            cert=setup['certificate'],
-                                           ca=setup['ca'])
+                                           ca=setup['ca'],
+                                           timeout=setup['timeout'])
         data = proxy.get_statistics()
         for key, value in list(data.items()):
             data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]])
diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py
index 942477a49..c9f3d3f58 100644
--- a/src/lib/Server/Admin/Reports.py
+++ b/src/lib/Server/Admin/Reports.py
@@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode):
         except (IOError, XMLSyntaxError):
             self.errExit("StatReports: Failed to parse %s" % (stats_file))
 
+        try:
+            encoding = self.cfp.get('components', 'encoding')
+        except:
+            encoding = 'UTF-8'
+
         if not clientspath:
             try:
                 clientspath = "%s/Metadata/clients.xml" % \
@@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode):
         try:
             load_stats(clientsdata,
                        statsdata,
+                       encoding,
                        verb,
                        self.log,
                        quick=quick,
diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py
index fd5794f88..2cb085346 100644
--- a/src/lib/Server/Admin/Xcmd.py
+++ b/src/lib/Server/Admin/Xcmd.py
@@ -20,7 +20,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode):
             'password': Bcfg2.Options.SERVER_PASSWORD,
             'key': Bcfg2.Options.SERVER_KEY,
             'certificate': Bcfg2.Options.CLIENT_CERT,
-            'ca': Bcfg2.Options.CLIENT_CA
+            'ca': Bcfg2.Options.CLIENT_CA,
+            'timeout': Bcfg2.Options.CLIENT_TIMEOUT,
             }
         setup = Bcfg2.Options.OptionParser(optinfo)
         setup.parse(sys.argv[2:])
@@ -31,7 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode):
                                            key=setup['key'],
                                            cert=setup['certificate'],
                                            ca=setup['ca'],
-                                           timeout=180)
+                                           timeout=setup['timeout'])
         if len(setup['args']) == 0:
             print("Usage: xcmd <xmlrpc method> <optional arguments>")
             return
diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py
index 8915492a3..b34d7108c 100644
--- a/src/lib/Server/Admin/__init__.py
+++ b/src/lib/Server/Admin/__init__.py
@@ -113,7 +113,8 @@ class MetadataCore(Mode):
     def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None):
         Mode.__init__(self, configfile)
         options = {'plugins': Bcfg2.Options.SERVER_PLUGINS,
-                   'configfile': Bcfg2.Options.CFILE}
+                   'configfile': Bcfg2.Options.CFILE,
+                   'encoding': Bcfg2.Options.ENCODING}
         setup = Bcfg2.Options.OptionParser(options)
         setup.hm = usage
         setup.parse(sys.argv[1:])
@@ -126,7 +127,7 @@ class MetadataCore(Mode):
         try:
             self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(),
                                                 setup['plugins'],
-                                                'foo', 'UTF-8')
+                                                'foo', setup['encoding'])
         except Bcfg2.Server.Core.CoreInitError:
             msg = sys.exc_info()[1]
             self.errExit("Core load failed because %s" % msg)
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index 8f9d3e746..5adfa5381 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -384,7 +384,7 @@ class Core(Component):
         # clear dynamic groups
         self.metadata.cgroups[meta.hostname] = []
         try:
-            xpdata = lxml.etree.XML(probedata)
+            xpdata = lxml.etree.XML(probedata.encode('utf-8'))
         except:
             self.logger.error("Failed to parse probe data from client %s" % \
                               (address[0]))
@@ -433,7 +433,7 @@ class Core(Component):
     @exposed
     def RecvStats(self, address, stats):
         """Act on statistics upload."""
-        sdata = lxml.etree.XML(stats)
+        sdata = lxml.etree.XML(stats.encode('utf-8'))
         client = self.metadata.resolve_client(address)
         self.process_statistics(client, sdata)
         return "<ok/>"
diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py
index aa822409c..bf774f695 100644
--- a/src/lib/Server/Hostbase/backends.py
+++ b/src/lib/Server/Hostbase/backends.py
@@ -57,12 +57,14 @@ class NISBackend(object):
 
             return user
 
-        except NISAUTHError, e:
+        except NISAUTHError:
+            e = sys.exc_info()[1]
             return None
 
 
     def get_user(self, user_id):
         try:
             return User.objects.get(pk=user_id)
-        except User.DoesNotExist, e:
+        except User.DoesNotExist:
+            e = sys.exc_info()[1]
             return None
diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py
index 21b462c86..f3db26f67 100644
--- a/src/lib/Server/Hostbase/ldapauth.py
+++ b/src/lib/Server/Hostbase/ldapauth.py
@@ -69,7 +69,8 @@ class ldapauth(object):
                                     None)
             result_type, result_data = conn.result(result_id, 0)
             return ('success', 'User profile found', result_data,)
-        except ldap.LDAPError, e:
+        except ldap.LDAPError:
+            e = sys.exc_info()[1]
             #connection failed
             return ('error', 'LDAP connect failed', e,)
 
@@ -86,7 +87,8 @@ class ldapauth(object):
                                     None)
             result_type, result_data = conn.result(result_id, 0)
             return ('success', 'User profile found', result_data,)
-        except ldap.LDAPError, e:
+        except ldap.LDAPError:
+            e = sys.exc_info()[1]
             #connection failed
             return ('error', 'LDAP connect failed', e,)
 
@@ -108,7 +110,8 @@ class ldapauth(object):
             raw_obj = result_data[0][1]
             distinguishedName = raw_obj['distinguishedName']
             return ('success', distinguishedName[0],)
-        except ldap.LDAPError, e:
+        except ldap.LDAPError:
+            e = sys.exc_info()[1]
             #connection failed
             return ('error', 'LDAP connect failed', e,)
 
@@ -134,7 +137,8 @@ class ldapauth(object):
             self.is_superuser = False
 
             return
-        except KeyError, e:
+        except KeyError:
+            e = sys.exc_info()[1]
             raise LDAPAUTHError("Portions of the LDAP User profile not present")
 
     def member_of(self):
diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py
index c44c7bf16..4e641f13c 100644
--- a/src/lib/Server/Hostbase/settings.py
+++ b/src/lib/Server/Hostbase/settings.py
@@ -1,9 +1,10 @@
-from ConfigParser import ConfigParser, NoSectionError, NoOptionError
 import os.path
+# Compatibility import
+from Bcfg2.Bcfg2Py3k import ConfigParser
 
 PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
 
-c = ConfigParser()
+c = ConfigParser.ConfigParser()
 #This needs to be configurable one day somehow
 c.read(['./bcfg2.conf'])
 
diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py
index 8e86cc564..09443d4c0 100644
--- a/src/lib/Server/Lint/Comments.py
+++ b/src/lib/Server/Lint/Comments.py
@@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
             props = self.core.plugins['Properties']
             for propfile, pdata in props.store.entries.items():
                 if os.path.splitext(propfile)[1] == ".xml":
-                    self.check_xml(pdata.name, pdata.data, 'properties')
+                    self.check_xml(pdata.name, pdata.xdata, 'properties')
 
     def check_metadata(self):
         """ check metadata files for required headers """
diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py
new file mode 100644
index 000000000..1e177acff
--- /dev/null
+++ b/src/lib/Server/Lint/MergeFiles.py
@@ -0,0 +1,71 @@
+import os
+from copy import deepcopy
+from difflib import SequenceMatcher
+import Bcfg2.Options
+import Bcfg2.Server.Lint
+
+class MergeFiles(Bcfg2.Server.Lint.ServerPlugin):
+    """ find Probes or Cfg files with multiple similar files that
+    might be merged into one """
+
+    @Bcfg2.Server.Lint.returnErrors
+    def Run(self):
+        if 'Cfg' in self.core.plugins:
+            self.check_cfg()
+        if 'Probes' in self.core.plugins:
+            self.check_probes()
+
+    def check_cfg(self):
+        for filename, entryset in self.core.plugins['Cfg'].entries.items():
+            for mset in self.get_similar(entryset.entries):
+                self.LintError("merge-cfg",
+                               "The following files are similar: %s. "
+                               "Consider merging them into a single Genshi "
+                               "template." %
+                               ", ".join([os.path.join(filename, p)
+                                          for p in mset]))
+
+    def check_probes(self):
+        probes = self.core.plugins['Probes'].probes.entries
+        for mset in self.get_similar(probes):
+                self.LintError("merge-cfg",
+                               "The following probes are similar: %s. "
+                               "Consider merging them into a single probe." %
+                               ", ".join([p for p in mset]))
+
+    def get_similar(self, entries):
+        if "threshold" in self.config:
+            # accept threshold either as a percent (e.g., "threshold=75") or
+            # as a ratio (e.g., "threshold=.75")
+            threshold = float(self.config['threshold'])
+            if threshold > 1:
+                threshold /= 100
+        else:
+            threshold = 0.75
+        rv = []
+        elist = entries.items()
+        while elist:
+            result = self._find_similar(elist.pop(0), deepcopy(elist),
+                                        threshold)
+            if len(result) > 1:
+                elist = [(fname, fdata)
+                         for fname, fdata in elist
+                         if fname not in result]
+                rv.append(result)
+        return rv
+
+    def _find_similar(self, ftuple, others, threshold):
+        fname, fdata = ftuple
+        rv = [fname]
+        while others:
+            cname, cdata = others.pop(0)
+            sm = SequenceMatcher(None, fdata.data, cdata.data)
+            # perform progressively more expensive comparisons
+            if (sm.real_quick_ratio() > threshold and
+                sm.quick_ratio() > threshold and
+                sm.ratio() > threshold):
+                rv.extend(self._find_similar((cname, cdata), deepcopy(others),
+                                             threshold))
+        return rv
+
+        
diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py
index 834608378..8a8406e73 100644
--- a/src/lib/Server/Lint/Validate.py
+++ b/src/lib/Server/Lint/Validate.py
@@ -1,10 +1,12 @@
+import fnmatch
 import glob
 import lxml.etree
 import os
-import fnmatch
+from subprocess import Popen, PIPE, STDOUT
+import sys
+
 import Bcfg2.Options
 import Bcfg2.Server.Lint
-from subprocess import Popen, PIPE, STDOUT
 
 class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
     """ Ensure that the repo validates """
@@ -14,7 +16,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
         self.filesets = {"metadata:groups":"%s/metadata.xsd",
                          "metadata:clients":"%s/clients.xsd",
                          "info":"%s/info.xsd",
-                         "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd",
+                         "%s/Bundler/*.xml":"%s/bundle.xsd",
+                         "%s/Bundler/*.genshi":"%s/bundle.xsd",
                          "%s/Pkgmgr/*.xml":"%s/pkglist.xsd",
                          "%s/Base/*.xml":"%s/base.xsd",
                          "%s/Rules/*.xml":"%s/rules.xsd",
@@ -33,23 +36,27 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
 
     @Bcfg2.Server.Lint.returnErrors
     def Run(self):
-        self.schemadir = self.config['schema']
+        schemadir = self.config['schema']
         
-        for schemaname, path in self.filesets.items():
+        for path, schemaname in self.filesets.items():
             try:
                 filelist = self.filelists[path]
             except KeyError:
                 filelist = []
-                
+
             if filelist:
                 # avoid loading schemas for empty file lists
                 try:
                     schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname %
                                                                    schemadir))
+                except IOError:
+                    e = sys.exc_info()[1]
+                    self.LintError("input-output-error", e.message)
+                    continue
                 except:
                     self.LintError("schema-failed-to-parse",
-                                   "Failed to process schema %s",
-                                   schemaname % schemadir)
+                                   "Failed to process schema %s" %
+                                   (schemaname % schemadir))
                     continue
                 for filename in filelist:
                     self.validate(filename, schemaname % schemadir,
diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py
index 3b89d1f9e..63cb2463b 100644
--- a/src/lib/Server/Lint/__init__.py
+++ b/src/lib/Server/Lint/__init__.py
@@ -4,6 +4,7 @@ __all__ = ['Bundles',
            'Comments',
            'Duplicates',
            'InfoXML',
+           'MergeFiles',
            'Pkgmgr',
            'RequiredAttrs',
            'Validate']
@@ -11,6 +12,7 @@ __all__ = ['Bundles',
 import logging
 import os.path
 from copy import copy
+import textwrap
 import lxml.etree
 import Bcfg2.Logger
 
@@ -84,7 +86,10 @@ class ErrorHandler (object):
                "properties-schema-not-found":"warning",
                "xml-failed-to-parse":"error",
                "xml-failed-to-read":"error",
-               "xml-failed-to-verify":"error",}
+               "xml-failed-to-verify":"error",
+               "merge-cfg":"warning",
+               "merge-probes":"warning",
+               "input-output-error": "error"}
 
     def __init__(self, config=None):
         self.errors = 0
@@ -92,6 +97,9 @@ class ErrorHandler (object):
 
         self.logger = logging.getLogger('bcfg2-lint')
 
+        self._wrapper = textwrap.TextWrapper(initial_indent = "  ",
+                                             subsequent_indent = "  ")
+
         self._handlers = {}
         if config is not None:
             for err, action in config.items():
@@ -116,26 +124,42 @@ class ErrorHandler (object):
             self._handlers[err](msg)
             self.logger.debug("    (%s)" % err)
         else:
-            self.logger.info("Unknown error %s" % err)
+            # assume that it's an error, but complain
+            self.error(msg)
+            self.logger.warning("Unknown error %s" % err)
 
     def error(self, msg):
         """ log an error condition """
         self.errors += 1
-        lines = msg.splitlines()
-        self.logger.error("ERROR: %s" % lines.pop())
-        [self.logger.error("  %s" % l) for l in lines]
+        self._log(msg, self.logger.error, prefix="ERROR: ")
 
     def warn(self, msg):
         """ log a warning condition """
         self.warnings += 1
-        lines = msg.splitlines()
-        self.logger.warning("WARNING: %s" % lines.pop())
-        [self.logger.warning("  %s" % l) for l in lines]
+        self._log(msg, self.logger.warning, prefix="WARNING: ")
 
     def debug(self, msg):
         """ log a silent/debug condition """
-        lines = msg.splitlines()
-        [self.logger.debug("%s" % l) for l in lines]
+        self._log(msg, self.logger.debug)
+
+    def _log(self, msg, logfunc, prefix=""):
+        # a message may itself consist of multiple lines.  wrap() will
+        # elide them all into a single paragraph, which we don't want.
+        # so we split the message into its paragraphs and wrap each
+        # paragraph individually.  this means, unfortunately, that we
+        # lose textwrap's built-in initial indent functionality,
+        # because we want to only treat the very first line of the
+        # first paragraph specially.  so we do some silliness.
+        rawlines = msg.splitlines()
+        firstline = True
+        for rawline in rawlines:
+            lines = self._wrapper.wrap(rawline)
+            for line in lines:
+                if firstline:
+                    logfunc("%s%s" % (prefix, line.lstrip()))
+                    firstline = False
+                else:
+                    logfunc(line)
 
 
 class ServerlessPlugin (Plugin):
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index cd2b63656..30c4f9686 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked):
     def Index(self):
         """Build local data structures."""
         try:
-            xdata = XML(self.data)
+            self.xdata = XML(self.data)
         except XMLSyntaxError:
             logger.error("Failed to parse %s" % (self.name))
             return
-        self.label = xdata.attrib[self.__identifier__]
-        self.entries = xdata.getchildren()
+        self.entries = self.xdata.getchildren()
+        if self.__identifier__ is not None:
+            self.label = self.xdata.attrib[self.__identifier__]
 
     def __iter__(self):
         return iter(self.entries)
@@ -455,53 +456,51 @@ class SingleXMLFileBacked(XMLFileBacked):
 
 class StructFile(XMLFileBacked):
     """This file contains a set of structure file formatting logic."""
+    __identifier__ = None
+    
     def __init__(self, name):
         XMLFileBacked.__init__(self, name)
-        self.fragments = {}
-
-    def Index(self):
-        """Build internal data structures."""
-        try:
-            xdata = lxml.etree.XML(self.data)
-        except lxml.etree.XMLSyntaxError:
-            logger.error("Failed to parse file %s" % self.name)
-            return
-        self.fragments = {}
-        work = {lambda x: True: xdata.getchildren()}
-        while work:
-            (predicate, worklist) = work.popitem()
-            self.fragments[predicate] = \
-                                      [item for item in worklist
-                                       if (item.tag != 'Group' and
-                                           item.tag != 'Client' and
-                                           not isinstance(item,
-                                                          lxml.etree._Comment))]
-            for item in worklist:
-                cmd = None
-                if item.tag == 'Group':
-                    if item.get('negate', 'false').lower() == 'true':
-                        cmd = "lambda x:'%s' not in x.groups and predicate(x)"
-                    else:
-                        cmd = "lambda x:'%s' in x.groups and predicate(x)"
-                elif item.tag == 'Client':
-                    if item.get('negate', 'false').lower() == 'true':
-                        cmd = "lambda x:x.hostname != '%s' and predicate(x)"
-                    else:
-                        cmd = "lambda x:x.hostname == '%s' and predicate(x)"
-                # else, ignore item
-                if cmd is not None:
-                    newpred = eval(cmd % item.get('name'),
-                                   {'predicate':predicate})
-                    work[newpred] = item.getchildren()
-
+        self.matches = {}
+
+    def _match(self, item, metadata):
+        """ recursive helper for Match() """
+        if isinstance(item, lxml.etree._Comment):
+            return []
+        elif item.tag == 'Group':
+            rv = []
+            if ((item.get('negate', 'false').lower() == 'true' and
+                 item.get('name') not in metadata.groups) or
+                (item.get('negate', 'false').lower() == 'false' and
+                 item.get('name') in metadata.groups)):
+                for child in item.iterchildren():
+                    rv.extend(self._match(child, metadata))
+            return rv
+        elif item.tag == 'Client':
+            rv = []
+            if ((item.get('negate', 'false').lower() == 'true' and
+                 item.get('name') != metadata.hostname) or
+                (item.get('negate', 'false').lower() == 'false' and
+                 item.get('name') == metadata.hostname)):
+                for child in item.iterchildren():
+                    rv.extend(self._match(child, metadata))
+            return rv
+        else:
+            rv = copy.deepcopy(item)
+            for child in rv.iterchildren():
+                rv.remove(child)
+            for child in item.iterchildren():
+                rv.extend(self._match(child, metadata))
+            return [rv]
+            
     def Match(self, metadata):
         """Return matching fragments of independent."""
-        matching = [frag for (pred, frag) in list(self.fragments.items())
-                    if pred(metadata)]
-        if matching:
-            return reduce(lambda x, y: x + y, matching)
-        logger.error("File %s got null match" % (self.name))
-        return []
+        if metadata.hostname not in self.matches:
+            rv = []
+            for child in self.entries:
+                rv.extend(self._match(child, metadata))
+            logger.debug("File %s got %d match(es)" % (self.name, len(rv)))
+            self.matches[metadata.hostname] = rv
+        return self.matches[metadata.hostname]
 
 
 class INode:
@@ -644,9 +643,9 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked):
 
     def BindEntry(self, entry, metadata):
         """Check package lists of package entries."""
-        [src.Cache(metadata) for src in list(self.entries.values())]
         name = entry.get('name')
-        if not src.cache:
+        if False in [src.Cache(metadata) for src in
+                     list(self.entries.values())]:
             self.logger.error("Called before data loaded")
             raise PluginExecutionError
         matching = [src for src in list(self.entries.values())
@@ -693,6 +692,9 @@ class Specificity:
         self.prio = prio
         self.delta = delta
 
+    def __lt__(self, other):
+        return self.__cmp__(other) < 0
+
     def matches(self, metadata):
         return self.all or \
                self.hostname == metadata.hostname or \
diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py
index 5e7d89727..5de57a87c 100644
--- a/src/lib/Server/Plugins/Base.py
+++ b/src/lib/Server/Plugins/Base.py
@@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin,
     __version__ = '$Id$'
     __author__ = 'bcfg-dev@mcs.anl.gov'
     __child__ = Bcfg2.Server.Plugin.StructFile
+    deprecated = True
 
     """Base creates independent clauses based on client metadata."""
     def __init__(self, core, datastore):
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 998bacc19..23ba0a745 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -4,8 +4,11 @@ __revision__ = '$Revision$'
 import binascii
 import logging
 import lxml
+import operator
 import os
+import os.path
 import re
+import stat
 import sys
 import tempfile
 
@@ -22,9 +25,10 @@ except:
 logger = logging.getLogger('Bcfg2.Plugins.Cfg')
 
 
+# py3k compatibility
 def u_str(string, encoding):
     if sys.hexversion >= 0x03000000:
-        return str(string, encoding)
+        return string.encode(encoding)
     else:
         return unicode(string, encoding)
 
@@ -94,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path,
                                               entry_type, encoding)
         self.specific = CfgMatcher(path.split('/')[-1])
+        path = path
 
     def sort_by_specific(self, one, other):
         return cmp(one.specific, other.specific)
@@ -104,7 +109,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         """
         matching = [ent for ent in list(self.entries.values()) if \
                     ent.specific.matches(metadata)]
-        matching.sort(self.sort_by_specific)
+        matching.sort(key=operator.attrgetter('specific'))
         non_delta = [matching.index(m) for m in matching
                      if not m.specific.delta]
         if not non_delta:
@@ -118,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         self.bind_info_to_entry(entry, metadata)
         used = self.get_pertinent_entries(metadata)
         basefile = used.pop(0)
+        if entry.get('perms').lower() == 'inherit':
+            # use on-disk permissions
+            fname = "%s/%s" % (self.path, entry.get('name'))
+            entry.set('perms',
+                      str(oct(stat.S_IMODE(os.stat(fname).st_mode))))
         if entry.tag == 'Path':
             entry.set('type', 'file')
         if basefile.name.endswith(".genshi"):
@@ -134,9 +144,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
                                            metadata=metadata,
                                            path=basefile.name).filter(removecomment)
                 try:
-                    data = stream.render('text', strip_whitespace=False)
+                    data = stream.render('text', encoding=self.encoding,
+                                         strip_whitespace=False)
                 except TypeError:
-                    data = stream.render('text')
+                    data = stream.render('text', encoding=self.encoding)
                 if data == '':
                     entry.set('empty', 'true')
             except Exception:
@@ -160,6 +171,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
                 logger.error("Failed to decode %s: %s" % (entry.get('name'), e))
                 logger.error("Please verify you are using the proper encoding.")
                 raise Bcfg2.Server.Plugin.PluginExecutionError
+            except ValueError:
+                e = sys.exc_info()[1]
+                logger.error("Error in specification for %s" % entry.get('name'))
+                logger.error("%s" % e)
+                logger.error("You need to specify base64 encoding for %s." %
+                             entry.get('name'))
+                raise Bcfg2.Server.Plugin.PluginExecutionError
         if entry.text in ['', None]:
             entry.set('empty', 'true')
 
@@ -185,10 +203,15 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
     def write_update(self, specific, new_entry, log):
         if 'text' in new_entry:
             name = self.build_filename(specific)
-            if name.endswith(".genshi"):
+            if os.path.exists("%s.genshi" % name):
                 logger.error("Cfg: Unable to pull data for genshi types")
                 raise Bcfg2.Server.Plugin.PluginExecutionError
-            open(name, 'w').write(new_entry['text'])
+            try:
+                etext = new_entry['text'].encode(self.encoding)
+            except:
+                logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding))
+                raise Bcfg2.Server.Plugin.PluginExecutionError
+            open(name, 'w').write(etext)
             if log:
                 logger.info("Wrote file %s" % name)
         badattr = [attr for attr in ['owner', 'group', 'perms']
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index 5ef1920e1..103fb7353 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
             try:
                 Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata,
                                                              container,
+                                                             self.core.encoding,
                                                              0,
                                                              logger,
                                                              True,
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index ca6e43851..6570f2912 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                                   (address[0]))
                 return False
         # populate the session cache
-        if user != 'root':
+        if user.decode('utf-8') != 'root':
             self.session_cache[address] = (time.time(), client)
         return True
 
diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py
index ea2e79ccc..ec0f294dd 100644
--- a/src/lib/Server/Plugins/Probes.py
+++ b/src/lib/Server/Plugins/Probes.py
@@ -1,12 +1,100 @@
 import lxml.etree
+import operator
 import re
 
+try:
+    import json
+    has_json = True
+except ImportError:
+    try:
+        import simplejson as json
+        has_json = True
+    except ImportError:
+        has_json = False
+
+try:
+    import syck
+    has_syck = True
+except ImportError:
+    has_syck = False
+    try:
+        import yaml
+        has_yaml = True
+    except ImportError:
+        has_yaml = False
+
 import Bcfg2.Server.Plugin
 
 specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)")
 probe_matcher = re.compile("(.*/)?(?P<basename>\S+)")
 
 
+class ProbeData (object):
+    """ a ProbeData object emulates a str object, but also has .xdata
+    and .json properties to provide convenient ways to use ProbeData
+    objects as XML or JSON data """
+    def __init__(self, data):
+        self.data = data
+        self._xdata = None
+        self._json = None
+        self._yaml = None
+
+    def __str__(self):
+        return str(self.data)
+    
+    def __repr__(self):
+        return repr(self.data)
+    
+    def __getattr__(self, name):
+        """ make ProbeData act like a str object """
+        return getattr(self.data, name)
+
+    def __complex__(self):
+        return complex(self.data)
+
+    def __int__(self):
+        return int(self.data)
+
+    def __long__(self):
+        return long(self.data)
+
+    def __float__(self):
+        return float(self.data)
+
+    @property
+    def xdata(self):
+        if self._xdata is None:
+            try:
+                self._xdata = lxml.etree.XML(self.data)
+            except lxml.etree.XMLSyntaxError:
+                pass
+        return self._xdata
+
+    @property
+    def json(self):
+        if self._json is None and has_json:
+            try:
+                self._json = json.loads(self.data)
+            except ValueError:
+                pass
+        return self._json
+
+    @property
+    def yaml(self):
+        if self._yaml is None:
+            if has_yaml:
+                try:
+                    self._yaml = yaml.load(self.data)
+                except yaml.YAMLError:
+                    pass
+            elif has_syck:
+                try:
+                    self._yaml = syck.load(self.data)
+                except syck.error:
+                    pass
+        return self._yaml
+
+
 class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
     ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$")
 
@@ -27,7 +115,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
         ret = []
         build = dict()
         candidates = self.get_matching(metadata)
-        candidates.sort(lambda x, y: cmp(x.specific, y.specific))
+        candidates.sort(key=operator.attrgetter('specific'))
         for entry in candidates:
             rem = specific_probe_matcher.match(entry.name)
             if not rem:
@@ -80,7 +168,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             cx = lxml.etree.SubElement(top, 'Client', name=client)
             for probe in sorted(probed):
                 lxml.etree.SubElement(cx, 'Probe', name=probe,
-                                      value=self.probedata[client][probe])
+                                      value=str(self.probedata[client][probe]))
             for group in sorted(self.cgroups[client]):
                 lxml.etree.SubElement(cx, "Group", name=group)
         data = lxml.etree.tostring(top, encoding='UTF-8',
@@ -90,7 +178,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
         except IOError:
             self.logger.error("Failed to write probed.xml")
-        datafile.write(data)
+        datafile.write(data.decode('utf-8'))
 
     def load_data(self):
         try:
@@ -105,7 +193,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             self.cgroups[client.get('name')] = []
             for pdata in client:
                 if (pdata.tag == 'Probe'):
-                    self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value')
+                    self.probedata[client.get('name')][pdata.get('name')] = \
+                        ProbeData(pdata.get('value'))
                 elif (pdata.tag == 'Group'):
                     self.cgroups[client.get('name')].append(pdata.get('name'))
 
@@ -128,9 +217,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             self.logger.error("Got null response to probe %s from %s" % \
                               (data.get('name'), client.hostname))
             try:
-                self.probedata[client.hostname].update({data.get('name'): ''})
+                self.probedata[client.hostname].update({data.get('name'):
+                                                        ProbeData('')})
             except KeyError:
-                self.probedata[client.hostname] = {data.get('name'): ''}
+                self.probedata[client.hostname] = \
+                    {data.get('name'): ProbeData('')}
             return
         dlines = data.text.split('\n')
         self.logger.debug("%s:probe:%s:%s" % (client.hostname,
@@ -141,11 +232,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
                 if newgroup not in self.cgroups[client.hostname]:
                     self.cgroups[client.hostname].append(newgroup)
                 dlines.remove(line)
-        dtext = "\n".join(dlines)
+        dobj = ProbeData("\n".join(dlines))
         try:
-            self.probedata[client.hostname].update({data.get('name'): dtext})
+            self.probedata[client.hostname].update({data.get('name'): dobj})
         except KeyError:
-            self.probedata[client.hostname] = {data.get('name'): dtext}
+            self.probedata[client.hostname] = {data.get('name'): dobj}
 
     def get_additional_groups(self, meta):
         return self.cgroups.get(meta.hostname, list())
diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py
index dea797a10..54c5def57 100644
--- a/src/lib/Server/Plugins/Properties.py
+++ b/src/lib/Server/Plugins/Properties.py
@@ -6,44 +6,7 @@ import Bcfg2.Server.Plugin
 
 class PropertyFile(Bcfg2.Server.Plugin.StructFile):
     """Class for properties files."""
-    def Index(self):
-        """Build internal data structures."""
-        if type(self.data) is not lxml.etree._Element:
-            try:
-                self.data = lxml.etree.XML(self.data)
-            except lxml.etree.XMLSyntaxError:
-                Bcfg2.Server.Plugin.logger.error("Failed to parse %s" %
-                                                 self.name)
-
-        self.fragments = {}
-        work = {lambda x: True: self.data.getchildren()}
-        while work:
-            (predicate, worklist) = work.popitem()
-            self.fragments[predicate] = \
-                                      [item for item in worklist
-                                       if (item.tag != 'Group' and
-                                           item.tag != 'Client' and
-                                           not isinstance(item,
-                                                          lxml.etree._Comment))]
-            for item in worklist:
-                cmd = None
-                if item.tag == 'Group':
-                    if item.get('negate', 'false').lower() == 'true':
-                        cmd = "lambda x:'%s' not in x.groups and predicate(x)"
-                    else:
-                        cmd = "lambda x:'%s' in x.groups and predicate(x)"
-                elif item.tag == 'Client':
-                    if item.get('negate', 'false').lower() == 'true':
-                        cmd = "lambda x:x.hostname != '%s' and predicate(x)"
-                    else:
-                        cmd = "lambda x:x.hostname == '%s' and predicate(x)"
-                # else, ignore item
-                if cmd is not None:
-                    newpred = eval(cmd % item.get('name'),
-                                   {'predicate':predicate})
-                    work[newpred] = item.getchildren()
-
-
+    pass
 
 class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
     __child__ = PropertyFile
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
index cf0998aaa..4a33c0cb0 100644
--- a/src/lib/Server/Plugins/SSHbase.py
+++ b/src/lib/Server/Plugins/SSHbase.py
@@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     __author__ = 'bcfg-dev@mcs.anl.gov'
 
     pubkeys = ["ssh_host_dsa_key.pub.H_%s",
-                "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"]
+               "ssh_host_rsa_key.pub.H_%s",
+               "ssh_host_key.pub.H_%s"]
     hostkeys = ["ssh_host_dsa_key.H_%s",
-                "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"]
-    keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key',
-                   'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub',
-                   'ssh_host_key.pub']
+                "ssh_host_rsa_key.H_%s",
+                "ssh_host_key.H_%s"]
+    keypatterns = ["ssh_host_dsa_key",
+                   "ssh_host_rsa_key",
+                   "ssh_host_key",
+                   "ssh_host_dsa_key.pub",
+                   "ssh_host_rsa_key.pub",
+                   "ssh_host_key.pub"]
 
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
@@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     def get_skn(self):
         """Build memory cache of the ssh known hosts file."""
         if not self.__skn:
-            self.__skn = "\n".join([str(value.data) for key, value in \
+            self.__skn = "\n".join([value.data.decode() for key, value in \
                                     list(self.entries.items()) if \
                                     key.endswith('.static')])
             names = dict()
@@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
                         self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname)
                     continue
                 self.__skn += "%s %s" % (','.join(names[hostname]),
-                                         self.entries[pubkey].data)
+                                         self.entries[pubkey].data.decode())
         return self.__skn
 
     def set_skn(self, value):
@@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
         hostkeys.sort()
         for hostkey in hostkeys:
             entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
-                self.entries[hostkey].data)
+                self.entries[hostkey].data.decode())
         permdata = {'owner': 'root',
                     'group': 'root',
                     'type': 'file',
diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py
index 8b6bad574..aeb3b9f74 100644
--- a/src/lib/Server/Plugins/Snapshots.py
+++ b/src/lib/Server/Plugins/Snapshots.py
@@ -28,6 +28,7 @@ datafields = {
              }
 
 
+# py3k compatibility
 def u_str(string):
     if sys.hexversion >= 0x03000000:
         return string
diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
index b6a3c2599..68774cec6 100755
--- a/src/lib/Server/Reports/importscript.py
+++ b/src/lib/Server/Reports/importscript.py
@@ -38,7 +38,7 @@ import platform
 from Bcfg2.Bcfg2Py3k import ConfigParser
 
 
-def build_reason_kwargs(r_ent):
+def build_reason_kwargs(r_ent, encoding, logger):
     binary_file = False
     if r_ent.get('current_bfile', False):
         binary_file = True
@@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent):
         rc_diff = r_ent.get('current_diff')
     else:
         rc_diff = ''
+    if not binary_file:
+        try:
+            rc_diff = rc_diff.decode(encoding)
+        except:
+            logger.error("Reason isn't %s encoded, cannot decode it" % encoding)
+            rc_diff = ''
     return dict(owner=r_ent.get('owner', default=""),
                 current_owner=r_ent.get('current_owner', default=""),
                 group=r_ent.get('group', default=""),
@@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent):
                 is_binary=binary_file)
 
 
-def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
+def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''):
     clients = {}
     [clients.__setitem__(c.name, c) \
         for c in Client.objects.all()]
@@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
             for (xpath, type) in pattern:
                 for x in statistics.findall(xpath):
                     counter_fields[type] = counter_fields[type] + 1
-                    kargs = build_reason_kwargs(x)
+                    kargs = build_reason_kwargs(x, encoding, logger)
 
                     try:
                         rr = None
@@ -270,6 +276,11 @@ if __name__ == '__main__':
         print("StatReports: Failed to parse %s" % (statpath))
         raise SystemExit(1)
 
+    try:
+        encoding = cf.get('components', 'encoding')
+    except:
+        encoding = 'UTF-8'
+
     if not clientpath:
         try:
             clientspath = "%s/Metadata/clients.xml" % \
@@ -288,6 +299,7 @@ if __name__ == '__main__':
     update_database()
     load_stats(clientsdata,
                statsdata,
+               encoding,
                verb,
                logger,
                quick=q,
diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html
index 6ef4c9aff..ec9a17468 100644
--- a/src/lib/Server/Reports/reports/templates/base.html
+++ b/src/lib/Server/Reports/reports/templates/base.html
@@ -87,7 +87,7 @@
   <div style='clear:both'></div>
 </div><!-- document -->
   <div id="footer">
-    <span>Bcfg2 Version 1.2.0pre2</span>
+    <span>Bcfg2 Version 1.2.0pre3</span>
     </div>
 
 <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div>
diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py
index 291528e2e..2e30125f9 100644
--- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py
+++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py
@@ -15,6 +15,7 @@ try:
 except:
     colorize = False
 
+# py3k compatibility
 def u_str(string):
     if sys.hexversion >= 0x03000000:
         return string
diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py
index 2aa35f1ec..f30c38a05 100644
--- a/src/lib/Server/Snapshots/model.py
+++ b/src/lib/Server/Snapshots/model.py
@@ -7,6 +7,7 @@ from sqlalchemy.orm import relation, backref
 from sqlalchemy.ext.declarative import declarative_base
 
 
+# py3k compatibility
 def u_str(string):
     if sys.hexversion >= 0x03000000:
         return string