Merge branch 'master' of git.mcs.anl.gov:bcfg2

8 files changed, 147 insertions, 62 deletions

diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py
index 5e7d89727..5de57a87c 100644
--- a/src/lib/Server/Plugins/Base.py
+++ b/src/lib/Server/Plugins/Base.py
@@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin,
     __version__ = '$Id$'
     __author__ = 'bcfg-dev@mcs.anl.gov'
     __child__ = Bcfg2.Server.Plugin.StructFile
+    deprecated = True
 
     """Base creates independent clauses based on client metadata."""
     def __init__(self, core, datastore):
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 998bacc19..23ba0a745 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -4,8 +4,11 @@ __revision__ = '$Revision$'
 import binascii
 import logging
 import lxml
+import operator
 import os
+import os.path
 import re
+import stat
 import sys
 import tempfile
 
@@ -22,9 +25,10 @@ except:
 logger = logging.getLogger('Bcfg2.Plugins.Cfg')
 
 
+# py3k compatibility
 def u_str(string, encoding):
     if sys.hexversion >= 0x03000000:
-        return str(string, encoding)
+        return string.encode(encoding)
     else:
         return unicode(string, encoding)
 
@@ -94,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path,
                                               entry_type, encoding)
         self.specific = CfgMatcher(path.split('/')[-1])
+        path = path
 
     def sort_by_specific(self, one, other):
         return cmp(one.specific, other.specific)
@@ -104,7 +109,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         """
         matching = [ent for ent in list(self.entries.values()) if \
                     ent.specific.matches(metadata)]
-        matching.sort(self.sort_by_specific)
+        matching.sort(key=operator.attrgetter('specific'))
         non_delta = [matching.index(m) for m in matching
                      if not m.specific.delta]
         if not non_delta:
@@ -118,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         self.bind_info_to_entry(entry, metadata)
         used = self.get_pertinent_entries(metadata)
         basefile = used.pop(0)
+        if entry.get('perms').lower() == 'inherit':
+            # use on-disk permissions
+            fname = "%s/%s" % (self.path, entry.get('name'))
+            entry.set('perms',
+                      str(oct(stat.S_IMODE(os.stat(fname).st_mode))))
         if entry.tag == 'Path':
             entry.set('type', 'file')
         if basefile.name.endswith(".genshi"):
@@ -134,9 +144,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
                                            metadata=metadata,
                                            path=basefile.name).filter(removecomment)
                 try:
-                    data = stream.render('text', strip_whitespace=False)
+                    data = stream.render('text', encoding=self.encoding,
+                                         strip_whitespace=False)
                 except TypeError:
-                    data = stream.render('text')
+                    data = stream.render('text', encoding=self.encoding)
                 if data == '':
                     entry.set('empty', 'true')
             except Exception:
@@ -160,6 +171,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
                 logger.error("Failed to decode %s: %s" % (entry.get('name'), e))
                 logger.error("Please verify you are using the proper encoding.")
                 raise Bcfg2.Server.Plugin.PluginExecutionError
+            except ValueError:
+                e = sys.exc_info()[1]
+                logger.error("Error in specification for %s" % entry.get('name'))
+                logger.error("%s" % e)
+                logger.error("You need to specify base64 encoding for %s." %
+                             entry.get('name'))
+                raise Bcfg2.Server.Plugin.PluginExecutionError
         if entry.text in ['', None]:
             entry.set('empty', 'true')
 
@@ -185,10 +203,15 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
     def write_update(self, specific, new_entry, log):
         if 'text' in new_entry:
             name = self.build_filename(specific)
-            if name.endswith(".genshi"):
+            if os.path.exists("%s.genshi" % name):
                 logger.error("Cfg: Unable to pull data for genshi types")
                 raise Bcfg2.Server.Plugin.PluginExecutionError
-            open(name, 'w').write(new_entry['text'])
+            try:
+                etext = new_entry['text'].encode(self.encoding)
+            except:
+                logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding))
+                raise Bcfg2.Server.Plugin.PluginExecutionError
+            open(name, 'w').write(etext)
             if log:
                 logger.info("Wrote file %s" % name)
         badattr = [attr for attr in ['owner', 'group', 'perms']
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index 5ef1920e1..103fb7353 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
             try:
                 Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata,
                                                              container,
+                                                             self.core.encoding,
                                                              0,
                                                              logger,
                                                              True,
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index ca6e43851..6570f2912 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                                   (address[0]))
                 return False
         # populate the session cache
-        if user != 'root':
+        if user.decode('utf-8') != 'root':
             self.session_cache[address] = (time.time(), client)
         return True
 
diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py
index ea2e79ccc..ec0f294dd 100644
--- a/src/lib/Server/Plugins/Probes.py
+++ b/src/lib/Server/Plugins/Probes.py
@@ -1,12 +1,100 @@
 import lxml.etree
+import operator
 import re
 
+try:
+    import json
+    has_json = True
+except ImportError:
+    try:
+        import simplejson as json
+        has_json = True
+    except ImportError:
+        has_json = False
+
+try:
+    import syck
+    has_syck = True
+except ImportError:
+    has_syck = False
+    try:
+        import yaml
+        has_yaml = True
+    except ImportError:
+        has_yaml = False
+
 import Bcfg2.Server.Plugin
 
 specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)")
 probe_matcher = re.compile("(.*/)?(?P<basename>\S+)")
 
 
+class ProbeData (object):
+    """ a ProbeData object emulates a str object, but also has .xdata
+    and .json properties to provide convenient ways to use ProbeData
+    objects as XML or JSON data """
+    def __init__(self, data):
+        self.data = data
+        self._xdata = None
+        self._json = None
+        self._yaml = None
+
+    def __str__(self):
+        return str(self.data)
+    
+    def __repr__(self):
+        return repr(self.data)
+    
+    def __getattr__(self, name):
+        """ make ProbeData act like a str object """
+        return getattr(self.data, name)
+
+    def __complex__(self):
+        return complex(self.data)
+
+    def __int__(self):
+        return int(self.data)
+
+    def __long__(self):
+        return long(self.data)
+
+    def __float__(self):
+        return float(self.data)
+
+    @property
+    def xdata(self):
+        if self._xdata is None:
+            try:
+                self._xdata = lxml.etree.XML(self.data)
+            except lxml.etree.XMLSyntaxError:
+                pass
+        return self._xdata
+
+    @property
+    def json(self):
+        if self._json is None and has_json:
+            try:
+                self._json = json.loads(self.data)
+            except ValueError:
+                pass
+        return self._json
+
+    @property
+    def yaml(self):
+        if self._yaml is None:
+            if has_yaml:
+                try:
+                    self._yaml = yaml.load(self.data)
+                except yaml.YAMLError:
+                    pass
+            elif has_syck:
+                try:
+                    self._yaml = syck.load(self.data)
+                except syck.error:
+                    pass
+        return self._yaml
+
+
 class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
     ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$")
 
@@ -27,7 +115,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
         ret = []
         build = dict()
         candidates = self.get_matching(metadata)
-        candidates.sort(lambda x, y: cmp(x.specific, y.specific))
+        candidates.sort(key=operator.attrgetter('specific'))
         for entry in candidates:
             rem = specific_probe_matcher.match(entry.name)
             if not rem:
@@ -80,7 +168,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             cx = lxml.etree.SubElement(top, 'Client', name=client)
             for probe in sorted(probed):
                 lxml.etree.SubElement(cx, 'Probe', name=probe,
-                                      value=self.probedata[client][probe])
+                                      value=str(self.probedata[client][probe]))
             for group in sorted(self.cgroups[client]):
                 lxml.etree.SubElement(cx, "Group", name=group)
         data = lxml.etree.tostring(top, encoding='UTF-8',
@@ -90,7 +178,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
         except IOError:
             self.logger.error("Failed to write probed.xml")
-        datafile.write(data)
+        datafile.write(data.decode('utf-8'))
 
     def load_data(self):
         try:
@@ -105,7 +193,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             self.cgroups[client.get('name')] = []
             for pdata in client:
                 if (pdata.tag == 'Probe'):
-                    self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value')
+                    self.probedata[client.get('name')][pdata.get('name')] = \
+                        ProbeData(pdata.get('value'))
                 elif (pdata.tag == 'Group'):
                     self.cgroups[client.get('name')].append(pdata.get('name'))
 
@@ -128,9 +217,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
             self.logger.error("Got null response to probe %s from %s" % \
                               (data.get('name'), client.hostname))
             try:
-                self.probedata[client.hostname].update({data.get('name'): ''})
+                self.probedata[client.hostname].update({data.get('name'):
+                                                        ProbeData('')})
             except KeyError:
-                self.probedata[client.hostname] = {data.get('name'): ''}
+                self.probedata[client.hostname] = \
+                    {data.get('name'): ProbeData('')}
             return
         dlines = data.text.split('\n')
         self.logger.debug("%s:probe:%s:%s" % (client.hostname,
@@ -141,11 +232,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin,
                 if newgroup not in self.cgroups[client.hostname]:
                     self.cgroups[client.hostname].append(newgroup)
                 dlines.remove(line)
-        dtext = "\n".join(dlines)
+        dobj = ProbeData("\n".join(dlines))
         try:
-            self.probedata[client.hostname].update({data.get('name'): dtext})
+            self.probedata[client.hostname].update({data.get('name'): dobj})
         except KeyError:
-            self.probedata[client.hostname] = {data.get('name'): dtext}
+            self.probedata[client.hostname] = {data.get('name'): dobj}
 
     def get_additional_groups(self, meta):
         return self.cgroups.get(meta.hostname, list())
diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py
index dea797a10..54c5def57 100644
--- a/src/lib/Server/Plugins/Properties.py
+++ b/src/lib/Server/Plugins/Properties.py
@@ -6,44 +6,7 @@ import Bcfg2.Server.Plugin
 
 class PropertyFile(Bcfg2.Server.Plugin.StructFile):
     """Class for properties files."""
-    def Index(self):
-        """Build internal data structures."""
-        if type(self.data) is not lxml.etree._Element:
-            try:
-                self.data = lxml.etree.XML(self.data)
-            except lxml.etree.XMLSyntaxError:
-                Bcfg2.Server.Plugin.logger.error("Failed to parse %s" %
-                                                 self.name)
-
-        self.fragments = {}
-        work = {lambda x: True: self.data.getchildren()}
-        while work:
-            (predicate, worklist) = work.popitem()
-            self.fragments[predicate] = \
-                                      [item for item in worklist
-                                       if (item.tag != 'Group' and
-                                           item.tag != 'Client' and
-                                           not isinstance(item,
-                                                          lxml.etree._Comment))]
-            for item in worklist:
-                cmd = None
-                if item.tag == 'Group':
-                    if item.get('negate', 'false').lower() == 'true':
-                        cmd = "lambda x:'%s' not in x.groups and predicate(x)"
-                    else:
-                        cmd = "lambda x:'%s' in x.groups and predicate(x)"
-                elif item.tag == 'Client':
-                    if item.get('negate', 'false').lower() == 'true':
-                        cmd = "lambda x:x.hostname != '%s' and predicate(x)"
-                    else:
-                        cmd = "lambda x:x.hostname == '%s' and predicate(x)"
-                # else, ignore item
-                if cmd is not None:
-                    newpred = eval(cmd % item.get('name'),
-                                   {'predicate':predicate})
-                    work[newpred] = item.getchildren()
-
-
+    pass
 
 class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
     __child__ = PropertyFile
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
index cf0998aaa..4a33c0cb0 100644
--- a/src/lib/Server/Plugins/SSHbase.py
+++ b/src/lib/Server/Plugins/SSHbase.py
@@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     __author__ = 'bcfg-dev@mcs.anl.gov'
 
     pubkeys = ["ssh_host_dsa_key.pub.H_%s",
-                "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"]
+               "ssh_host_rsa_key.pub.H_%s",
+               "ssh_host_key.pub.H_%s"]
     hostkeys = ["ssh_host_dsa_key.H_%s",
-                "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"]
-    keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key',
-                   'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub',
-                   'ssh_host_key.pub']
+                "ssh_host_rsa_key.H_%s",
+                "ssh_host_key.H_%s"]
+    keypatterns = ["ssh_host_dsa_key",
+                   "ssh_host_rsa_key",
+                   "ssh_host_key",
+                   "ssh_host_dsa_key.pub",
+                   "ssh_host_rsa_key.pub",
+                   "ssh_host_key.pub"]
 
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
@@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     def get_skn(self):
         """Build memory cache of the ssh known hosts file."""
         if not self.__skn:
-            self.__skn = "\n".join([str(value.data) for key, value in \
+            self.__skn = "\n".join([value.data.decode() for key, value in \
                                     list(self.entries.items()) if \
                                     key.endswith('.static')])
             names = dict()
@@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
                         self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname)
                     continue
                 self.__skn += "%s %s" % (','.join(names[hostname]),
-                                         self.entries[pubkey].data)
+                                         self.entries[pubkey].data.decode())
         return self.__skn
 
     def set_skn(self, value):
@@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
         hostkeys.sort()
         for hostkey in hostkeys:
             entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
-                self.entries[hostkey].data)
+                self.entries[hostkey].data.decode())
         permdata = {'owner': 'root',
                     'group': 'root',
                     'type': 'file',
diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py
index 8b6bad574..aeb3b9f74 100644
--- a/src/lib/Server/Plugins/Snapshots.py
+++ b/src/lib/Server/Plugins/Snapshots.py
@@ -28,6 +28,7 @@ datafields = {
              }
 
 
+# py3k compatibility
 def u_str(string):
     if sys.hexversion >= 0x03000000:
         return string