Add a number of features to SSHbase:

* Support for group-specific host keys
* Support for fully static host- and group-specific ssh_known_hosts
* (Support for totally generic host keys and ssh_known_hosts, too, but that's pretty useless.)
* Support for info.xml, info, and :info files; only info.xml is likely to be useful, with the <Path> directive

4 files changed, 228 insertions, 153 deletions

diff --git a/src/lib/Bcfg2Py3k.py b/src/lib/Bcfg2Py3k.py
index 4803bf8b2..ee05b7e41 100644
--- a/src/lib/Bcfg2Py3k.py
+++ b/src/lib/Bcfg2Py3k.py
@@ -63,11 +63,17 @@ except ImportError:
     import http.client as httplib
 
 # print to file compatibility
-def u_str(string):
+def u_str(string, encoding=None):
     if sys.hexversion >= 0x03000000:
-        return string
+        if encoding is not None:
+            return string.encode(encoding)
+        else:
+            return string
     else:
-        return unicode(string)
+        if encoding is not None:
+            return unicode(string, encoding)
+        else:
+            return unicode(string)
 
 """
 In order to use the new syntax for printing to a file, we need to do
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index c09a37ed8..190a205e6 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -934,9 +934,29 @@ class EntrySet:
         self.specific = re.compile(pattern)
 
     def get_matching(self, metadata):
-        return [item for item in list(self.entries.values()) \
+        return [item for item in list(self.entries.values())
                 if item.specific.matches(metadata)]
 
+    def best_matching(self, metadata):
+        """ Return the appropriate interpreted template from the set of
+        available templates. """
+        matching = self.get_matching(metadata)
+
+        hspec = [ent for ent in matching if ent.specific.hostname]
+        if hspec:
+            return hspec[0]
+
+        gspec = [ent for ent in matching if ent.specific.group]
+        if gspec:
+            gspec.sort(self.group_sortfunc)
+            return gspec[-1]
+
+        aspec = [ent for ent in matching if ent.specific.all]
+        if aspec:
+            return aspec[0]
+
+        raise PluginExecutionError
+
     def handle_event(self, event):
         """Handle FAM events for the TemplateSet."""
         action = event.code2str()
@@ -1042,22 +1062,7 @@ class EntrySet:
     def bind_entry(self, entry, metadata):
         """Return the appropriate interpreted template from the set of available templates."""
         self.bind_info_to_entry(entry, metadata)
-        matching = self.get_matching(metadata)
-
-        hspec = [ent for ent in matching if ent.specific.hostname]
-        if hspec:
-            return hspec[0].bind_entry(entry, metadata)
-
-        gspec = [ent for ent in matching if ent.specific.group]
-        if gspec:
-            gspec.sort(self.group_sortfunc)
-            return gspec[-1].bind_entry(entry, metadata)
-
-        aspec = [ent for ent in matching if ent.specific.all]
-        if aspec:
-            return aspec[0].bind_entry(entry, metadata)
-
-        raise PluginExecutionError
+        return self.best_matching(metadata).bind_entry(entry, metadata)
 
 
 class GroupSpool(Plugin, Generator):
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index f202628cd..0a791f171 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -12,6 +12,7 @@ import stat
 import sys
 import tempfile
 from subprocess import Popen, PIPE
+from Bcfg2.Bcfg2Py3k import u_str
 
 import Bcfg2.Server.Plugin
 
@@ -33,14 +34,6 @@ except:
 logger = logging.getLogger('Bcfg2.Plugins.Cfg')
 
 
-# py3k compatibility
-def u_str(string, encoding):
-    if sys.hexversion >= 0x03000000:
-        return string.encode(encoding)
-    else:
-        return unicode(string, encoding)
-
-
 # snipped from TGenshi
 def removecomment(stream):
     """A genshi filter that removes comments from the stream."""
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
index e4a9be44c..d31405a57 100644
--- a/src/lib/Server/Plugins/SSHbase.py
+++ b/src/lib/Server/Plugins/SSHbase.py
@@ -9,11 +9,77 @@ import sys
 import tempfile
 from subprocess import Popen, PIPE
 import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import u_str
+
+if sys.hexversion >= 0x03000000:
+    from functools import reduce
+
+import logging
+logger = logging.getLogger(__name__)
+
+DEBUG = logger.error
+
+class KeyData(Bcfg2.Server.Plugin.SpecificData):
+    def __init__(self, name, specific, encoding):
+        Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific,
+                                                  encoding)
+        self.encoding = encoding
+    
+    def bind_entry(self, entry, metadata):
+        entry.set('type', 'file')
+        if entry.get('encoding') == 'base64':
+            entry.text = binascii.b2a_base64(self.data)
+        else:
+            try:
+                entry.text = u_str(self.data, self.encoding)
+            except UnicodeDecodeError:
+                e = sys.exc_info()[1]
+                logger.error("Failed to decode %s: %s" % (entry.get('name'), e))
+                logger.error("Please verify you are using the proper encoding.")
+                raise Bcfg2.Server.Plugin.PluginExecutionError
+            except ValueError:
+                e = sys.exc_info()[1]
+                logger.error("Error in specification for %s" %
+                             entry.get('name'))
+                logger.error(str(e))
+                logger.error("You need to specify base64 encoding for %s." %
+                             entry.get('name'))
+                raise Bcfg2.Server.Plugin.PluginExecutionError
+        if entry.text in ['', None]:
+            entry.set('empty', 'true')
+
+class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet):
+    def __init__(self, basename, path):
+        if basename.startswith("ssh_host_key"):
+            encoding = "base64"
+        else:
+            encoding = None
+        Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData,
+                                              encoding)
+        self.metadata = {'owner': 'root',
+                         'group': 'root',
+                         'type': 'file'}
+        if encoding is not None:
+            self.metadata['encoding'] = encoding
+        if basename.endswith('.pub'):
+            self.metadata['perms'] = '0644'
+        else:
+            self.metadata['perms'] = '0600'
+            self.metadata['sensitive'] = 'true'
+
+
+class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet):
+    def __init__(self, path):
+        Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path,
+                                              KeyData, None)
+        self.metadata = {'owner': 'root',
+                         'group': 'root',
+                         'type': 'file',
+                         'perms': '0644'}
 
 
 class SSHbase(Bcfg2.Server.Plugin.Plugin,
               Bcfg2.Server.Plugin.Generator,
-              Bcfg2.Server.Plugin.DirectoryBacked,
               Bcfg2.Server.Plugin.PullTarget):
     """
        The sshbase generator manages ssh host keys (both v1 and v2)
@@ -38,13 +104,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     __version__ = '$Id$'
     __author__ = 'bcfg-dev@mcs.anl.gov'
 
-    pubkeys = ["ssh_host_dsa_key.pub.H_%s",
-               "ssh_host_ecdsa_key.pub.H_%s",
-               "ssh_host_rsa_key.pub.H_%s",
-               "ssh_host_key.pub.H_%s"]
-    hostkeys = ["ssh_host_dsa_key.H_%s",
-                "ssh_host_rsa_key.H_%s",
-                "ssh_host_key.H_%s"]
     keypatterns = ["ssh_host_dsa_key",
                    "ssh_host_ecdsa_key",
                    "ssh_host_rsa_key",
@@ -58,41 +117,39 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Generator.__init__(self)
         Bcfg2.Server.Plugin.PullTarget.__init__(self)
-        try:
-            Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data,
-                                                         self.core.fam)
-        except OSError:
-            ioerr = sys.exc_info()[1]
-            self.logger.error("Failed to load SSHbase repository from %s" \
-                              % (self.data))
-            self.logger.error(ioerr)
-            raise Bcfg2.Server.Plugin.PluginInitError
-        self.Entries = {'Path':
-                        {'/etc/ssh/ssh_known_hosts': self.build_skn,
-                         '/etc/ssh/ssh_host_dsa_key': self.build_hk,
-                         '/etc/ssh/ssh_host_ecdsa_key': self.build_hk,
-                         '/etc/ssh/ssh_host_rsa_key': self.build_hk,
-                         '/etc/ssh/ssh_host_dsa_key.pub': self.build_hk,
-                         '/etc/ssh/ssh_host_ecdsa_key.pub': self.build_hk,
-                         '/etc/ssh/ssh_host_rsa_key.pub': self.build_hk,
-                         '/etc/ssh/ssh_host_key': self.build_hk,
-                         '/etc/ssh/ssh_host_key.pub': self.build_hk}}
         self.ipcache = {}
         self.namecache = {}
         self.__skn = False
 
+        core.fam.AddMonitor(self.data, self)
+
+        self.static = dict()
+        self.entries = dict()
+        self.Entries['Path'] = dict()
+
+        self.entries['/etc/ssh/ssh_known_hosts'] = KnownHostsEntrySet(self.data)
+        self.Entries['Path']['/etc/ssh/ssh_known_hosts'] = self.build_skn
+        for keypattern in self.keypatterns:
+            self.entries["/etc/ssh/" + keypattern] = HostKeyEntrySet(keypattern,
+                                                                     self.data)
+            self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk
+
     def get_skn(self):
         """Build memory cache of the ssh known hosts file."""
         if not self.__skn:
-            self.__skn = "\n".join([value.data.decode() for key, value in \
-                                    list(self.entries.items()) if \
-                                    key.endswith('.static')])
-            names = dict()
             # if no metadata is registered yet, defer
             if len(self.core.metadata.query.all()) == 0:
                 self.__skn = False
                 return self.__skn
-            for cmeta in self.core.metadata.query.all():
+
+            skn = [s.data.decode().rstrip()
+                   for s in list(self.static.values())]
+
+            mquery = self.core.metadata.query
+            
+            # build hostname cache
+            names = dict()
+            for cmeta in mquery.all():
                 names[cmeta.hostname] = set([cmeta.hostname])
                 names[cmeta.hostname].update(cmeta.aliases)
                 newnames = set()
@@ -114,20 +171,33 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
                         except:
                             continue
                 names[cmeta.hostname] = sorted(names[cmeta.hostname])
-            # now we have our name cache
-            pubkeys = [pubk for pubk in list(self.entries.keys()) \
-                       if pubk.find('.pub.H_') != -1]
+
+            pubkeys = [pubk for pubk in list(self.entries.keys())
+                       if pubk.endswith('.pub')]
             pubkeys.sort()
-            badnames = set()
             for pubkey in pubkeys:
-                hostname = pubkey.split('H_')[1]
-                if hostname not in names:
-                    if hostname not in badnames:
-                        badnames.add(hostname)
-                        self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname)
-                    continue
-                self.__skn += "%s %s" % (','.join(names[hostname]),
-                                         self.entries[pubkey].data.decode())
+                for entry in self.entries[pubkey].entries.values():
+                    specific = entry.specific
+                    if specific.hostname and specific.hostname in names:
+                        hostnames = names[specific.hostname]
+                    elif specific.group:
+                        hostnames = \
+                            reduce(lambda x, y: x + y,
+                                   [names[cmeta.hostname]
+                                    for cmeta in \
+                                    mquery.by_groups([specific.group])], [])
+                    elif specific.all:
+                        # a generic key for all hosts?  really?
+                        hostnames = reduce(lambda x, y: x + y,
+                                           list(names.values()), [])
+                    if not hostnames:
+                        self.logger.info("Unknown key %s, skipping" %
+                                         entry.name)
+                    
+                    skn.append("%s %s" % (','.join(hostnames),
+                                          entry.data.decode().rstrip()))
+
+            self.__skn = "\n".join(skn)
         return self.__skn
 
     def set_skn(self, value):
@@ -137,28 +207,37 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
 
     def HandleEvent(self, event=None):
         """Local event handler that does skn regen on pubkey change."""
-        Bcfg2.Server.Plugin.DirectoryBacked.HandleEvent(self, event)
-        if event and '_key.pub.H_' in event.filename:
-            self.skn = False
-        if event and event.filename.endswith('.static'):
-            self.skn = False
-        if not self.__skn:
-            if (len(list(self.entries.keys()))) >= (len(os.listdir(self.data)) - 1):
-                _ = self.skn
-
-    def HandlesEntry(self, entry, _):
-        """Handle key entries dynamically."""
-        return entry.tag == 'Path' and \
-               ([fpat for fpat in self.keypatterns
-                 if entry.get('name').endswith(fpat)]
-                or entry.get('name').endswith('ssh_known_hosts'))
-
-    def HandleEntry(self, entry, metadata):
-        """Bind data."""
-        if entry.get('name').endswith('ssh_known_hosts'):
-            return self.build_skn(entry, metadata)
-        else:
-            return self.build_hk(entry, metadata)
+        # skip events we don't care about
+        action = event.code2str()
+        if action == "endExist" or event.filename == self.data:
+            return
+        
+        for entry in list(self.entries.values()):
+            if entry.specific.match(event.filename):
+                entry.handle_event(event)
+                if event.filename.endswith(".pub"):
+                    self.skn = False
+                return
+
+        if event.filename in ['info', 'info.xml', ':info']:
+            for entry in list(self.entries.values()):
+                entry.handle_event(event)
+            return
+
+        if event.filename.endswith('.static'):
+            if action == "deleted" and event.filename in self.static:
+                del self.static[event.filename]
+                self.skn = False
+            else:
+                self.static[event.filename] = \
+                    Bcfg2.Server.Plugin.FileBacked(os.path.join(self.data,
+                                                                event.filename))
+                self.static[event.filename].HandleEvent(event)
+                self.skn = False
+            return
+
+        self.logger.warn("SSHbase: Got unknown event %s %s" %
+                         (event.filename, action))
 
     def get_ipcache_entry(self, client):
         """Build a cache of dns results."""
@@ -208,72 +287,64 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
 
     def build_skn(self, entry, metadata):
         """This function builds builds a host specific known_hosts file."""
-        client = metadata.hostname
-        entry.text = self.skn
-        hostkeys = [keytmpl % client for keytmpl in self.pubkeys \
-                        if (keytmpl % client) in self.entries]
-        hostkeys.sort()
-        for hostkey in hostkeys:
-            entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
-                self.entries[hostkey].data.decode())
-        permdata = {'owner': 'root',
-                    'group': 'root',
-                    'type': 'file',
-                    'perms': '0644'}
-        [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
+        try:
+            rv = self.entries[entry.get('name')].bind_entry(entry, metadata)
+        except Bcfg2.Server.Plugin.PluginExecutionError:
+            client = metadata.hostname
+            entry.text = self.skn
+            hostkeys = []
+            for k in self.keypatterns:
+                if k.endswith(".pub"):
+                    try:
+                        hostkeys.append(self.entries["/etc/ssh/" +
+                                                     k].best_matching(metadata))
+                    except Bcfg2.Server.Plugin.PluginExecutionError:
+                        pass
+            hostkeys.sort()
+            for hostkey in hostkeys:
+                entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
+                    hostkey.data.decode().rstrip())
+            self.entries[entry.get('name')].bind_info_to_entry(entry, metadata)
 
     def build_hk(self, entry, metadata):
         """This binds host key data into entries."""
-        client = metadata.hostname
-        filename = "%s.H_%s" % (entry.get('name').split('/')[-1], client)
-        if filename not in list(self.entries.keys()):
+        rv = None
+        try:
+            rv = self.entries[entry.get('name')].bind_entry(entry, metadata)
+        except Bcfg2.Server.Plugin.PluginExecutionError:
+            filename = entry.get('name').split('/')[-1]
             self.GenerateHostKeyPair(client, filename)
-        # Service the FAM events queued up by the key generation so
-        # the data structure entries will be available for binding.
-        #
-        # NOTE: We wait for up to ten seconds. There is some potential
-        # for race condition, because if the file monitor doesn't get
-        # notified about the new key files in time, those entries
-        # won't be available for binding. In practice, this seems
-        # "good enough".
-        tries = 0
-        while not filename in self.entries:
-            if tries >= 10:
-                self.logger.error("%s still not registered" % filename)
-                raise Bcfg2.Server.Plugin.PluginExecutionError
-            self.fam.handle_events_in_interval(1)
-            tries += 1
-        keydata = self.entries[filename].data
-        permdata = {'owner': 'root',
-                    'group': 'root',
-                    'type': 'file'}
-        if entry.get('name')[-4:] == '.pub':
-            permdata['perms'] = '0644'
-        else:
-            permdata['perms'] = '0600'
-            permdata['sensitive'] = 'true'
-        [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
-        if "ssh_host_key.H_" == filename[:15]:
-            entry.attrib['encoding'] = 'base64'
-            entry.text = binascii.b2a_base64(keydata)
-        else:
-            entry.text = keydata
+            # Service the FAM events queued up by the key generation
+            # so the data structure entries will be available for
+            # binding.
+            #
+            # NOTE: We wait for up to ten seconds. There is some
+            # potential for race condition, because if the file
+            # monitor doesn't get notified about the new key files in
+            # time, those entries won't be available for binding. In
+            # practice, this seems "good enough".
+            tries = 0
+            while rv is None:
+                if tries >= 10:
+                    self.logger.error("%s still not registered" % filename)
+                    raise Bcfg2.Server.Plugin.PluginExecutionError
+                self.fam.handle_events_in_interval(1)
+                tries += 1
+                try:
+                    rv = self.entries[entry.get('name')].bind_entry()
+                except Bcfg2.Server.Plugin.PluginExecutionError:
+                    pass
+        return rv
 
     def GenerateHostKeyPair(self, client, filename):
         """Generate new host key pair for client."""
-        filename = filename.split('.')[0] # no trailing ".pub", please
-        if filename == 'ssh_host_rsa_key':
-            hostkey = 'ssh_host_rsa_key.H_%s' % client
-            keytype = 'rsa'
-        elif filename == 'ssh_host_dsa_key':
-            hostkey = 'ssh_host_dsa_key.H_%s' % client
-            keytype = 'dsa'
-        elif filename == 'ssh_host_ecdsa_key':
-            hostkey = 'ssh_host_ecdsa_key.H_%s' % client
-            keytype = 'ecdsa'
-        elif filename == 'ssh_host_key':
-            hostkey = 'ssh_host_key.H_%s' % client
-            keytype = 'rsa1'
+        match = re.search(r'(ssh_host_(?:((?:ec)?d|rsa)_)?key)', filename)
+        if match:
+            hostkey = "%s.H_%s" % (match.group(1), client)
+            if match.group(2):
+                keytype = match.group(2)
+            else:
+                keytype = 'rsa1'
         else:
             return