added support for validating Cfg file contents using external commands

7 files changed, 241 insertions, 102 deletions

diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt
index 1c8f6f11a..ba02b929d 100644
--- a/doc/server/plugins/generators/cfg.txt
+++ b/doc/server/plugins/generators/cfg.txt
@@ -93,9 +93,65 @@ classes.
     one. That way if a hostname changes or an extra copy of a particular
     client is built, it will get the same changes as the original.
 
+Templates
+=========
+
+Genshi Templates
+----------------
+
+Genshi templates maybe used for entries as well.  Any file ending in .genshi
+will be processed using the new template style (like .newtxt in the TGenshi
+plugin).
+
+Cheetah Templates
+-----------------
+
+Cheetah templates maybe used for entries as well.  Simply name your file
+with a .cheetah extenstion and it will be processed like the TCheetah
+plugin.
+
+Notes on Using Templates
+------------------------
+
+Templates can be host and group specific as well.  Deltas will not be
+processed for any genshi or cheetah base file.
+
+.. note::
+
+    If you are using templating in combination with host-specific
+    or group-specific files, you will need to ensure that the ``.genshi``
+    or ``.cheetah`` extension is at the **end** of the filename. Using the
+    examples from above for *host.example.com* and group *server* you would
+    have the following (using genshi only)::
+
+        Cfg/etc/fstab/fstab.H_host.example.com.genshi
+        Cfg/etc/fstab/fstab.G50_server.genshi
+
+Genshi templates take precence over cheetah templates.  For example, if
+two files exist named
+
+    Cfg/etc/fstab/fstab.genshi
+    Cfg/etc/fstab/fstab.cheetah
+
+the cheetah template is ignored.  But you can mix genshi and cheetah when
+using different host-specific or group-specific files.  For example:
+
+    Cfg/etc/fstab/fstab.H_host.example.com.genshi
+    Cfg/etc/fstab/fstab.G50_server.cheetah
+
 Deltas
 ======
 
+.. note::
+
+    In Bcfg2 1.3 and newer, deltas are deprecated.  It is recommended
+    that you use templates instead.  The
+    :ref:`TemplateHelper plugin
+    <server-plugins-connectors-templatehelper>` comes with an example
+    helper that can be used to include other files easily, a subset of
+    cat file functionality.  ``bcfg2-lint`` checks for deltas and
+    warns about them.
+
 Bcfg2 has finer grained control over how to deliver configuration
 files to a host. Let's say we have a Group named file-server. Members
 of this group need the exact same ``/etc/motd`` as all other hosts except
@@ -150,51 +206,44 @@ file. The reason the other deltas aren't applied to *foo.example.com*
 is because a **.H_** delta is more specific than a **.G##_** delta. Bcfg2
 applies all the deltas at the most specific level.
 
-Templates
-=========
-
-Genshi Templates
-----------------
-
-Genshi templates maybe used for entries as well.  Any file ending in .genshi
-will be processed using the new template style (like .newtxt in the TGenshi
-plugin).
+Content Validation
+==================
 
-Cheetah Templates
------------------
+To ensure that files with invalid content are not pushed out, you can
+provide a content validation script that will be run against each
+file.  Create a file called ``:test`` inside the directory for the
+file you want to test.  For example::
 
-Cheetah templates maybe used for entries as well.  Simply name your file
-with a .cheetah extenstion and it will be processed like the TCheetah
-plugin.
+  Cfg/etc/sudoers/:test
 
-Notes on Using Templates
-------------------------
+You can also create host- and group-specific validators::
 
-Templates can be host and group specific as well.  Deltas will not be
-processed for any genshi or cheetah base file.
+  Cfg/etc/sudoers/:test.G80_foogroup
+  Cfg/etc/sudoers/:test.H_bar.example.com
 
-.. note::
+A validator script has the following attributes:
 
-    If you are using templating in combination with host-specific
-    or group-specific files, you will need to ensure that the ``.genshi``
-    or ``.cheetah`` extension is at the **end** of the filename. Using the
-    examples from above for *host.example.com* and group *server* you would
-    have the following (using genshi only)::
+* It must be executable, or specify a valid bangpath;
+* The entire content of the file is passed to the validator on
+  stdin;
+* The validator is not called with any flags or arguments;
+* The validator must return 0 on success and non-zero on failure; and
+* The validator must output a sensible error message on failure.
 
-        Cfg/etc/fstab/fstab.H_host.example.com.genshi
-        Cfg/etc/fstab/fstab.G50_server.genshi
+For ``sudoers``, a very simple validator is::
 
-Genshi templates take precence over cheetah templates.  For example, if
-two files exist named
+  #!/bin/sh
+  visudo -cf -
 
-    Cfg/etc/fstab/fstab.genshi
-    Cfg/etc/fstab/fstab.cheetah
+This uses the ``visudo`` command's built-in validation.
 
-the cheetah template is ignored.  But you can mix genshi and cheetah when
-using different host-specific or group-specific files.  For example:
+.. note:
 
-    Cfg/etc/fstab/fstab.H_host.example.com.genshi
-    Cfg/etc/fstab/fstab.G50_server.cheetah
+    Before 1.3 is released, it will be possible to disable validation
+    in the configuration, but enable it for ``bcfg2-test``.  This is
+    recommended for heavily-used servers, since running an external
+    command is fairly resource intensive and could quickly exhaust the
+    file descriptors of a server.
 
 File permissions
 ================
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py
new file mode 100644
index 000000000..f0c1109ec
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py
@@ -0,0 +1,33 @@
+import os
+import shlex
+import logging
+import Bcfg2.Server.Plugin
+from subprocess import Popen, PIPE
+from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError
+
+logger = logging.getLogger(__name__)
+
+class CfgExternalCommandVerifier(CfgVerifier):
+    __basenames__ = [':test']
+
+    def verify_entry(self, entry, metadata, data):
+        proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        err = proc.communicate(input=data)[1]
+        rv = proc.wait()
+        if rv != 0:
+            raise CfgVerificationError(err)
+
+    def handle_event(self, event):
+        if event.code2str() == 'deleted':
+            return
+        self.cmd = []
+        if not os.access(self.name, os.X_OK):
+            bangpath = open(self.name).readline().strip()
+            if bangpath.startswith("#!"):
+                self.cmd.extend(shlex.split(bangpath[2:].strip()))
+            else:
+                msg = "Cannot execute %s" % self.name
+                logger.error(msg)
+                raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+        self.cmd.append(self.name)
+    
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
index 6c4e6ad51..2c0a076d7 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
@@ -33,9 +33,9 @@ class CfgGenshiGenerator(CfgGenerator):
             raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
 
     @classmethod
-    def ignore(cls, basename, event):
+    def ignore(cls, event, basename=None):
         return (event.filename.endswith(".genshi_include") or
-                CfgGenerator.ignore(basename, event))
+                CfgGenerator.ignore(event, basename=basename))
 
     def get_data(self, entry, metadata):
         fname = entry.get('realname', entry.get('name'))
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py
index 35aaa0442..8e962efb4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py
@@ -2,10 +2,10 @@ import logging
 import Bcfg2.Server.Plugin
 from Bcfg2.Server.Plugins.Cfg import CfgInfo
 
-logger = logging.getLogger('Bcfg2.Plugins.Cfg')
+logger = logging.getLogger(__name__)
 
 class CfgInfoXML(CfgInfo):
-    names = ['info.xml']
+    __basenames__ = ['info.xml']
 
     def __init__(self, path):
         CfgInfo.__init__(self, path)
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py
index 9616f8bba..54c17c6c5 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py
@@ -2,10 +2,10 @@ import logging
 import Bcfg2.Server.Plugin
 from Bcfg2.Server.Plugins.Cfg import CfgInfo
 
-logger = logging.getLogger('Bcfg2.Plugins.Cfg')
+logger = logging.getLogger(__name__)
 
 class CfgLegacyInfo(CfgInfo):
-    names = ['info', ':info']
+    __basenames__ = ['info', ':info']
 
     def bind_info_to_entry(self, entry, metadata):
         self._set_info(entry, self.metadata)
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py
index 3351209f3..8e9aab465 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py
@@ -2,7 +2,7 @@ import logging
 import Bcfg2.Server.Plugin
 from Bcfg2.Server.Plugins.Cfg import CfgGenerator
 
-logger = logging.getLogger('Bcfg2.Plugins.Cfg')
+logger = logging.getLogger(__name__)
 
 class CfgPlaintextGenerator(CfgGenerator):
     pass
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
index 9bac50e44..f59890574 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
@@ -11,13 +11,15 @@ import lxml.etree
 import Bcfg2.Server.Plugin
 from Bcfg2.Bcfg2Py3k import u_str
 
-logger = logging.getLogger('Bcfg2.Plugins.Cfg')
+logger = logging.getLogger(__name__)
 
 PROCESSORS = None
 
 class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData):
+    __basenames__ = []
     __extensions__ = []
     __ignore__ = []
+    __specific__ = True
 
     def __init__(self, fname, spec, encoding):
         Bcfg2.Server.Plugin.SpecificData.__init__(self, fname, spec, encoding)
@@ -25,26 +27,54 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData):
         self.regex = self.__class__.get_regex(fname)
 
     @classmethod
-    def get_regex(cls, fname, extensions=None):
+    def get_regex(cls, fname=None, extensions=None):
         if extensions is None:
             extensions = cls.__extensions__
+        if cls.__basenames__:
+            fname = '|'.join(cls.__basenames__)
 
-        base_re = '^(?P<basename>%s)(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))' % re.escape(fname)
+        components = ['^(?P<basename>%s)' % fname]
+        if cls.__specific__:
+            components.append('(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))')
         if extensions:
-            base_re += '\\.(?P<extension>%s)' % '|'.join(extensions)
-        base_re += '$'
-        return re.compile(base_re)
+            components.append('\\.(?P<extension>%s)' % '|'.join(extensions))
+        components.append('$')
+        return re.compile("".join(components))
     
     @classmethod
-    def handles(cls, basename, event):
-        return (event.filename.startswith(os.path.basename(basename)) and
-                cls.get_regex(os.path.basename(basename)).match(event.filename))
+    def handles(cls, event, basename=None):
+        if cls.__basenames__:
+            basenames = cls.__basenames__
+        else:
+            basenames = [basename]
+
+        # do simple non-regex matching first
+        match = False
+        for bname in basenames:
+            if event.filename.startswith(os.path.basename(bname)):
+                match = True
+                break
+        return (match and
+                cls.get_regex(fname=os.path.basename(basename)).match(event.filename))
 
     @classmethod
-    def ignore(cls, basename, event):
-        return (cls.__ignore__ and
-                event.filename.startswith(os.path.basename(basename)) and
-                cls.get_regex(os.path.basename(basename),
+    def ignore(cls, event, basename=None):
+        if not cls.__ignore__:
+            return False
+
+        if cls.__basenames__:
+            basenames = cls.__basenames__
+        else:
+            basenames = [basename]
+
+        # do simple non-regex matching first
+        match = False
+        for bname in basenames:
+            if event.filename.startswith(os.path.basename(bname)):
+                match = True
+                break
+        return (match and
+                cls.get_regex(fname=os.path.basename(basename),
                               extensions=cls.__ignore__).match(event.filename))
         
 
@@ -56,30 +86,25 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData):
 
 
 class CfgGenerator(CfgBaseFileMatcher):
+    """ CfgGenerators generate the initial content of a file """
     def get_data(self, entry, metadata):
         return self.data
 
 
 class CfgFilter(CfgBaseFileMatcher):
+    """ CfgFilters modify the initial content of a file after it's
+    been generated """
     def modify_data(self, entry, metadata, data):
         raise NotImplementedError
 
 
-class CfgInfo(Bcfg2.Server.Plugin.SpecificData):
-    names = []
-    regex = re.compile('^$')
-
-    def __init__(self, path):
-        self.path = path
-        self.name = os.path.basename(path)
-
-    @classmethod
-    def handles(cls, basename, event):
-        return event.filename in cls.names or cls.regex.match(event.filename)
+class CfgInfo(CfgBaseFileMatcher):
+    """ CfgInfos provide metadata (owner, group, paranoid, etc.) for a
+    file entry """
+    __specific__ = False
 
-    @classmethod
-    def ignore(cls, basename, event):
-        return False
+    def __init__(self, fname):
+        CfgBaseFileMatcher.__init__(self, fname, None, None)
 
     def bind_info_to_entry(self, entry, metadata):
         raise NotImplementedError
@@ -87,29 +112,33 @@ class CfgInfo(Bcfg2.Server.Plugin.SpecificData):
     def _set_info(self, entry, info):
         for key, value in list(info.items()):
             entry.attrib.__setitem__(key, value)
-    
-    def __str__(self):
-        return "%s(%s)" % (self.__class__.__name__, self.name)
+
+
+class CfgVerifier(CfgBaseFileMatcher):
+    """ Verifiers validate entries """
+    def verify_entry(self, entry, metadata, data):
+        raise NotImplementedError
+
+
+class CfgVerificationError(Exception):
+    pass
 
 
 class CfgDefaultInfo(CfgInfo):
     def __init__(self, defaults):
-        self.name = ''
+        CfgInfo.__init__(self, '')
         self.defaults = defaults
 
-    def handles(self, event):
-        return False
-
     def bind_info_to_entry(self, entry, metadata):
         self._set_info(entry, self.defaults)
 
+DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.default_file_metadata)
 
 class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
     def __init__(self, basename, path, entry_type, encoding):
         Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path,
                                               entry_type, encoding)
         self.specific = None
-        self.default_info = CfgDefaultInfo(self.metadata)
         self.load_processors()
 
     def load_processors(self):
@@ -128,38 +157,41 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
                                  submodule[1])
                 proc = getattr(module, submodule[1])
                 if set(proc.__mro__).intersection([CfgInfo, CfgFilter,
-                                                   CfgGenerator]):
+                                                   CfgGenerator, CfgVerifier]):
                     PROCESSORS.append(proc)
 
     def handle_event(self, event):
         action = event.code2str()
         
-        for proc in PROCESSORS:
-            if proc.handles(self.path, event):
-                self.debug_log("%s handling %s event on %s" %
-                               (proc.__name__, action, event.filename))
-                if action in ['exists', 'created']:
-                    self.entry_init(event, proc)
-                elif event.filename not in self.entries:
-                    logger.warning("Got %s event for unknown file %s" %
-                                   (action, event.filename))
-                    if action == 'changed':
-                        # received a bogus changed event; warn, but
-                        # treat it like a created event
-                        self.entry_init(event, proc)
-                elif action == 'changed':
-                    self.entries[event.filename].handle_event(event)
-                elif action == 'deleted':
-                    del self.entries[event.filename]
-                return
-            elif proc.ignore(self.path, event):
+        if event.filename not in self.entries:
+            if action not in ['exists', 'created', 'changed']:
+                # process a bogus changed event like a created
                 return
+                
+            for proc in PROCESSORS:
+                if proc.handles(event, basename=self.path):
+                    if action == 'changed':
+                        # warn about a bogus 'changed' event, but
+                        # handle it like a 'created'
+                        logger.warning("Got %s event for unknown file %s" %
+                                       (action, event.filename))
+                    self.debug_log("%s handling %s event on %s" %
+                                   (proc.__name__, action, event.filename))
+                    self.entry_init(event, proc)
+                    return
+                elif proc.ignore(event, basename=self.path):
+                    return
+        elif action == 'changed':
+            self.entries[event.filename].handle_event(event)
+        elif action == 'deleted':
+            del self.entries[event.filename]
+            return
 
-        logger.error("Could not process filename %s; ignoring" %
-                     event.filename)
+        logger.error("Could not process event %s for %s; ignoring" %
+                     (action, event.filename))
 
     def entry_init(self, event, proc):
-        if CfgBaseFileMatcher in proc.__mro__:
+        if proc.__specific__:
             Bcfg2.Server.Plugin.EntrySet.entry_init(
                 self, event, entry_type=proc,
                 specific=proc.get_regex(os.path.basename(self.path)))
@@ -175,9 +207,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         info_handlers = []
         generators = []
         filters = []
+        verifiers = []
         for ent in self.entries.values():
-            if (hasattr(ent, 'specific') and
-                not ent.specific.matches(metadata)):
+            if ent.__specific__ and not ent.specific.matches(metadata):
                 continue
             if isinstance(ent, CfgInfo):
                 info_handlers.append(ent)
@@ -185,8 +217,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
                 generators.append(ent)
             elif isinstance(ent, CfgFilter):
                 filters.append(ent)
+            elif isinstance(ent, CfgVerifier):
+                verifiers.append(ent)
 
-        self.default_info.bind_info_to_entry(entry, metadata)
+        DEFAULT_INFO.bind_info_to_entry(entry, metadata)
         if len(info_handlers) > 1:
             logger.error("More than one info supplier found for %s: %s" %
                          (self.name, info_handlers))
@@ -212,6 +246,29 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         for fltr in filters:
             data = fltr.modify_data(entry, metadata, data)
 
+        # TODO: disable runtime verification in config, but let
+        # bcfg2-test turn it back on dynamically.  need to sort out
+        # config files first.
+
+        # we can have multiple verifiers, but we only want to use the
+        # best matching verifier of each class
+        verifiers_by_class = dict()
+        for verifier in verifiers:
+            cls = verifier.__class__.__name__
+            if cls not in verifiers_by_class:
+                verifiers_by_class[cls] = [verifier]
+            else:
+                verifiers_by_class[cls].append(verifier)
+        for verifiers in verifiers_by_class.values():
+            verifier = self.best_matching(metadata, verifiers)
+            try:
+                verifier.verify_entry(entry, metadata, data)
+            except CfgVerificationError:
+                msg = "Data for %s for %s failed to verify: %s" % \
+                    (entry.get('name'), metadata.hostname, sys.exc_info()[1])
+                logger.error(msg)
+                raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+                
         if entry.get('encoding') == 'base64':
             data = binascii.b2a_base64(data)
         else: