Handle FAM monitor failures more gracefully:

* Where possible, create the file or directory that is about to be
  monitored.  This ensures that content can be added later without
  need to restart Bcfg2.  (Otherwise, adding the monitor would fail,
  and so when you did create the file in question, bcfg2-server would
  never be notified of it.)
* When not possible, give better error messages.

13 files changed, 91 insertions, 101 deletions

diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 382f11e50..8ceb8cfc1 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -420,6 +420,10 @@ class BaseCore(object):
         except PluginInitError:
             self.logger.error("Failed to instantiate plugin %s" % plugin,
                               exc_info=1)
+        except OSError:
+            err = sys.exc_info()[1]
+            self.logger.error("Failed to add a file monitor while "
+                              "instantiating plugin %s: %s" % (plugin, err))
         except:
             self.logger.error("Unexpected instantiation failure for plugin %s"
                               % plugin, exc_info=1)
diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py
index f7bc08717..ecd970b54 100644
--- a/src/lib/Bcfg2/Server/Plugin/base.py
+++ b/src/lib/Bcfg2/Server/Plugin/base.py
@@ -97,15 +97,21 @@ class Plugin(Debuggable):
         :param datastore: The path to the Bcfg2 repository on the
                           filesystem
         :type datastore: string
-        :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError`
+        :raises: :exc:`OSError` if adding a file monitor failed;
+                 :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError`
+                 on other errors
 
         .. autoattribute:: Bcfg2.Server.Plugin.base.Debuggable.__rmi__
         """
+        Debuggable.__init__(self, name=self.name)
         self.Entries = {}
         self.core = core
         self.data = os.path.join(datastore, self.name)
+        if not os.path.exists(self.data):
+            self.logger.warning("%s: %s does not exist, creating" %
+                                (self.name, self.data))
+            os.makedirs(self.data)
         self.running = True
-        Debuggable.__init__(self, name=self.name)
 
     @classmethod
     def init_repo(cls, repo):
@@ -125,5 +131,11 @@ class Plugin(Debuggable):
         self.debug_log("Shutting down %s plugin" % self.name)
         self.running = False
 
+    def set_debug(self, debug):
+        for entry in self.Entries.values():
+            if isinstance(entry, Debuggable):
+                entry.set_debug(debug)
+        return Debuggable.set_debug(self, debug)
+
     def __str__(self):
         return "%s Plugin" % self.__class__.__name__
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index 0b81077a3..448d8b3b6 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -193,7 +193,7 @@ class PluginDatabaseModel(object):
         app_label = "Server"
 
 
-class FileBacked(object):
+class FileBacked(Debuggable):
     """ This object caches file data in memory. FileBacked objects are
     principally meant to be used as a part of
     :class:`Bcfg2.Server.Plugin.helpers.DirectoryBacked`. """
@@ -206,7 +206,7 @@ class FileBacked(object):
                     changes
         :type fam: Bcfg2.Server.FileMonitor.FileMonitor
         """
-        object.__init__(self)
+        Debuggable.__init__(self)
 
         #: A string containing the raw data in this file
         self.data = ''
@@ -246,7 +246,7 @@ class FileBacked(object):
         return "%s: %s" % (self.__class__.__name__, self.name)
 
 
-class DirectoryBacked(object):
+class DirectoryBacked(Debuggable):
     """ DirectoryBacked objects represent a directory that contains
     files, represented by objects of the type listed in
     :attr:`__child__`, and other directories recursively.  It monitors
@@ -280,7 +280,7 @@ class DirectoryBacked(object):
         .. -----
         .. autoattribute:: __child__
         """
-        object.__init__(self)
+        Debuggable.__init__(self)
 
         self.data = os.path.normpath(data)
         self.fam = fam
@@ -299,8 +299,17 @@ class DirectoryBacked(object):
         self.handles = {}
 
         # Monitor everything in the plugin's directory
+        if not os.path.exists(self.data):
+            self.logger.warning("%s does not exist, creating" % self.data)
+            os.makedirs(self.data)
         self.add_directory_monitor('')
 
+    def set_debug(self, debug):
+        for entry in self.entries.values():
+            if isinstance(entry, Debuggable):
+                entry.set_debug(debug)
+        return Debuggable.set_debug(self, debug)
+
     def __getitem__(self, key):
         return self.entries[key]
 
@@ -459,7 +468,11 @@ class XMLFileBacked(FileBacked):
     #: behavior, set ``__identifier__`` to ``None``.
     __identifier__ = 'name'
 
-    def __init__(self, filename, fam=None, should_monitor=False):
+    #: If ``create`` is set, then it overrides the ``create`` argument
+    #: to the constructor.
+    create = None
+
+    def __init__(self, filename, fam=None, should_monitor=False, create=None):
         """
         :param filename: The full path to the file to cache and monitor
         :type filename: string
@@ -474,6 +487,13 @@ class XMLFileBacked(FileBacked):
                                :class:`Bcfg2.Server.Plugin.helpers.XMLDirectoryBacked`
                                object).
         :type should_monitor: bool
+        :param create: Create the file if it doesn't exist.
+                       ``create`` can be either an
+                       :class:`lxml.etree._Element` object, which will
+                       be used as initial content, or a string, which
+                       will be used as the name of the (empty) tag
+                       that will be the initial content of the file.
+        :type create: lxml.etree._Element or string
 
         .. -----
         .. autoattribute:: __identifier__
@@ -497,6 +517,17 @@ class XMLFileBacked(FileBacked):
         #: "Extra" files included in this file by XInclude.
         self.extras = []
 
+        if ((create or (self.create is not None and self.create))
+            and not os.path.exists(self.name)):
+            toptag = create or self.create
+            self.logger.warning("%s does not exist, creating" % self.name)
+            if hasattr(toptag, "getroottree"):
+                el = toptag
+            else:
+                el = lxml.etree.Element(toptag)
+            el.getroottree().write(self.name, xml_declaration=False,
+                                   pretty_print=True)
+
         #: Whether or not to monitor this file for changes.
         self.should_monitor = should_monitor
         if fam and should_monitor:
@@ -776,8 +807,8 @@ class XMLSrc(XMLFileBacked):
     __cacheobj__ = dict
     __priority_required__ = True
 
-    def __init__(self, filename, fam=None, should_monitor=False):
-        XMLFileBacked.__init__(self, filename, fam, should_monitor)
+    def __init__(self, filename, fam=None, should_monitor=False, create=None):
+        XMLFileBacked.__init__(self, filename, fam, should_monitor, create)
         self.items = {}
         self.cache = None
         self.pnode = None
@@ -1450,8 +1481,6 @@ class GroupSpool(Plugin, Generator):
     def __init__(self, core, datastore):
         Plugin.__init__(self, core, datastore)
         Generator.__init__(self)
-        if self.data[-1] == '/':
-            self.data = self.data[:-1]
 
         #: See :class:`Bcfg2.Server.Plugins.interfaces.Generator` for
         #: details on the Entries attribute.
diff --git a/src/lib/Bcfg2/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py
index d662da60a..a18204d60 100644
--- a/src/lib/Bcfg2/Server/Plugins/Base.py
+++ b/src/lib/Bcfg2/Server/Plugins/Base.py
@@ -20,13 +20,8 @@ class Base(Bcfg2.Server.Plugin.Plugin,
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Structure.__init__(self)
-        try:
-            Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self,
-                                                            self.data,
-                                                            self.core.fam)
-        except OSError:
-            self.logger.error("Failed to load Base repository")
-            raise Bcfg2.Server.Plugin.PluginInitError
+        Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data,
+                                                        self.core.fam)
 
     def BuildStructures(self, metadata):
         """Build structures for client described by metadata."""
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index 7030c1574..24301155d 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -92,16 +92,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
         Bcfg2.Server.Plugin.Structure.__init__(self)
         self.encoding = core.setup['encoding']
         self.__child__ = self.template_dispatch
-        try:
-            Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self,
-                                                            self.data,
-                                                            self.core.fam)
-        except OSError:
-            err = sys.exc_info()[1]
-            msg = "Failed to load Bundle repository %s: %s" % (self.data, err)
-            self.logger.error(msg)
-            raise Bcfg2.Server.Plugin.PluginInitError(msg)
-
+        Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data,
+                                                        self.core.fam)
         global SETUP
         SETUP = core.setup
 
diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py
index eae18fdfe..66f299bc9 100644
--- a/src/lib/Bcfg2/Server/Plugins/Decisions.py
+++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py
@@ -2,7 +2,6 @@
 blacklist certain entries. """
 
 import os
-import sys
 import lxml.etree
 import Bcfg2.Server.Plugin
 
@@ -40,18 +39,10 @@ class Decisions(Bcfg2.Server.Plugin.EntrySet,
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Decision.__init__(self)
-
         Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list',
-                                              self.data,
-                                              DecisionFile,
+                                              self.data, DecisionFile,
                                               core.setup['encoding'])
-        try:
-            core.fam.AddMonitor(self.data, self)
-        except OSError:
-            err = sys.exc_info()[1]
-            msg = 'Adding filemonitor for %s failed: %s' % (self.data, err)
-            self.logger.error(msg)
-            raise Bcfg2.Server.Plugin.PluginInitError(msg)
+        core.fam.AddMonitor(self.data, self)
 
     def HandleEvent(self, event):
         """ Handle events on Decision files by passing them off to
diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py
index 5ec0d7280..882c22c49 100644
--- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py
+++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py
@@ -67,7 +67,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin,
             Bcfg2.Server.Plugin.StructFile(os.path.join(self.data,
                                                         'config.xml'),
                                            fam=core.fam,
-                                           should_monitor=True)
+                                           should_monitor=True,
+                                           create=self.name)
         self.entries = dict()
         self.probes = dict()
 
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
index 5716a134f..8ce3fcd3a 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -3,7 +3,6 @@
 import os
 import re
 import sys
-import logging
 import Bcfg2.Server.Lint
 import Bcfg2.Server.Plugin
 from Bcfg2.Utils import PackedDigitRange
@@ -67,6 +66,7 @@ class PatternMap(object):
 class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked):
     """ representation of GroupPatterns config.xml """
     __identifier__ = None
+    create = 'GroupPatterns'
 
     def __init__(self, filename, core=None):
         try:
@@ -77,7 +77,6 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked):
                                                    should_monitor=True)
         self.core = core
         self.patterns = []
-        self.logger = logging.getLogger(self.__class__.__name__)
 
     def Index(self):
         Bcfg2.Server.Plugin.XMLFileBacked.Index(self)
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 8fb3a0998..9adc8a442 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -94,9 +94,11 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked):
         # then we immediately set should_monitor to the proper value,
         # so that XInclude'd files get properly watched
         fpath = os.path.join(metadata.data, basefile)
+        toptag = os.path.splitext(basefile)[0].title()
         Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath,
                                                    fam=metadata.core.fam,
-                                                   should_monitor=False)
+                                                   should_monitor=False,
+                                                   create=toptag)
         self.should_monitor = watch_clients
         self.metadata = metadata
         self.basefile = basefile
diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
index c39bd4c42..466665382 100644
--- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
+++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
@@ -5,26 +5,9 @@ import re
 import sys
 import glob
 import socket
-import logging
 import Bcfg2.Server
 import Bcfg2.Server.Plugin
 
-LOGGER = logging.getLogger(__name__)
-
-
-class NagiosGenConfig(Bcfg2.Server.Plugin.StructFile):
-    """ NagiosGen config file handler """
-
-    def __init__(self, filename, fam):
-        # create config.xml if missing
-        if not os.path.exists(filename):
-            LOGGER.warning("NagiosGen: %s missing. "
-                           "Creating empty one for you." % filename)
-            open(filename, "w").write("<NagiosGen></NagiosGen>")
-
-        Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam,
-                                                should_monitor=True)
-
 
 class NagiosGen(Bcfg2.Server.Plugin.Plugin,
                 Bcfg2.Server.Plugin.Generator):
@@ -36,8 +19,11 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin,
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Generator.__init__(self)
-        self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'),
-                                      core.fam)
+        self.config = \
+            Bcfg2.Server.Plugin.StructFile(os.path.join(self.data,
+                                                        'config.xml'),
+                                           core.fam, should_monitor=True,
+                                           create=self.name)
         self.Entries = {'Path':
                         {'/etc/nagiosgen.status': self.createhostconfig,
                          '/etc/nagios/nagiosgen.cfg': self.createserverconfig}}
diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py
index ebc03197e..07b04f3f0 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ohai.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py
@@ -69,10 +69,6 @@ class Ohai(Bcfg2.Server.Plugin.Plugin,
         self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai',
                                         interpreter='/bin/sh')
         self.probe.text = PROBECODE
-        try:
-            os.stat(self.data)
-        except OSError:
-            os.makedirs(self.data)
         self.cache = OhaiCache(self.data)
 
     def GetProbes(self, _):
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
index 739320cb0..ff57d57e8 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
@@ -16,6 +16,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile,
     each ``Source`` tag. """
 
     __identifier__ = None
+    create = "Sources"
 
     def __init__(self, filename, cachepath, fam, packages, setup):
         """
@@ -39,14 +40,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile,
                  If ``sources.xml`` cannot be read
         """
         Bcfg2.Server.Plugin.Debuggable.__init__(self)
-        try:
-            Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam,
-                                                    should_monitor=True)
-        except OSError:
-            err = sys.exc_info()[1]
-            msg = "Packages: Failed to read configuration file: %s" % err
-            self.logger.error(msg)
-            raise Bcfg2.Server.Plugin.PluginInitError(msg)
+        Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam,
+                                                should_monitor=True)
 
         #: The full path to the directory where
         #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data
diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py
index 3ebad40e3..e97f66675 100644
--- a/src/lib/Bcfg2/Server/Plugins/Properties.py
+++ b/src/lib/Bcfg2/Server/Plugins/Properties.py
@@ -266,8 +266,13 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile):
         return repr(self.xdata)
 
 
-class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
-    """ A collection of properties files. """
+class Properties(Bcfg2.Server.Plugin.Plugin,
+                 Bcfg2.Server.Plugin.Connector,
+                 Bcfg2.Server.Plugin.DirectoryBacked):
+    """ The properties plugin maps property files into client metadata
+    instances. """
+
+    #: Extensions that are understood by Properties.
     extensions = ["xml"]
     if HAS_JSON:
         extensions.append("json")
@@ -284,14 +289,18 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
     #: Ignore XML schema (``.xsd``) files
     ignore = re.compile(r'.*\.xsd$')
 
-    def __init__(self, data, fam):
-        Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, data, fam)
+    def __init__(self, core, datastore):
+        global SETUP  # pylint: disable=W0603
+        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+        Bcfg2.Server.Plugin.Connector.__init__(self)
+        Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam)
+        SETUP = core.setup
 
         #: Instead of creating children of this object with a static
         #: object, we use :func:`property_dispatcher` to create a
         #: child of the appropriate subclass of :class:`PropertyFile`
         self.__child__ = self.property_dispatcher
-    __init__.__doc__ = Bcfg2.Server.Plugin.DirectoryBacked.__init__.__doc__
+    __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__
 
     def property_dispatcher(self, fname, fam):
         """ Dispatch an event on a Properties file to the
@@ -314,30 +323,9 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
             raise Bcfg2.Server.Plugin.PluginExecutionError(
                 "Properties: Unknown extension %s" % fname)
 
-
-class Properties(Bcfg2.Server.Plugin.Plugin,
-                 Bcfg2.Server.Plugin.Connector):
-    """ The properties plugin maps property files into client metadata
-    instances. """
-
-    def __init__(self, core, datastore):
-        global SETUP  # pylint: disable=W0603
-        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
-        Bcfg2.Server.Plugin.Connector.__init__(self)
-        SETUP = core.setup
-        try:
-            self.store = PropDirectoryBacked(self.data, core.fam)
-        except OSError:
-            err = sys.exc_info()[1]
-            self.logger.error("Error while creating Properties store: %s" %
-                              err)
-            raise Bcfg2.Server.Plugin.PluginInitError
-
-    __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__
-
     def get_additional_data(self, metadata):
         rv = dict()
-        for fname, pfile in self.store.entries.items():
+        for fname, pfile in self.entries.items():
             rv[fname] = pfile.get_additional_data(metadata)
         return rv
     get_additional_data.__doc__ = \