Merge branch 'master' of https://github.com/mikemccllstr/bcfg2

5 files changed, 230 insertions, 108 deletions

diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index 5adfa5381..91b6a3555 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -90,42 +90,49 @@ class Core(Component):
                              "Unloading %s" % (p, bl))
             for plug in bl:
                 del self.plugins[plug]
-        # This section loads the experimental plugins
+        # This section logs the experimental plugins
         expl = [plug for (name, plug) in list(self.plugins.items())
                 if plug.experimental]
         if expl:
             logger.info("Loading experimental plugin(s): %s" % \
                         (" ".join([x.name for x in expl])))
             logger.info("NOTE: Interfaces subject to change")
+        # This section logs the deprecated plugins
         depr = [plug for (name, plug) in list(self.plugins.items())
                 if plug.deprecated]
-        # This section loads the deprecated plugins
         if depr:
             logger.info("Loading deprecated plugin(s): %s" % \
                         (" ".join([x.name for x in depr])))
 
-        mlist = [p for p in list(self.plugins.values()) if \
-                 isinstance(p, Bcfg2.Server.Plugin.Metadata)]
+        mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata)
         if len(mlist) == 1:
             self.metadata = mlist[0]
         else:
             logger.error("No Metadata Plugin loaded; failed to instantiate Core")
             raise CoreInitError("No Metadata Plugin")
-        self.statistics = [plugin for plugin in list(self.plugins.values())
-                           if isinstance(plugin, Bcfg2.Server.Plugin.Statistics)]
-        self.pull_sources = [plugin for plugin in self.statistics
-                             if isinstance(plugin, Bcfg2.Server.Plugin.PullSource)]
-        self.generators = [plugin for plugin in list(self.plugins.values())
-                           if isinstance(plugin, Bcfg2.Server.Plugin.Generator)]
-        self.structures = [plugin for plugin in list(self.plugins.values())
-                           if isinstance(plugin, Bcfg2.Server.Plugin.Structure)]
-        self.connectors = [plugin for plugin in list(self.plugins.values())
-                           if isinstance(plugin, Bcfg2.Server.Plugin.Connector)]
+        self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics)
+        self.pull_sources = self.plugins_by_type(Bcfg2.Server.Plugin.PullSource)
+        self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator)
+        self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure)
+        self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector)
         self.ca = ca
         self.fam_thread = threading.Thread(target=self._file_monitor_thread)
         if start_fam_thread:
             self.fam_thread.start()
 
+    def plugins_by_type(self, base_cls):
+        """Return a list of loaded plugins that match the passed type.
+
+        The returned list is sorted in ascending order by the Plugins'
+        sort_order value. The sort_order defaults to 500 in Plugin.py,
+        but can be overridden by individual plugins. Plugins with the
+        same numerical sort_order value are sorted in alphabetical
+        order by their name.
+        """
+        return sorted([plugin for plugin in self.plugins.values()
+                       if isinstance(plugin, base_cls)],
+                      key=lambda p: (p.sort_order, p.name))
+
     def _file_monitor_thread(self):
         """The thread for monitor the files."""
         famfd = self.fam.fileno()
@@ -141,9 +148,8 @@ class Core(Component):
             except:
                 continue
             # VCS plugin periodic updates
-            for plugin in list(self.plugins.values()):
-                if isinstance(plugin, Bcfg2.Server.Plugin.Version):
-                    self.revision = plugin.get_revision()
+            for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version):
+                self.revision = plugin.get_revision()
 
     def init_plugins(self, plugin):
         """Handling for the plugins."""
@@ -176,23 +182,33 @@ class Core(Component):
             for plugin in list(self.plugins.values()):
                 plugin.shutdown()
 
-    def validate_data(self, metadata, data, base_cls):
+    def validate_structures(self, metadata, data):
         """Checks the data structure."""
-        for plugin in list(self.plugins.values()):
-            if isinstance(plugin, base_cls):
-                try:
-                    if base_cls == Bcfg2.Server.Plugin.StructureValidator:
-                        plugin.validate_structures(metadata, data)
-                    elif base_cls == Bcfg2.Server.Plugin.GoalValidator:
-                        plugin.validate_goals(metadata, data)
-                except Bcfg2.Server.Plugin.ValidationError:
-                    err = sys.exc_info()[1]
-                    logger.error("Plugin %s structure validation failed: %s" \
-                                 % (plugin.name, err.message))
-                    raise
-                except:
-                    logger.error("Plugin %s: unexpected structure validation failure" \
-                                 % (plugin.name), exc_info=1)
+        for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator):
+            try:
+                plugin.validate_structures(metadata, data)
+            except Bcfg2.Server.Plugin.ValidationError:
+                err = sys.exc_info()[1]
+                logger.error("Plugin %s structure validation failed: %s" \
+                             % (plugin.name, err.message))
+                raise
+            except:
+                logger.error("Plugin %s: unexpected structure validation failure" \
+                             % (plugin.name), exc_info=1)
+
+    def validate_goals(self, metadata, data):
+        """Checks that the config matches the goals enforced by the plugins."""
+        for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator):
+            try:
+                plugin.validate_goals(metadata, data)
+            except Bcfg2.Server.Plugin.ValidationError:
+                err = sys.exc_info()[1]
+                logger.error("Plugin %s goal validation failed: %s" \
+                             % (plugin.name, err.message))
+                raise
+            except:
+                logger.error("Plugin %s: unexpected goal validation failure" \
+                             % (plugin.name), exc_info=1)
 
     def GetStructures(self, metadata):
         """Get all structures for client specified by metadata."""
@@ -276,8 +292,7 @@ class Core(Component):
             logger.error("error in GetStructures", exc_info=1)
             return lxml.etree.Element("error", type='structure error')
 
-        self.validate_data(meta, structures,
-                           Bcfg2.Server.Plugin.StructureValidator)
+        self.validate_structures(meta, structures)
 
         # Perform altsrc consistency checking
         esrcs = {}
@@ -297,7 +312,7 @@ class Core(Component):
                 config.append(astruct)
             except:
                 logger.error("error in BindStructure", exc_info=1)
-        self.validate_data(meta, config, Bcfg2.Server.Plugin.GoalValidator)
+        self.validate_goals(meta, config)
         logger.info("Generated config for %s in %.03fs" % \
                     (client, time.time() - start))
         return config
@@ -305,10 +320,9 @@ class Core(Component):
     def GetDecisions(self, metadata, mode):
         """Get data for the decision list."""
         result = []
-        for plugin in list(self.plugins.values()):
+        for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision):
             try:
-                if isinstance(plugin, Bcfg2.Server.Plugin.Decision):
-                    result += plugin.GetDecisions(metadata, mode)
+                result += plugin.GetDecisions(metadata, mode)
             except:
                 logger.error("Plugin: %s failed to generate decision list" \
                              % plugin.name, exc_info=1)
@@ -354,8 +368,7 @@ class Core(Component):
             name = self.metadata.resolve_client(address)
             meta = self.build_metadata(name)
 
-            for plugin in [p for p in list(self.plugins.values()) \
-                           if isinstance(p, Bcfg2.Server.Plugin.Probing)]:
+            for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing):
                 for probe in plugin.GetProbes(meta):
                     resp.append(probe)
             return lxml.etree.tostring(resp, encoding='UTF-8',
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 17547be13..a7ab9feab 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -36,7 +36,7 @@ mdata_setup = Bcfg2.Options.OptionParser(opts)
 mdata_setup.parse([])
 del mdata_setup['args']
 
-logger = logging.getLogger('Bcfg2.Plugin')
+logger = logging.getLogger('Bcfg2.Server.Plugin')
 
 default_file_metadata = mdata_setup
 
@@ -81,7 +81,17 @@ class Plugin(object):
     deprecated = False
     conflicts = []
 
+    # Default sort_order to 500. Plugins of the same type are
+    # processed in order of ascending sort_order value. Plugins with
+    # the same sort_order are sorted alphabetically by their name.
+    sort_order = 500
+
     def __init__(self, core, datastore):
+        """Initialize the plugin.
+        
+        :param core: the Bcfg2.Server.Core initializing the plugin
+        :param datastore: the filesystem path of Bcfg2's repository
+        """
         object.__init__(self)
         self.Entries = {}
         self.core = core
@@ -370,9 +380,22 @@ class DirectoryBacked(object):
         object.__init__(self)
         self.name = name
         self.fam = fam
+
+        # self.entries contains information about the files monitored
+        # by this object.... The keys of the dict are the relative
+        # paths to the files. The values are the objects (of type
+        # __child__) that handle their contents.
         self.entries = {}
-        self.inventory = False
-        fam.AddMonitor(name, self)
+
+        # self.handles contains information about the directories
+        # monitored by this object. The keys of the dict are the
+        # values returned by the initial fam.AddMonitor() call (which
+        # appear to be integers). The values are the relative paths of
+        # the directories.
+        self.handles = {}
+
+        # Monitor everything in the plugin's directory
+        self.add_directory_monitor('')
 
     def __getitem__(self, key):
         return self.entries[key]
@@ -380,46 +403,103 @@ class DirectoryBacked(object):
     def __iter__(self):
         return iter(list(self.entries.items()))
 
-    def AddEntry(self, name):
-        """Add new entry to data structures upon file creation."""
-        if name == '':
-            logger.info("got add for empty name")
-        elif name in self.entries:
-            self.entries[name].HandleEvent()
-        else:
-            if ((name[-1] == '~') or
-                (name[:2] == '.#') or
-                (name[-4:] == '.swp') or
-                (name in ['SCCS', '.svn'])):
-                return
-            if not self.patterns.match(name):
+    def add_directory_monitor(self, relative):
+        """Add a new directory to FAM structures for monitoring.
+
+        :param relative: Path name to monitor. This must be relative
+        to the plugin's directory. An empty string value ("") will
+        cause the plugin directory itself to be monitored.
+        """
+        dirpathname = os.path.join(self.data, relative)
+        if relative not in self.handles.values():
+            if not posixpath.isdir(dirpathname):
+                logger.error("Failed to open directory %s" % (dirpathname))
                 return
-            self.entries[name] = self.__child__('%s/%s' % (self.name, name))
-            self.entries[name].HandleEvent()
+            reqid = self.fam.AddMonitor(dirpathname, self)
+            self.handles[reqid] = relative
 
     def HandleEvent(self, event):
-        """Propagate fam events to underlying objects."""
+        """Handle FAM/Gamin events.
+        
+        This method is invoked by FAM/Gamin when it detects a change
+        to a filesystem object we have requsted to be monitored.
+
+        This method manages the lifecycle of events related to the
+        monitored objects, adding them to our indiciess and creating
+        objects of type __child__ that actually do the domain-specific
+        processing. When appropriate, it propogates events those
+        objects by invoking their HandleEvent in turn.
+        """
         action = event.code2str()
-        if event.filename == '':
-            logger.info("Got event for blank filename")
+
+        # Exclude events for actions and filesystem paths we don't
+        # care about
+        if action == 'endExist':
             return
-        if action == 'exists':
-            if event.filename != self.name:
-                self.AddEntry(event.filename)
-        elif action == 'created':
-            self.AddEntry(event.filename)
-        elif action == 'changed':
-            if event.filename in self.entries:
-                self.entries[event.filename].HandleEvent(event)
-        elif action == 'deleted':
-            if event.filename in self.entries:
-                del self.entries[event.filename]
-        elif action in ['endExist']:
-            pass
+        elif os.path.isabs(event.filename[0]):
+            # After AddDirectoryMonitor calls, we receive an 'exists'
+            # event with the just-added directory and its absolute
+            # path name. Ignore these.
+            return
+        elif event.filename == '':
+            logger.warning("Got event for blank filename")
+            return
+
+        # Calculate the absolute and relative paths this event refers to
+        abspath = os.path.join(self.data, self.handles[event.requestID],
+                               event.filename)
+        relpath = os.path.join(self.handles[event.requestID], event.filename)
+
+        if action == 'deleted':
+            for key in self.entries.keys():
+                if key.startswith(relpath):
+                    del self.entries[key]
+            for handle in self.handles.keys():
+                if self.handles[handle].startswith(relpath):
+                    del self.handles[handle]
+        elif posixpath.isdir(abspath):
+            # Deal with events for directories
+            if action in ['exists', 'created']:
+                self.add_directory_monitor(relpath)
+            elif action == 'changed' and relpath in self.entries:
+                # Ownerships, permissions or timestamps changed on the
+                # directory. None of these should affect the contents
+                # of the files, though it could change our ability to
+                # access them.
+                #
+                # It seems like the right thing to do is to cancel
+                # monitoring the directory and then begin monitoring
+                # it again. But the current FileMonitor class doesn't
+                # support canceling, so at least let the user know
+                # that a restart might be a good idea.
+                logger.warn("Directory properties for %s changed, please " +
+                            " consider restarting the server" % (abspath))
+            else:
+                logger.warn("Got unknown dir event %s %s %s" % (event.requestID,
+                                                                event.code2str(),
+                                                                abspath))
         else:
-            print("Got unknown event %s %s %s" % (event.requestID,
-                                                  event.code2str(),
-                                                  event.filename))
+            # Deal with events for non-directories
+            if action in ['exists', 'created']:
+                if ((event.filename[-1] == '~') or
+                    (event.filename[:2] == '.#') or
+                    (event.filename[-4:] == '.swp') or
+                    (event.filename in ['SCCS', '.svn'])):
+                    return
+                if not self.patterns.match(event.filename):
+                    return
+                self.entries[relpath] = self.__child__('%s/%s' % (self.name,
+                                                                  relpath))
+                self.entries[relpath].HandleEvent(event)
+            elif action == 'changed':
+                if relpath in self.entries:
+                    self.entries[relpath].HandleEvent(event)
+                else:
+                    logger.warn("Got %s event for unexpected path %s" % (action, abspath))
+            else:
+                logger.warn("Got unknown file event %s %s %s" % (event.requestID,
+                                                                 event.code2str(),
+                                                                 abspath))
 
 
 class XMLFileBacked(FileBacked):
diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py
index 01ad3c78b..bf0c42416 100644
--- a/src/lib/Server/Plugins/Bundler.py
+++ b/src/lib/Server/Plugins/Bundler.py
@@ -3,6 +3,7 @@ __revision__ = '$Revision$'
 
 import copy
 import lxml.etree
+import os
 import re
 import sys
 
@@ -74,8 +75,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
         """Build all structures for client (metadata)."""
         bundleset = []
         for bundlename in metadata.bundles:
-            entries = [item for (key, item) in list(self.entries.items()) if \
-                       self.patterns.match(key).group('name') == bundlename]
+            entries = [item for (key, item) in self.entries.items() if \
+                       self.patterns.match(os.path.basename(key)).group('name') == bundlename]
             if len(entries) == 0:
                 continue
             elif len(entries) == 1:
diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py
index 389645232..482d457af 100644
--- a/src/lib/Server/Plugins/Deps.py
+++ b/src/lib/Server/Plugins/Deps.py
@@ -49,6 +49,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir,
     __author__ = 'bcfg-dev@mcs.anl.gov'
     __child__ = DepXMLSrc
 
+    # Override the default sort_order (of 500) so that this plugin
+    # gets handled after others running at the default. In particular,
+    # we want to run after Packages, so we can see the final set of
+    # packages that will be installed on the client.
+    sort_order = 750
+
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.StructureValidator.__init__(self)
@@ -59,45 +65,29 @@ class Deps(Bcfg2.Server.Plugin.PrioDir,
         Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event)
 
     def validate_structures(self, metadata, structures):
+        """Examine the passed structures and append any additional
+        prerequisite entries as defined by the files in Deps.
+        """
         entries = []
-        prereqs = []
         for structure in structures:
             for entry in structure.getchildren():
-                if (entry.tag, entry.get('name')) not in entries \
-                       and not isinstance(entry, lxml.etree._Comment):
-                    entries.append((entry.tag, entry.get('name')))
+                tag = entry.tag
+                if tag.startswith('Bound'):
+                    tag = tag[5:]
+                if (tag, entry.get('name')) not in entries \
+                        and not isinstance(entry, lxml.etree._Comment):
+                    entries.append((tag, entry.get('name')))
         entries.sort()
         entries = tuple(entries)
         gdata = list(metadata.groups)
         gdata.sort()
         gdata = tuple(gdata)
+
+        # Check to see if we have cached the prereqs already
         if (entries, gdata) in self.cache:
             prereqs = self.cache[(entries, gdata)]
         else:
-            [src.Cache(metadata) for src in list(self.entries.values())]
-
-            toexamine = list(entries[:])
-            while toexamine:
-                entry = toexamine.pop()
-                matching = [src for src in list(self.entries.values())
-                            if src.cache and entry[0] in src.cache[1]
-                            and entry[1] in src.cache[1][entry[0]]]
-                if len(matching) > 1:
-                    prio = [int(src.priority) for src in matching]
-                    if prio.count(max(prio)) > 1:
-                        self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" %
-                                          (entry[0].lower(), metadata.hostname, entry[1]))
-                        raise Bcfg2.Server.Plugin.PluginExecutionError
-                    index = prio.index(max(prio))
-                    matching = [matching[index]]
-
-                if not matching:
-                    continue
-                elif len(matching) == 1:
-                    for prq in matching[0].cache[1][entry[0]][entry[1]]:
-                        if prq not in prereqs and prq not in entries:
-                            toexamine.append(prq)
-                            prereqs.append(prq)
+            prereqs = self.calculate_prereqs(metadata, entries)
             self.cache[(entries, gdata)] = prereqs
 
         newstruct = lxml.etree.Element("Independent")
@@ -107,3 +97,40 @@ class Deps(Bcfg2.Server.Plugin.PrioDir,
             except:
                 self.logger.error("Failed to add dep entry for %s:%s" % (tag, name))
         structures.append(newstruct)
+
+
+    def calculate_prereqs(self, metadata, entries):
+        """Calculate the prerequisites defined in Deps for the passed
+        set of entries.
+        """
+        prereqs = []
+        [src.Cache(metadata) for src in self.entries.values()]
+
+        toexamine = list(entries[:])
+        while toexamine:
+            entry = toexamine.pop()
+            matching = [src for src in list(self.entries.values())
+                        if src.cache and entry[0] in src.cache[1]
+                        and entry[1] in src.cache[1][entry[0]]]
+            if len(matching) > 1:
+                prio = [int(src.priority) for src in matching]
+                if prio.count(max(prio)) > 1:
+                    self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" %
+                                      (entry[0].lower(), metadata.hostname, entry[1]))
+                    raise Bcfg2.Server.Plugin.PluginExecutionError
+                index = prio.index(max(prio))
+                matching = [matching[index]]
+            elif len(matching) == 1:
+                for prq in matching[0].cache[1][entry[0]][entry[1]]:
+                    # XML comments seem to show up in the cache as a
+                    # tuple with item 0 being callable. The logic
+                    # below filters them out. Would be better to
+                    # exclude them when we load the cache in the first
+                    # place.
+                    if prq not in prereqs and prq not in entries and not callable(prq[0]):
+                        toexamine.append(prq)
+                        prereqs.append(prq)
+            else:
+                continue
+
+        return prereqs
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index 7fc34f178..bfe1ac053 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -222,6 +222,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
     __version__ = '$Id$'
     __author__ = 'bcfg-dev@mcs.anl.gov'
     name = "Metadata"
+    sort_order = 500
 
     def __init__(self, core, datastore, watch_clients=True):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)