Merge branch 'maint'

Conflicts:
	doc/server/plugins/generators/cfg.txt
	doc/server/plugins/generators/tcheetah.txt
	src/lib/Bcfg2/Server/Admin/Xcmd.py
	src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py

8 files changed, 173 insertions, 50 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
index 7af69ec81..8a787751c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
@@ -24,6 +24,24 @@ from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \
 #: facility for passing it otherwise.
 CFG = None
 
+_HANDLERS = []
+
+
+def handlers():
+    """ A list of Cfg handler classes. Loading the handlers must
+    be done at run-time, not at compile-time, or it causes a
+    circular import and Bad Things Happen."""
+    if not _HANDLERS:
+        for submodule in walk_packages(path=__path__, prefix=__name__ + "."):
+            mname = submodule[1].rsplit('.', 1)[-1]
+            module = getattr(__import__(submodule[1]).Server.Plugins.Cfg,
+                             mname)
+            hdlr = getattr(module, mname)
+            if issubclass(hdlr, CfgBaseFileMatcher):
+                _HANDLERS.append(hdlr)
+        _HANDLERS.sort(key=operator.attrgetter("__priority__"))
+    return _HANDLERS
+
 
 class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData,
                          Bcfg2.Server.Plugin.Debuggable):
@@ -432,24 +450,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
             entry.set_debug(debug)
         return rv
 
-    @property
-    def handlers(self):
-        """ A list of Cfg handler classes. Loading the handlers must
-        be done at run-time, not at compile-time, or it causes a
-        circular import and Bad Things Happen."""
-        if self._handlers is None:
-            self._handlers = []
-            for submodule in walk_packages(path=__path__,
-                                           prefix=__name__ + "."):
-                mname = submodule[1].rsplit('.', 1)[-1]
-                module = getattr(__import__(submodule[1]).Server.Plugins.Cfg,
-                                 mname)
-                hdlr = getattr(module, mname)
-                if CfgBaseFileMatcher in hdlr.__mro__:
-                    self._handlers.append(hdlr)
-            self._handlers.sort(key=operator.attrgetter("__priority__"))
-        return self._handlers
-
     def handle_event(self, event):
         """ Dispatch a FAM event to :func:`entry_init` or the
         appropriate child handler object.
@@ -466,7 +466,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
                 # process a bogus changed event like a created
                 return
 
-            for hdlr in self.handlers:
+            for hdlr in handlers():
                 if hdlr.handles(event, basename=self.path):
                     if action == 'changed':
                         # warn about a bogus 'changed' event, but
@@ -546,10 +546,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
 
     def bind_entry(self, entry, metadata):
         self.bind_info_to_entry(entry, metadata)
-        data = self._generate_data(entry, metadata)
-
-        for fltr in self.get_handlers(metadata, CfgFilter):
-            data = fltr.modify_data(entry, metadata, data)
+        data, generator = self._generate_data(entry, metadata)
+
+        if generator is not None:
+            # apply no filters if the data was created by a CfgCreator
+            for fltr in self.get_handlers(metadata, CfgFilter):
+                if fltr.specific <= generator.specific:
+                    # only apply filters that are as specific or more
+                    # specific than the generator used for this entry.
+                    # Note that specificity comparison is backwards in
+                    # this sense, since it's designed to sort from
+                    # most specific to least specific.
+                    data = fltr.modify_data(entry, metadata, data)
 
         if self.setup['validate']:
             try:
@@ -658,7 +666,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
         :type entry: lxml.etree._Element
         :param metadata: The client metadata to generate data for
         :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
-        :returns: string - the data for the entry
+        :returns: tuple of (string, generator) - the data for the
+                  entry and the generator used to generate it (or
+                  None, if data was created)
         """
         try:
             generator = self.best_matching(metadata,
@@ -667,10 +677,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
         except PluginExecutionError:
             # if no creators or generators exist, _create_data()
             # raises an appropriate exception
-            return self._create_data(entry, metadata)
+            return (self._create_data(entry, metadata), None)
 
         try:
-            return generator.get_data(entry, metadata)
+            return (generator.get_data(entry, metadata), generator)
         except:
             msg = "Cfg: Error rendering %s: %s" % (entry.get("name"),
                                                    sys.exc_info()[1])
@@ -837,10 +847,13 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin):
     def Run(self):
         for basename, entry in list(self.core.plugins['Cfg'].entries.items()):
             self.check_pubkey(basename, entry)
+        self.check_missing_files()
 
     @classmethod
     def Errors(cls):
-        return {"no-pubkey-xml": "warning"}
+        return {"no-pubkey-xml": "warning",
+                "unknown-cfg-files": "error",
+                "extra-cfg-files": "error"}
 
     def check_pubkey(self, basename, entry):
         """ check that privkey.xml files have corresponding pubkey.xml
@@ -862,3 +875,41 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin):
                 self.LintError("no-pubkey-xml",
                                "%s has no corresponding pubkey.xml at %s" %
                                (basename, pubkey))
+
+    def check_missing_files(self):
+        """ check that all files on the filesystem are known to Cfg """
+        cfg = self.core.plugins['Cfg']
+
+        # first, collect ignore patterns from handlers
+        ignore = []
+        for hdlr in handlers():
+            ignore.extend(hdlr.__ignore__)
+
+        # next, get a list of all non-ignored files on the filesystem
+        all_files = set()
+        for root, _, files in os.walk(cfg.data):
+            all_files.update(os.path.join(root, fname)
+                             for fname in files
+                             if not any(fname.endswith("." + i)
+                                        for i in ignore))
+
+        # next, get a list of all files known to Cfg
+        cfg_files = set()
+        for root, eset in cfg.entries.items():
+            cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname)
+                             for fname in eset.entries.keys())
+
+        # finally, compare the two
+        unknown_files = all_files - cfg_files
+        extra_files = cfg_files - all_files
+        if unknown_files:
+            self.LintError(
+                "unknown-cfg-files",
+                "Files on the filesystem could not be understood by Cfg: %s" %
+                "; ".join(unknown_files))
+        if extra_files:
+            self.LintError(
+                "extra-cfg-files",
+                "Cfg has entries for files that do not exist on the "
+                "filesystem: %s\nThis is probably a bug." %
+                "; ".join(extra_files))
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 507973fa6..a9b622637 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -40,6 +40,8 @@ if HAS_DJANGO:
         """ dict-like object to make it easier to access client bcfg2
         versions from the database """
 
+        create = False
+
         def __getitem__(self, key):
             try:
                 return MetadataClientModel.objects.get(hostname=key).version
diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
index 1736becc7..71128d64c 100644
--- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
+++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
@@ -9,6 +9,8 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin,
                   Bcfg2.Server.Plugin.GoalValidator):
     """POSIXCompat is a goal validator plugin for POSIX entries."""
 
+    create = False
+
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.GoalValidator.__init__(self)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index a4b17f05a..98add2ccf 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -674,7 +674,10 @@ class YumCollection(Collection):
             gdicts.append(dict(group=group, type=ptype))
 
         if self.use_yum:
-            return self.call_helper("get_groups", inputdata=gdicts)
+            try:
+                return self.call_helper("get_groups", inputdata=gdicts)
+            except ValueError:
+                return dict()
         else:
             pkgs = dict()
             for gdict in gdicts:
@@ -837,12 +840,13 @@ class YumCollection(Collection):
             return Collection.complete(self, packagelist)
 
         if packagelist:
-            result = \
-                self.call_helper("complete",
-                                 dict(packages=list(packagelist),
-                                      groups=list(self.get_relevant_groups())))
-            if not result:
-                # some sort of error, reported by call_helper()
+            try:
+                result = self.call_helper(
+                    "complete",
+                    dict(packages=list(packagelist),
+                         groups=list(self.get_relevant_groups())))
+            except ValueError:
+                # error reported by call_helper()
                 return set(), packagelist
             # json doesn't understand sets or tuples, so we get back a
             # lists of lists (packages) and a list of unicode strings
@@ -873,28 +877,39 @@ class YumCollection(Collection):
                   ``bcfg2-yum-helper`` command.
         """
         cmd = [self.helper, "-c", self.cfgfile]
-        verbose = self.debug_flag or self.setup['verbose']
-        if verbose:
+        if self.setup['verbose']:
+            cmd.append("-v")
+        if self.debug_flag:
+            if not self.setup['verbose']:
+                # ensure that running in debug gets -vv, even if
+                # verbose is not enabled
+                cmd.append("-v")
             cmd.append("-v")
         cmd.append(command)
-        self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose)
+        self.debug_log("Packages: running %s" % " ".join(cmd))
         if inputdata:
             result = self.cmd.run(cmd, inputdata=json.dumps(inputdata))
         else:
             result = self.cmd.run(cmd)
         if not result.success:
+            errlines = result.error.splitlines()
             self.logger.error("Packages: error running bcfg2-yum-helper: %s" %
-                              result.error)
+                              errlines[0])
+            for line in errlines[1:]:
+                self.logger.error("Packages: %s" % line)
         elif result.stderr:
+            errlines = result.stderr.splitlines()
             self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" %
-                           result.stderr, flag=verbose)
+                           errlines[0])
+            for line in errlines[1:]:
+                self.debug_log("Packages: %s" % line)
         try:
             return json.loads(result.stdout)
         except ValueError:
             err = sys.exc_info()[1]
             self.logger.error("Packages: error reading bcfg2-yum-helper "
                               "output: %s" % err)
-            return None
+            raise
 
     def setup_data(self, force_update=False):
         """ Do any collection-level data setup tasks. This is called
@@ -920,13 +935,21 @@ class YumCollection(Collection):
         if force_update:
             # we call this twice: one to clean up data from the old
             # config, and once to clean up data from the new config
-            self.call_helper("clean")
+            try:
+                self.call_helper("clean")
+            except ValueError:
+                # error reported by call_helper
+                pass
 
         os.unlink(self.cfgfile)
         self.write_config()
 
         if force_update:
-            self.call_helper("clean")
+            try:
+                self.call_helper("clean")
+            except ValueError:
+                # error reported by call_helper
+                pass
 
 
 class YumSource(Source):
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index e97607093..6827c3d1f 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -14,6 +14,7 @@ from Bcfg2.Server.Statistics import track_statistics
 
 try:
     from django.db import models
+    from django.core.exceptions import MultipleObjectsReturned
     HAS_DJANGO = True
 
     class ProbesDataModel(models.Model,
@@ -254,12 +255,15 @@ class Probes(Bcfg2.Server.Plugin.Probing,
 
         for group in self.cgroups[client.hostname]:
             try:
-                ProbesGroupsModel.objects.get(hostname=client.hostname,
-                                              group=group)
-            except ProbesGroupsModel.DoesNotExist:
-                grp = ProbesGroupsModel(hostname=client.hostname,
-                                        group=group)
-                grp.save()
+                ProbesGroupsModel.objects.get_or_create(
+                    hostname=client.hostname,
+                    group=group)
+            except MultipleObjectsReturned:
+                ProbesGroupsModel.objects.filter(hostname=client.hostname,
+                                                 group=group).delete()
+                ProbesGroupsModel.objects.get_or_create(
+                    hostname=client.hostname,
+                    group=group)
         ProbesGroupsModel.objects.filter(
             hostname=client.hostname).exclude(
                 group__in=self.cgroups[client.hostname]).delete()
diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py
index c3a2221f6..41e6bf8b5 100644
--- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py
+++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py
@@ -6,7 +6,9 @@ import Bcfg2.Server.Plugin
 class ServiceCompat(Bcfg2.Server.Plugin.Plugin,
                     Bcfg2.Server.Plugin.GoalValidator):
     """ Use old-style service modes for older clients """
-    name = 'ServiceCompat'
+
+    create = False
+
     __author__ = 'bcfg-dev@mcs.anl.gov'
     mode_map = {('true', 'true'): 'default',
                 ('interactive', 'true'): 'interactive_only',
diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py
index 34a6e89e0..dfe864d48 100644
--- a/src/lib/Bcfg2/Server/Plugins/Svn.py
+++ b/src/lib/Bcfg2/Server/Plugins/Svn.py
@@ -60,9 +60,48 @@ class Svn(Bcfg2.Server.Plugin.Version):
             self.client.callback_conflict_resolver = \
                 self.get_conflict_resolver(choice)
 
+            try:
+                if self.core.setup.cfp.get(
+                        "svn",
+                        "always_trust").lower() == "true":
+                    self.client.callback_ssl_server_trust_prompt = \
+                        self.ssl_server_trust_prompt
+            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+                self.logger.debug("Svn: Using subversion cache for SSL "
+                                  "certificate trust")
+
+            try:
+                if (self.core.setup.cfp.get("svn", "user") and
+                    self.core.setup.cfp.get("svn", "password")):
+                    self.client.callback_get_login = \
+                        self.get_login
+            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+                self.logger.info("Svn: Using subversion cache for "
+                                 "password-based authetication")
+
         self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" %
                           self.vcs_path)
 
+    # pylint: disable=W0613
+    def get_login(self, realm, username, may_save):
+        """ PySvn callback to get credentials for HTTP basic authentication """
+        self.logger.debug("Svn: Logging in with username: %s" %
+                          self.core.setup.cfp.get("svn", "user"))
+        return True, \
+            self.core.setup.cfp.get("svn", "user"), \
+            self.core.setup.cfp.get("svn", "password"), \
+            False
+    # pylint: enable=W0613
+
+    def ssl_server_trust_prompt(self, trust_dict):
+        """ PySvn callback to always trust SSL certificates from SVN server """
+        self.logger.debug("Svn: Trusting SSL certificate from %s, "
+                          "issued by %s for realm %s" %
+                          (trust_dict['hostname'],
+                           trust_dict['issuer_dname'],
+                           trust_dict['realm']))
+        return True, trust_dict['failures'], False
+
     def get_conflict_resolver(self, choice):
         """ Get a PySvn conflict resolution callback """
         def callback(conflict_description):
diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
index 050ba3b3e..77bdd6576 100644
--- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
+++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
@@ -114,7 +114,7 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin):
 
     def Run(self):
         for helper in self.core.plugins['TemplateHelper'].entries.values():
-            if self.HandlesFile(helper):
+            if self.HandlesFile(helper.name):
                 self.check_helper(helper.name)
 
     def check_helper(self, helper):