Merge branch 'maint' into master

Signed-off-by: Sol Jerome <sol.jerome@gmail.com>

Conflicts:
	doc/appendix/guides/import-existing-ssh-keys.txt
	misc/bcfg2.spec
	src/lib/Bcfg2/Client/Tools/VCS.py
	src/lib/Bcfg2/Client/Tools/YUM.py
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Reporting/Collector.py
	src/lib/Bcfg2/Reporting/Storage/DjangoORM.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/__init__.py
	src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
	src/lib/Bcfg2/Server/Plugin/helpers.py
	src/lib/Bcfg2/Server/Plugins/Metadata.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
	src/lib/Bcfg2/settings.py
	src/sbin/bcfg2-crypt
	src/sbin/bcfg2-reports
	src/sbin/bcfg2-yum-helper
	testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py

14 files changed, 173 insertions, 26 deletions

diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 4f51ebe87..9c22d17ac 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -19,7 +19,7 @@ import Bcfg2.Server.Statistics
 import Bcfg2.Server.FileMonitor
 from itertools import chain
 from Bcfg2.Server.Cache import Cache
-from Bcfg2.Compat import xmlrpclib  # pylint: disable=W0622
+from Bcfg2.Compat import xmlrpclib, wraps  # pylint: disable=W0622
 from Bcfg2.Server.Plugin.exceptions import *  # pylint: disable=W0401,W0614
 from Bcfg2.Server.Plugin.interfaces import *  # pylint: disable=W0401,W0614
 from Bcfg2.Server.Plugin import track_statistics
@@ -74,6 +74,24 @@ def sort_xml(node, key=None):
     node[:] = sorted_children
 
 
+def close_db_connection(func):
+    """ Decorator that closes the Django database connection at the end of
+    the function.  This should decorate any exposed function that
+    might open a database connection. """
+    @wraps(func)
+    def inner(self, *args, **kwargs):
+        """ The decorated function """
+        rv = func(self, *args, **kwargs)
+        if self._database_available:  # pylint: disable=W0212
+            from django import db
+            self.logger.debug("%s: Closing database connection" %
+                              threading.current_thread().name)
+            db.close_connection()
+        return rv
+
+    return inner
+
+
 class CoreInitError(Exception):
     """ Raised when the server core cannot be initialized. """
     pass
@@ -196,6 +214,12 @@ class Core(object):
         self.revision = '-1'
 
         atexit.register(self.shutdown)
+        #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly,
+        #: then :mod:`atexit` calls it *again*, so it gets called
+        #: twice.  This is potentially bad, so we use
+        #: :attr:`Bcfg2.Server.Core._running` as a flag to determine
+        #: if the core needs to be shutdown, and only do it once.
+        self._running = True
 
         #: Threading event to signal worker threads (e.g.,
         #: :attr:`fam_thread`) to shutdown
@@ -403,14 +427,22 @@ class Core(object):
 
     def shutdown(self):
         """ Perform plugin and FAM shutdown tasks. """
-        self.logger.info("Shutting down core...")
+        if not self._running:
+            self.logger.debug("%s: Core already shut down" % self.name)
+            return
+        self.logger.info("%s: Shutting down core..." % self.name)
         if not self.terminate.isSet():
             self.terminate.set()
-            self.fam.shutdown()
-            self.logger.info("FAM shut down")
-            for plugin in list(self.plugins.values()):
-                plugin.shutdown()
-            self.logger.info("All plugins shut down")
+        self._running = False
+        self.fam.shutdown()
+        self.logger.info("%s: FAM shut down" % self.name)
+        for plugin in list(self.plugins.values()):
+            plugin.shutdown()
+        self.logger.info("%s: All plugins shut down" % self.name)
+        if self._database_available:
+            from django import db
+            self.logger.info("%s: Closing database connection" % self.name)
+            db.close_connection()
 
     @property
     def metadata_cache_mode(self):
@@ -601,9 +633,10 @@ class Core(object):
                 del entry.attrib['realname']
                 return ret
             except:
-                self.logger.error("Failed binding entry %s:%s with altsrc %s" %
-                                  (entry.tag, entry.get('realname'),
-                                   entry.get('name')))
+                self.logger.error(
+                    "Failed binding entry %s:%s with altsrc %s: %s" %
+                    (entry.tag, entry.get('realname'), entry.get('name'),
+                     sys.exc_info()[1]))
                 entry.set('name', oldname)
                 self.logger.error("Falling back to %s:%s" %
                                   (entry.tag, entry.get('name')))
@@ -1052,6 +1085,7 @@ class Core(object):
 
     @exposed
     @track_statistics()
+    @close_db_connection
     def DeclareVersion(self, address, version):
         """ Declare the client version.
 
@@ -1074,6 +1108,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def GetProbes(self, address):
         """ Fetch probes for the client.
 
@@ -1099,6 +1134,7 @@ class Core(object):
                                 (client, err))
 
     @exposed
+    @close_db_connection
     def RecvProbeData(self, address, probedata):
         """ Receive probe data from clients.
 
@@ -1146,6 +1182,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def AssertProfile(self, address, profile):
         """ Set profile for a client.
 
@@ -1165,6 +1202,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def GetConfig(self, address):
         """ Build config for a client by calling
         :func:`BuildConfiguration`.
@@ -1184,6 +1222,7 @@ class Core(object):
             self.critical_error("Metadata consistency failure for %s" % client)
 
     @exposed
+    @close_db_connection
     def RecvStats(self, address, stats):
         """ Act on statistics upload with :func:`process_statistics`.
 
@@ -1199,6 +1238,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def GetDecisionList(self, address, mode):
         """ Get the decision list for the client with :func:`GetDecisions`.
 
diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
index b8eb06aa1..c4b34a469 100644
--- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
+++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
@@ -212,7 +212,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent):
     AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__
 
     def shutdown(self):
-        if self.notifier:
+        if self.started and self.notifier:
             self.notifier.stop()
     shutdown.__doc__ = Pseudo.shutdown.__doc__
 
diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
index 109ace61f..22c97a0fe 100644
--- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
+++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
@@ -123,12 +123,30 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
 
     @classmethod
     def Errors(cls):
-        return {"unknown-entry-type": "error",
+        return {"missing-elements": "error",
+                "unknown-entry-type": "error",
                 "unknown-entry-tag": "error",
                 "required-attrs-missing": "error",
                 "required-attr-format": "error",
                 "extra-attrs": "warning"}
 
+    def check_default_acl(self, path):
+        """ Check that a default ACL contains either no entries or minimum
+        required entries """
+        defaults = 0
+        if path.xpath("ACL[@type='default' and @scope='user' and @user='']"):
+            defaults += 1
+        if path.xpath("ACL[@type='default' and @scope='group' and @group='']"):
+            defaults += 1
+        if path.xpath("ACL[@type='default' and @scope='other']"):
+            defaults += 1
+        if defaults > 0 and defaults < 3:
+            self.LintError(
+                "missing-elements",
+                "A Path must have either no default ACLs or at"
+                " least default:user::, default:group:: and"
+                " default:other::")
+
     def check_packages(self):
         """ Check Packages sources for Source entries with missing
         attributes. """
@@ -172,7 +190,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
                                               rules.name))
 
     def check_bundles(self):
-        """ Check bundles for BoundPath entries with missing
+        """ Check bundles for BoundPath and BoundPackage entries with missing
         attrs. """
         if 'Bundler' not in self.core.plugins:
             return
@@ -192,6 +210,15 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
                             "required-attrs-missing",
                             "Path tags require either a 'name' or 'glob' "
                             "attribute: \n%s" % self.RenderXML(path))
+                # ensure that abstract Package tags have either name
+                # or group specified
+                for package in xdata.xpath("//Package"):
+                    if ('name' not in package.attrib and
+                        'group' not in package.attrib):
+                        self.LintError(
+                            "required-attrs-missing",
+                            "Package tags require either a 'name' or 'group' "
+                            "attribute: \n%s" % self.RenderXML(package))
 
     def check_entry(self, entry, filename):
         """ Generic entry check.
@@ -231,6 +258,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
                     required_attrs['major'] = is_device_mode
                     required_attrs['minor'] = is_device_mode
 
+            if tag == 'Path':
+                self.check_default_acl(entry)
+
             if tag == 'ACL' and 'scope' in required_attrs:
                 required_attrs[entry.get('scope')] = is_username
 
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 3ad78ade4..0b3f1e24d 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -90,6 +90,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
                 "xml-failed-to-parse": "error",
                 "xml-failed-to-read": "error",
                 "xml-failed-to-verify": "error",
+                "xinclude-does-not-exist": "error",
                 "input-output-error": "error"}
 
     def check_properties(self):
@@ -115,6 +116,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
         try:
             xdata = lxml.etree.parse(filename)
             if self.files is None:
+                self._expand_wildcard_xincludes(xdata)
                 xdata.xinclude()
             return xdata
         except (lxml.etree.XIncludeError, SyntaxError):
@@ -132,6 +134,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
                            "Failed to open file %s" % filename)
             return False
 
+    def _expand_wildcard_xincludes(self, xdata):
+        """ a lightweight version of
+        :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """
+        xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE
+        for el in xdata.findall('//' + xinclude):
+            name = el.get("href")
+            if name.startswith("/"):
+                fpath = name
+            else:
+                fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name)
+
+            # expand globs in xinclude, a bcfg2-specific extension
+            extras = glob.glob(fpath)
+            if not extras:
+                msg = "%s: %s does not exist, skipping: %s" % \
+                      (xdata.docinfo.URL, name, self.RenderXML(el))
+                if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE):
+                    self.logger.debug(msg)
+                else:
+                    self.LintError("xinclude-does-not-exist", msg)
+
+            parent = el.getparent()
+            parent.remove(el)
+            for extra in extras:
+                if extra != xdata.docinfo.URL:
+                    lxml.etree.SubElement(parent, xinclude, href=extra)
+
     def validate(self, filename, schemafile, schema=None):
         """ Validate a file against the given schema.
 
diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
index 04151d764..6383a3c99 100644
--- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
+++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
@@ -10,7 +10,9 @@ import Bcfg2.Server.Lint
 
 try:
     import json
-except ImportError:
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
+except (ImportError, AttributeError):
     import simplejson as json
 
 
diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py
index 294963669..724b34d8d 100644
--- a/src/lib/Bcfg2/Server/MultiprocessingCore.py
+++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py
@@ -275,6 +275,7 @@ class ChildCore(Core):
     @exposed
     def GetConfig(self, client):
         """ Render the configuration for a client """
+        self.metadata.update_client_list()
         self.logger.debug("%s: Building configuration for %s" %
                           (self.name, client))
         return lxml.etree.tostring(self.BuildConfiguration(client))
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index aa8db2bc0..407e9df46 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -275,7 +275,8 @@ class PluginDatabaseModel(object):
     inherit from.  This is just a mixin; models must also inherit from
     django.db.models.Model to be valid Django models."""
 
-    class Meta:  # pylint: disable=C0111,W0232
+    class Meta(object):  # pylint: disable=W0232
+        """ Model metadata options """
         app_label = "Server"
 
 
diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py
index 622b69c79..c45d6fa84 100644
--- a/src/lib/Bcfg2/Server/Plugin/interfaces.py
+++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py
@@ -216,6 +216,10 @@ class Metadata(object):
         """
         raise NotImplementedError
 
+    def update_client_list(self):
+        """ Re-read the cached list of clients """
+        raise NotImplementedError
+
 
 class Connector(object):
     """ Connector plugins augment client metadata instances with
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 6ff256147..bf51ff678 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -686,7 +686,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 client = MetadataClientModel(hostname=client_name)
                 # pylint: enable=E1102
                 client.save()
-            self.clients = self.list_clients()
+            self.update_client_list()
             return client
         else:
             try:
@@ -739,7 +739,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                                       attribs, alias=True)
 
     def list_clients(self):
-        """ List all clients in client database """
+        """ List all clients in client database.
+
+        Making ``self.clients`` a property and reading the client list
+        dynamically from the database on every call to
+        ``self.clients`` can result in very high rates of database
+        reads, so we cache the ``list_clients()`` results to reduce
+        the database load.  When the database is in use, the client
+        list is reread periodically with
+        :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """
         if self._use_db:
             return set([c.hostname for c in MetadataClientModel.objects.all()])
         else:
@@ -790,7 +798,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 self.logger.warning(msg)
                 raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
             client.delete()
-            self.clients = self.list_clients()
+            self.update_client_list()
         else:
             return self._remove_xdata(self.clients_xml, "Client", client_name)
 
@@ -859,8 +867,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
             except KeyError:
                 self.clientgroups[clname] = [profile]
         self.states['clients.xml'] = True
-        if self._use_db:
-            self.clients = self.list_clients()
+        self.update_client_list()
 
     def _get_condition(self, element):
         """ Return a predicate that returns True if a client meets
@@ -1452,6 +1459,30 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         return True
     # pylint: enable=R0911,R0912
 
+    def update_client_list(self):
+        """ Re-read the client list from the database (if the database is in
+        use) """
+        if self._use_db:
+            self.logger.debug("Metadata: Re-reading client list from database")
+            old = set(self.clients)
+            self.clients = self.list_clients()
+            new = set(self.clients)
+            added = new - old
+            removed = old - new
+            self.logger.debug("Metadata: Added %s clients: %s" %
+                              (len(added), added))
+            self.logger.debug("Metadata: Removed %s clients: %s" %
+                              (len(removed), removed))
+            # we could do this with set.symmetric_difference(), but we
+            # want detailed numbers of added/removed clients for
+            # logging
+            for client in added.union(removed):
+                self.expire_cache(client)
+
+    def start_client_run(self, metadata):
+        """ Hook to reread client list if the database is in use """
+        self.update_client_list()
+
     def end_statistics(self, metadata):
         """ Hook to toggle clients in bootstrap mode """
         if self.auth.get(metadata.hostname,
diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py
index ba7baab11..c5fb46c97 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ohai.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py
@@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin
 
 try:
     import json
-except ImportError:
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
+except (ImportError, AttributeError):
     import simplejson as json
 
 PROBECODE = """#!/bin/sh
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index dba56eed2..c1d53a78e 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -72,6 +72,7 @@ class AptSource(Source):
     def read_files(self):
         bdeps = dict()
         bprov = dict()
+        self.essentialpkgs = set()
         depfnames = ['Depends', 'Pre-Depends']
         if self.recommended:
             depfnames.append('Recommends')
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 6a493c19d..3cfda9e9c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -90,7 +90,9 @@ try:
     import yum
     try:
         import json
-    except ImportError:
+        # py2.4 json library is structured differently
+        json.loads  # pylint: disable=W0104
+    except (ImportError, AttributeError):
         import simplejson as json
     HAS_YUM = True
 except ImportError:
@@ -354,8 +356,8 @@ class YumCollection(Collection):
                 self.__class__._helper = find_executable('bcfg2-yum-helper')
                 if not self.__class__._helper:
                     self.__class__._helper = "/usr/sbin/bcfg2-yum-helper"
-            # pylint: enable=W0212
-        return self._helper
+        return self.__class__._helper
+        # pylint: enable=W0212
 
     @property
     def use_yum(self):
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 553c16202..21d50ace6 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -51,8 +51,10 @@ def load_django_models():
 
 try:
     import json
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
     HAS_JSON = True
-except ImportError:
+except (ImportError, AttributeError):
     try:
         import simplejson as json
         HAS_JSON = True
diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py
index 87cee7029..04314218c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Properties.py
+++ b/src/lib/Bcfg2/Server/Plugins/Properties.py
@@ -13,8 +13,10 @@ from Bcfg2.Server.Plugin import PluginExecutionError
 
 try:
     import json
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
     HAS_JSON = True
-except ImportError:
+except (ImportError, AttributeError):
     try:
         import simplejson as json
         HAS_JSON = True