Merge branch 'maint' into master

Signed-off-by: Sol Jerome <sol.jerome@gmail.com>

Conflicts:
	doc/appendix/guides/import-existing-ssh-keys.txt
	misc/bcfg2.spec
	src/lib/Bcfg2/Client/Tools/VCS.py
	src/lib/Bcfg2/Client/Tools/YUM.py
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Reporting/Collector.py
	src/lib/Bcfg2/Reporting/Storage/DjangoORM.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/__init__.py
	src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
	src/lib/Bcfg2/Server/Plugin/helpers.py
	src/lib/Bcfg2/Server/Plugins/Metadata.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
	src/lib/Bcfg2/settings.py
	src/sbin/bcfg2-crypt
	src/sbin/bcfg2-reports
	src/sbin/bcfg2-yum-helper
	testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py

25 files changed, 375 insertions, 108 deletions

diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py
index 4f6953b2a..dc5fc6755 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py
@@ -4,6 +4,7 @@ import sys
 import Bcfg2.Client.XML
 from augeas import Augeas
 from Bcfg2.Client.Tools.POSIX.base import POSIXTool
+from Bcfg2.Client.Tools.POSIX.File import POSIXFile
 
 
 class AugeasCommand(object):
@@ -187,13 +188,14 @@ class Insert(AugeasCommand):
 class POSIXAugeas(POSIXTool):
     """ Handle <Path type='augeas'...> entries.  See
     :ref:`client-tools-augeas`. """
-
-    __handles__ = [('Path', 'augeas')]
-    __req__ = {'Path': ['type', 'name', 'setting', 'value']}
+    __req__ = ['name', 'mode', 'owner', 'group']
 
     def __init__(self, config):
         POSIXTool.__init__(self, config)
         self._augeas = dict()
+        # file tool for setting initial values of files that don't
+        # exist
+        self.filetool = POSIXFile(logger, setup, config)
 
     def get_augeas(self, entry):
         """ Get an augeas object for the given entry. """
@@ -214,9 +216,9 @@ class POSIXAugeas(POSIXTool):
         return self._augeas[entry.get("name")]
 
     def fully_specified(self, entry):
-        return entry.text is not None
+        return len(entry.getchildren()) != 0
 
-    def get_commands(self, entry, unverified=False):
+    def get_commands(self, entry):
         """ Get a list of commands to verify or install.
 
         @param entry: The entry to get commands from.
@@ -229,7 +231,7 @@ class POSIXAugeas(POSIXTool):
         """
         rv = []
         for cmd in entry.iterchildren():
-            if unverified and cmd.get("verified", "false") != "false":
+            if cmd.tag == "Initial":
                 continue
             if cmd.tag in globals():
                 rv.append(globals()[cmd.tag](cmd, self.get_augeas(entry),
@@ -266,7 +268,18 @@ class POSIXAugeas(POSIXTool):
 
     def install(self, entry):
         rv = True
-        for cmd in self.get_commands(entry, unverified=True):
+        if entry.get("current_exists", "true") == "false":
+            initial = entry.find("Initial")
+            if initial is not None:
+                self.logger.debug("Augeas: Setting initial data for %s" %
+                                  entry.get("name"))
+                file_entry = Bcfg2.Client.XML.Element("Path",
+                                                      **dict(entry.attrib))
+                file_entry.text = initial.text
+                self.filetool.install(file_entry)
+                # re-parse the file
+                self.get_augeas(entry).load()
+        for cmd in self.get_commands(entry):
             try:
                 cmd.install()
             except:  # pylint: disable=W0702
@@ -277,8 +290,7 @@ class POSIXAugeas(POSIXTool):
         try:
             self.get_augeas(entry).save()
         except:  # pylint: disable=W0702
-            self.logger.error(
-                "Failure saving Augeas changes to %s: %s" %
-                (entry.get("name"), sys.exc_info()[1]))
+            self.logger.error("Failure saving Augeas changes to %s: %s" %
+                              (entry.get("name"), sys.exc_info()[1]))
             rv = False
         return POSIXTool.install(self, entry) and rv
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py
index 712620206..8895eaae1 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py
@@ -217,18 +217,13 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
                 acl.delete_entry(aclentry)
         if os.path.isdir(path):
             defacl = posix1e.ACL(filedef=path)
-            if not defacl.valid():
-                # when a default ACL is queried on a directory that
-                # has no default ACL entries at all, you get an empty
-                # ACL, which is not valid.  in this circumstance, we
-                # just copy the access ACL to get a base valid ACL
-                # that we can add things to.
-                defacl = posix1e.ACL(acl=acl)
-            else:
-                for aclentry in defacl:
-                    if aclentry.tag_type in [posix1e.ACL_USER,
-                                             posix1e.ACL_GROUP]:
-                        defacl.delete_entry(aclentry)
+            for aclentry in defacl:
+                if aclentry.tag_type in [posix1e.ACL_USER,
+                                         posix1e.ACL_USER_OBJ,
+                                         posix1e.ACL_GROUP,
+                                         posix1e.ACL_GROUP_OBJ,
+                                         posix1e.ACL_OTHER]:
+                    defacl.delete_entry(aclentry)
         else:
             defacl = None
 
@@ -254,10 +249,16 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
             try:
                 if scope == posix1e.ACL_USER:
                     scopename = "user"
-                    aclentry.qualifier = self._norm_uid(qualifier)
+                    if qualifier:
+                        aclentry.qualifier = self._norm_uid(qualifier)
+                    else:
+                        aclentry.tag_type = posix1e.ACL_USER_OBJ
                 elif scope == posix1e.ACL_GROUP:
                     scopename = "group"
-                    aclentry.qualifier = self._norm_gid(qualifier)
+                    if qualifier:
+                        aclentry.qualifier = self._norm_gid(qualifier)
+                    else:
+                        aclentry.tag_type = posix1e.ACL_GROUP_OBJ
             except (OSError, KeyError):
                 err = sys.exc_info()[1]
                 self.logger.error("POSIX: Could not resolve %s %s: %s" %
@@ -358,7 +359,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
         try:
             # single octal digit
             rv = int(perms)
-            if rv > 0 and rv < 8:
+            if rv >= 0 and rv < 8:
                 return rv
             else:
                 self.logger.error("POSIX: Permissions digit out of range in "
@@ -388,13 +389,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
         """ Get a string representation of the given ACL.  aclkey must
         be a tuple of (<acl type>, <acl scope>, <qualifier>) """
         atype, scope, qualifier = aclkey
+        if not qualifier:
+            qualifier = ''
         acl_str = []
         if atype == 'default':
             acl_str.append(atype)
-        if scope == posix1e.ACL_USER:
+        if scope == posix1e.ACL_USER or scope == posix1e.ACL_USER_OBJ:
             acl_str.append("user")
-        elif scope == posix1e.ACL_GROUP:
+        elif scope == posix1e.ACL_GROUP or scope == posix1e.ACL_GROUP_OBJ:
             acl_str.append("group")
+        elif scope == posix1e.ACL_OTHER:
+            acl_str.append("other")
         acl_str.append(qualifier)
         acl_str.append(self._acl_perm2string(perms))
         return ":".join(acl_str)
@@ -414,7 +419,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
         """ Get data on the existing state of <path> -- e.g., whether
         or not it exists, owner, group, permissions, etc. """
         try:
-            ondisk = os.stat(path)
+            ondisk = os.lstat(path)
         except OSError:
             self.logger.debug("POSIX: %s does not exist" % path)
             return (False, None, None, None, None, None)
@@ -451,7 +456,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
 
         if HAS_SELINUX:
             try:
-                secontext = selinux.getfilecon(path)[1].split(":")[2]
+                secontext = selinux.lgetfilecon(path)[1].split(":")[2]
             except (OSError, KeyError):
                 err = sys.exc_info()[1]
                 self.logger.debug("POSIX: Could not get current SELinux "
@@ -460,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
         else:
             secontext = None
 
-        if HAS_ACLS:
+        if HAS_ACLS and not stat.S_ISLNK(ondisk[stat.ST_MODE]):
             acls = self._list_file_acls(path)
         else:
             acls = None
@@ -562,9 +567,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
         wanted = dict()
         for acl in entry.findall("ACL"):
             if acl.get("scope") == "user":
-                scope = posix1e.ACL_USER
+                if acl.get("user"):
+                    scope = posix1e.ACL_USER
+                else:
+                    scope = posix1e.ACL_USER_OBJ
             elif acl.get("scope") == "group":
-                scope = posix1e.ACL_GROUP
+                if acl.get("group"):
+                    scope = posix1e.ACL_GROUP
+                else:
+                    scope = posix1e.ACL_GROUP_OBJ
+            elif acl.get("scope") == "other":
+                scope = posix1e.ACL_OTHER
             else:
                 self.logger.error("POSIX: Unknown ACL scope %s" %
                                   acl.get("scope"))
@@ -573,7 +586,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
                 self.logger.error("POSIX: No permissions set for ACL: %s" %
                                   Bcfg2.Client.XML.tostring(acl))
                 continue
-            wanted[(acl.get("type"), scope, acl.get(acl.get("scope")))] = \
+            qual = acl.get(acl.get("scope"))
+            if not qual:
+                qual = ''
+            wanted[(acl.get("type"), scope, qual)] = \
                 self._norm_acl_perms(acl.get('perms'))
         return wanted
 
@@ -587,11 +603,12 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
             """ Given an ACL object, process it appropriately and add
             it to the return value """
             try:
+                qual = ''
                 if acl.tag_type == posix1e.ACL_USER:
                     qual = pwd.getpwuid(acl.qualifier)[0]
                 elif acl.tag_type == posix1e.ACL_GROUP:
                     qual = grp.getgrgid(acl.qualifier)[0]
-                else:
+                elif atype == "access" or acl.tag_type == posix1e.ACL_MASK:
                     return
             except (OSError, KeyError):
                 err = sys.exc_info()[1]
@@ -621,9 +638,38 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
                 _process_acl(acl, "default")
         return existing
 
-    def _verify_acls(self, entry, path=None):
+    def _verify_acls(self, entry, path=None):  # pylint: disable=R0912
         """ verify POSIX ACLs on the given entry.  return True if all
         ACLS are correct, false otherwise """
+        def _verify_acl(aclkey, perms):
+            """ Given ACL data, process it appropriately and add it to
+            missing or wrong lists if appropriate """
+            if aclkey not in existing:
+                missing.append(self._acl2string(aclkey, perms))
+            elif existing[aclkey] != perms:
+                wrong.append((self._acl2string(aclkey, perms),
+                              self._acl2string(aclkey, existing[aclkey])))
+            if path == entry.get("name"):
+                atype, scope, qual = aclkey
+                aclentry = Bcfg2.Client.XML.Element("ACL", type=atype,
+                                                    perms=str(perms))
+                if (scope == posix1e.ACL_USER or
+                    scope == posix1e.ACL_USER_OBJ):
+                    aclentry.set("scope", "user")
+                elif (scope == posix1e.ACL_GROUP or
+                      scope == posix1e.ACL_GROUP_OBJ):
+                    aclentry.set("scope", "group")
+                elif scope == posix1e.ACL_OTHER:
+                    aclentry.set("scope", "other")
+                else:
+                    self.logger.debug("POSIX: Unknown ACL scope %s on %s" %
+                                      (scope, path))
+                    return
+
+                if scope != posix1e.ACL_OTHER:
+                    aclentry.set(aclentry.get("scope"), qual)
+                entry.append(aclentry)
+
         if not HAS_ACLS:
             if entry.findall("ACL"):
                 self.logger.debug("POSIX: ACLs listed for %s but no pylibacl "
@@ -644,25 +690,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
         extra = []
         wrong = []
         for aclkey, perms in wanted.items():
-            if aclkey not in existing:
-                missing.append(self._acl2string(aclkey, perms))
-            elif existing[aclkey] != perms:
-                wrong.append((self._acl2string(aclkey, perms),
-                              self._acl2string(aclkey, existing[aclkey])))
-            if path == entry.get("name"):
-                atype, scope, qual = aclkey
-                aclentry = Bcfg2.Client.XML.Element("ACL", type=atype,
-                                                    perms=str(perms))
-                if scope == posix1e.ACL_USER:
-                    aclentry.set("scope", "user")
-                elif scope == posix1e.ACL_GROUP:
-                    aclentry.set("scope", "group")
-                else:
-                    self.logger.debug("POSIX: Unknown ACL scope %s on %s" %
-                                      (scope, path))
-                    continue
-                aclentry.set(aclentry.get("scope"), qual)
-                entry.append(aclentry)
+            _verify_acl(aclkey, perms)
 
         for aclkey, perms in existing.items():
             if aclkey not in wanted:
diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
index 58a3bbdfc..1a3b22506 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
@@ -160,7 +160,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
         """ Get a list of supplmentary groups that the user in the
         given entry is a member of """
         return [g for g in self.existing['POSIXGroup'].values()
-                if entry.get("name") in g[3] and g[0] != entry.get("group")]
+                if entry.get("name") in g[3] and g[0] != entry.get("group")
+                and self._in_managed_range('POSIXGroup', g[2])]
 
     def VerifyPOSIXUser(self, entry, _):
         """ Verify a POSIXUser entry """
diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py
index 20a172d3d..027d91c71 100644
--- a/src/lib/Bcfg2/Client/Tools/Systemd.py
+++ b/src/lib/Bcfg2/Client/Tools/Systemd.py
@@ -13,8 +13,6 @@ class Systemd(Bcfg2.Client.Tools.SvcTool):
     __handles__ = [('Service', 'systemd')]
     __req__ = {'Service': ['name', 'status']}
 
-    conflicts = ['Chkconfig']
-
     def get_svc_command(self, service, action):
         return "/bin/systemctl %s %s.service" % (action, service.get('name'))
 
diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py
index 4e8ac76a4..449503b55 100644
--- a/src/lib/Bcfg2/Client/Tools/VCS.py
+++ b/src/lib/Bcfg2/Client/Tools/VCS.py
@@ -165,12 +165,13 @@ class VCS(Bcfg2.Client.Tools.Tool):
 
     def Verifysvn(self, entry, _):
         """Verify svn repositories"""
+        # pylint: disable=E1101
         headrev = pysvn.Revision(pysvn.opt_revision_kind.head)
+        # pylint: enable=E1101
         client = pysvn.Client()
         try:
             cur_rev = str(client.info(entry.get('name')).revision.number)
-            server = client.info2(entry.get('sourceurl'),
-                                  headrev,
+            server = client.info2(entry.get('sourceurl'), headrev,
                                   recurse=False)
             if server:
                 server_rev = str(server[0][1].rev.number)
diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py
index 8bb87540c..7782581c1 100644
--- a/src/lib/Bcfg2/Client/Tools/YUM.py
+++ b/src/lib/Bcfg2/Client/Tools/YUM.py
@@ -612,34 +612,38 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
                 package_fail = True
                 stat['version_fail'] = True
                 # Just chose the first pkg for the error message
+                current_pkg = all_pkg_objs[0]
                 if virt_pkg:
                     provides = \
-                        [p for p in all_pkg_objs[0].provides
+                        [p for p in current_pkg.provides
                          if p[0] == entry.get("name")][0]
-                    entry.set('current_version', "%s:%s-%s" % provides[2])
+                    current_evr = provides[2]
                     self.logger.info(
                         "  %s: Wrong version installed.  "
                         "Want %s, but %s provides %s" %
                         (entry.get("name"),
                          nevra2string(nevra),
-                         nevra2string(all_pkg_objs[0]),
+                         nevra2string(current_pkg),
                          yum.misc.prco_tuple_to_string(provides)))
                 else:
-                    entry.set('current_version', "%s:%s-%s.%s" %
-                              (all_pkg_objs[0].epoch,
-                               all_pkg_objs[0].version,
-                               all_pkg_objs[0].release,
-                               all_pkg_objs[0].arch))
+                    current_evr = (current_pkg.epoch,
+                                   current_pkg.version,
+                                   current_pkg.release)
                     self.logger.info("  %s: Wrong version installed.  "
                                      "Want %s, but have %s" %
                                      (entry.get("name"),
                                       nevra2string(nevra),
-                                      nevra2string(all_pkg_objs[0])))
-                entry.set('version', "%s:%s-%s.%s" %
-                          (nevra.get('epoch', 'any'),
-                           nevra.get('version', 'any'),
-                           nevra.get('release', 'any'),
-                           nevra.get('arch', 'any')))
+                                      nevra2string(current_pkg)))
+                wanted_evr = (nevra.get('epoch', 'any'),
+                              nevra.get('version', 'any'),
+                              nevra.get('release', 'any'))
+                entry.set('current_version', "%s:%s-%s" % current_evr)
+                entry.set('version', "%s:%s-%s" % wanted_evr)
+                if yum.compareEVR(current_evr, wanted_evr) == 1:
+                    entry.set("package_fail_action", "downgrade")
+                else:
+                    entry.set("package_fail_action", "update")
+
                 qtext_versions.append("U(%s)" % str(all_pkg_objs[0]))
                 continue
 
@@ -912,6 +916,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
         install_pkgs = []
         gpg_keys = []
         upgrade_pkgs = []
+        downgrade_pkgs = []
         reinstall_pkgs = []
 
         def queue_pkg(pkg, inst, queue):
@@ -953,11 +958,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
                     if (not status.get('installed', False) and
                             Bcfg2.Options.setup.yum_install_missing):
                         queue_pkg(pkg, inst, install_pkgs)
-                    elif (status.get('version_fail', False) and
-                          Bcfg2.Options.setup.yum_fix_version):
-                        queue_pkg(pkg, inst, upgrade_pkgs)
-                    elif (status.get('verify_fail', False) and
-                          Bcfg2.Options.setup.yum_reinstall_broken):
+                    elif status.get('version_fail', False) and self.do_upgrade:
+                        if pkg.get("package_fail_action") == "downgrade":
+                            queue_pkg(pkg, inst, downgrade_pkgs)
+                        else:
+                            queue_pkg(pkg, inst, upgrade_pkgs)
+                    elif status.get('verify_fail', False) and self.do_reinst:
                         queue_pkg(pkg, inst, reinstall_pkgs)
                     else:
                         # Either there was no Install/Version/Verify
@@ -1019,6 +1025,19 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
                     self.logger.error("Error upgrading package %s: %s" %
                                       (pkg_arg, yume))
 
+        if len(downgrade_pkgs) > 0:
+            self.logger.info("Attempting to downgrade packages")
+
+            for inst in downgrade_pkgs:
+                pkg_arg = self.instance_status[inst].get('pkg').get('name')
+                self.logger.debug("Downgrading %s" % pkg_arg)
+                try:
+                    self.yumbase.downgrade(**build_yname(pkg_arg, inst))
+                except yum.Errors.YumBaseError:
+                    yume = sys.exc_info()[1]
+                    self.logger.error("Error downgrading package %s: %s" %
+                                      (pkg_arg, yume))
+
         if len(reinstall_pkgs) > 0:
             self.logger.info("Attempting to reinstall packages")
             for inst in reinstall_pkgs:
diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py
index 91d4ac5c6..4ba06abae 100644
--- a/src/lib/Bcfg2/Client/XML.py
+++ b/src/lib/Bcfg2/Client/XML.py
@@ -5,9 +5,29 @@
 # pylint: disable=E0611,W0611,W0613,C0103
 
 try:
-    from lxml.etree import Element, SubElement, XML, tostring
+    from lxml.etree import Element, SubElement, tostring, XMLParser
     from lxml.etree import XMLSyntaxError as ParseError
+    from lxml.etree import XML as _XML
+    from Bcfg2.Compat import wraps
     driver = 'lxml'
+
+    # libxml2 2.9.0+ doesn't parse 10M+ documents by default:
+    # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html
+    try:
+        _parser = XMLParser(huge_tree=True)
+    except TypeError:
+        _parser = XMLParser()
+
+    @wraps(_XML)
+    def XML(val, **kwargs):
+        """ unicode strings w/encoding declaration are not supported in
+        recent lxml.etree, so we try to read XML, and if it fails we try
+        encoding the string. """
+        kwargs.setdefault('parser', _parser)
+        try:
+            return _XML(val, **kwargs)
+        except ValueError:
+            return _XML(val.encode(), **kwargs)
 except ImportError:
     # lxml not available
     from xml.parsers.expat import ExpatError as ParseError
diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py
index 6c1dfdccb..12c9cdaa8 100644
--- a/src/lib/Bcfg2/Reporting/Collector.py
+++ b/src/lib/Bcfg2/Reporting/Collector.py
@@ -6,6 +6,7 @@ import time
 import threading
 
 # pylint: disable=E0611
+from lockfile import LockFailed, LockTimeout
 try:
     from lockfile.pidlockfile import PIDLockFile
     from lockfile import Error as PIDFileError
@@ -63,6 +64,8 @@ class ReportingCollector(object):
         bcfg2-admin"""
         self.terminate = None
         self.context = None
+        self.children = []
+        self.cleanup_threshold = 25
 
         if Bcfg2.Options.setup.debug:
             level = logging.DEBUG
@@ -106,12 +109,24 @@ class ReportingCollector(object):
         self.terminate = threading.Event()
         atexit.register(self.shutdown)
         self.context = daemon.DaemonContext(detach_process=True)
+        iter = 0
 
         if Bcfg2.Options.setup.daemon:
             self.logger.debug("Daemonizing")
             try:
                 self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon)
                 self.context.open()
+            except LockFailed:
+                self.logger.error("Failed to daemonize: %s" %
+                                  sys.exc_info()[1])
+                self.shutdown()
+                return
+            except LockTimeout:
+                self.logger.error("Failed to daemonize: "
+                                  "Failed to acquire lock on %s" %
+                                  self.setup['daemon'])
+                self.shutdown()
+                return
             except PIDFileError:
                 self.logger.error("Error writing pid file: %s" %
                                   sys.exc_info()[1])
@@ -128,6 +143,13 @@ class ReportingCollector(object):
                     continue
                 store_thread = ReportingStoreThread(interaction, self.storage)
                 store_thread.start()
+                self.children.append(store_thread)
+
+                iter += 1
+                if iter >= self.cleanup_threshold:
+                    self.reap_children()
+                    iter = 0
+
             except (SystemExit, KeyboardInterrupt):
                 self.logger.info("Shutting down")
                 self.shutdown()
@@ -147,3 +169,16 @@ class ReportingCollector(object):
                 pass
         if self.storage:
             self.storage.shutdown()
+
+    def reap_children(self):
+        """Join any non-live threads"""
+        newlist = []
+
+        self.logger.debug("Starting reap_children")
+        for child in self.children:
+            if child.isAlive():
+                newlist.append(child)
+            else:
+                child.join()
+                self.logger.debug("Joined child thread %s" % child.getName())
+        self.children = newlist
diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py
index 992687a85..b2d26d190 100644
--- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py
+++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py
@@ -14,6 +14,7 @@ from django.core import management
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.db.models import FieldDoesNotExist
 from django.core.cache import cache
+from django import db
 
 #Used by GetCurrentEntry
 import difflib
@@ -368,7 +369,12 @@ class DjangoORM(StorageBase):
             self._import_interaction(interaction)
         except:
             self.logger.error("Failed to import interaction: %s" %
-                              sys.exc_info()[1])
+                    traceback.format_exc().splitlines()[-1])
+        finally:
+            self.logger.info("%s: Closing database connection" %
+                             self.__class__.__name__)
+            db.close_connection()
+
 
     def validate(self):
         """Validate backend storage.  Should be called once when loaded"""
diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html
index 7edf3a949..ef6799c2b 100644
--- a/src/lib/Bcfg2/Reporting/templates/base.html
+++ b/src/lib/Bcfg2/Reporting/templates/base.html
@@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5
   <div style='clear:both'></div>
 </div><!-- document -->
   <div id="footer">
-    <span>Bcfg2 Version 1.3.3</span>
+    <span>Bcfg2 Version 1.3.4</span>
     </div>
 
 <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div>
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 4f51ebe87..9c22d17ac 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -19,7 +19,7 @@ import Bcfg2.Server.Statistics
 import Bcfg2.Server.FileMonitor
 from itertools import chain
 from Bcfg2.Server.Cache import Cache
-from Bcfg2.Compat import xmlrpclib  # pylint: disable=W0622
+from Bcfg2.Compat import xmlrpclib, wraps  # pylint: disable=W0622
 from Bcfg2.Server.Plugin.exceptions import *  # pylint: disable=W0401,W0614
 from Bcfg2.Server.Plugin.interfaces import *  # pylint: disable=W0401,W0614
 from Bcfg2.Server.Plugin import track_statistics
@@ -74,6 +74,24 @@ def sort_xml(node, key=None):
     node[:] = sorted_children
 
 
+def close_db_connection(func):
+    """ Decorator that closes the Django database connection at the end of
+    the function.  This should decorate any exposed function that
+    might open a database connection. """
+    @wraps(func)
+    def inner(self, *args, **kwargs):
+        """ The decorated function """
+        rv = func(self, *args, **kwargs)
+        if self._database_available:  # pylint: disable=W0212
+            from django import db
+            self.logger.debug("%s: Closing database connection" %
+                              threading.current_thread().name)
+            db.close_connection()
+        return rv
+
+    return inner
+
+
 class CoreInitError(Exception):
     """ Raised when the server core cannot be initialized. """
     pass
@@ -196,6 +214,12 @@ class Core(object):
         self.revision = '-1'
 
         atexit.register(self.shutdown)
+        #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly,
+        #: then :mod:`atexit` calls it *again*, so it gets called
+        #: twice.  This is potentially bad, so we use
+        #: :attr:`Bcfg2.Server.Core._running` as a flag to determine
+        #: if the core needs to be shutdown, and only do it once.
+        self._running = True
 
         #: Threading event to signal worker threads (e.g.,
         #: :attr:`fam_thread`) to shutdown
@@ -403,14 +427,22 @@ class Core(object):
 
     def shutdown(self):
         """ Perform plugin and FAM shutdown tasks. """
-        self.logger.info("Shutting down core...")
+        if not self._running:
+            self.logger.debug("%s: Core already shut down" % self.name)
+            return
+        self.logger.info("%s: Shutting down core..." % self.name)
         if not self.terminate.isSet():
             self.terminate.set()
-            self.fam.shutdown()
-            self.logger.info("FAM shut down")
-            for plugin in list(self.plugins.values()):
-                plugin.shutdown()
-            self.logger.info("All plugins shut down")
+        self._running = False
+        self.fam.shutdown()
+        self.logger.info("%s: FAM shut down" % self.name)
+        for plugin in list(self.plugins.values()):
+            plugin.shutdown()
+        self.logger.info("%s: All plugins shut down" % self.name)
+        if self._database_available:
+            from django import db
+            self.logger.info("%s: Closing database connection" % self.name)
+            db.close_connection()
 
     @property
     def metadata_cache_mode(self):
@@ -601,9 +633,10 @@ class Core(object):
                 del entry.attrib['realname']
                 return ret
             except:
-                self.logger.error("Failed binding entry %s:%s with altsrc %s" %
-                                  (entry.tag, entry.get('realname'),
-                                   entry.get('name')))
+                self.logger.error(
+                    "Failed binding entry %s:%s with altsrc %s: %s" %
+                    (entry.tag, entry.get('realname'), entry.get('name'),
+                     sys.exc_info()[1]))
                 entry.set('name', oldname)
                 self.logger.error("Falling back to %s:%s" %
                                   (entry.tag, entry.get('name')))
@@ -1052,6 +1085,7 @@ class Core(object):
 
     @exposed
     @track_statistics()
+    @close_db_connection
     def DeclareVersion(self, address, version):
         """ Declare the client version.
 
@@ -1074,6 +1108,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def GetProbes(self, address):
         """ Fetch probes for the client.
 
@@ -1099,6 +1134,7 @@ class Core(object):
                                 (client, err))
 
     @exposed
+    @close_db_connection
     def RecvProbeData(self, address, probedata):
         """ Receive probe data from clients.
 
@@ -1146,6 +1182,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def AssertProfile(self, address, profile):
         """ Set profile for a client.
 
@@ -1165,6 +1202,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def GetConfig(self, address):
         """ Build config for a client by calling
         :func:`BuildConfiguration`.
@@ -1184,6 +1222,7 @@ class Core(object):
             self.critical_error("Metadata consistency failure for %s" % client)
 
     @exposed
+    @close_db_connection
     def RecvStats(self, address, stats):
         """ Act on statistics upload with :func:`process_statistics`.
 
@@ -1199,6 +1238,7 @@ class Core(object):
         return True
 
     @exposed
+    @close_db_connection
     def GetDecisionList(self, address, mode):
         """ Get the decision list for the client with :func:`GetDecisions`.
 
diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
index b8eb06aa1..c4b34a469 100644
--- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
+++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
@@ -212,7 +212,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent):
     AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__
 
     def shutdown(self):
-        if self.notifier:
+        if self.started and self.notifier:
             self.notifier.stop()
     shutdown.__doc__ = Pseudo.shutdown.__doc__
 
diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
index 109ace61f..22c97a0fe 100644
--- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
+++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
@@ -123,12 +123,30 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
 
     @classmethod
     def Errors(cls):
-        return {"unknown-entry-type": "error",
+        return {"missing-elements": "error",
+                "unknown-entry-type": "error",
                 "unknown-entry-tag": "error",
                 "required-attrs-missing": "error",
                 "required-attr-format": "error",
                 "extra-attrs": "warning"}
 
+    def check_default_acl(self, path):
+        """ Check that a default ACL contains either no entries or minimum
+        required entries """
+        defaults = 0
+        if path.xpath("ACL[@type='default' and @scope='user' and @user='']"):
+            defaults += 1
+        if path.xpath("ACL[@type='default' and @scope='group' and @group='']"):
+            defaults += 1
+        if path.xpath("ACL[@type='default' and @scope='other']"):
+            defaults += 1
+        if defaults > 0 and defaults < 3:
+            self.LintError(
+                "missing-elements",
+                "A Path must have either no default ACLs or at"
+                " least default:user::, default:group:: and"
+                " default:other::")
+
     def check_packages(self):
         """ Check Packages sources for Source entries with missing
         attributes. """
@@ -172,7 +190,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
                                               rules.name))
 
     def check_bundles(self):
-        """ Check bundles for BoundPath entries with missing
+        """ Check bundles for BoundPath and BoundPackage entries with missing
         attrs. """
         if 'Bundler' not in self.core.plugins:
             return
@@ -192,6 +210,15 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
                             "required-attrs-missing",
                             "Path tags require either a 'name' or 'glob' "
                             "attribute: \n%s" % self.RenderXML(path))
+                # ensure that abstract Package tags have either name
+                # or group specified
+                for package in xdata.xpath("//Package"):
+                    if ('name' not in package.attrib and
+                        'group' not in package.attrib):
+                        self.LintError(
+                            "required-attrs-missing",
+                            "Package tags require either a 'name' or 'group' "
+                            "attribute: \n%s" % self.RenderXML(package))
 
     def check_entry(self, entry, filename):
         """ Generic entry check.
@@ -231,6 +258,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
                     required_attrs['major'] = is_device_mode
                     required_attrs['minor'] = is_device_mode
 
+            if tag == 'Path':
+                self.check_default_acl(entry)
+
             if tag == 'ACL' and 'scope' in required_attrs:
                 required_attrs[entry.get('scope')] = is_username
 
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 3ad78ade4..0b3f1e24d 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -90,6 +90,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
                 "xml-failed-to-parse": "error",
                 "xml-failed-to-read": "error",
                 "xml-failed-to-verify": "error",
+                "xinclude-does-not-exist": "error",
                 "input-output-error": "error"}
 
     def check_properties(self):
@@ -115,6 +116,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
         try:
             xdata = lxml.etree.parse(filename)
             if self.files is None:
+                self._expand_wildcard_xincludes(xdata)
                 xdata.xinclude()
             return xdata
         except (lxml.etree.XIncludeError, SyntaxError):
@@ -132,6 +134,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
                            "Failed to open file %s" % filename)
             return False
 
+    def _expand_wildcard_xincludes(self, xdata):
+        """ a lightweight version of
+        :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """
+        xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE
+        for el in xdata.findall('//' + xinclude):
+            name = el.get("href")
+            if name.startswith("/"):
+                fpath = name
+            else:
+                fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name)
+
+            # expand globs in xinclude, a bcfg2-specific extension
+            extras = glob.glob(fpath)
+            if not extras:
+                msg = "%s: %s does not exist, skipping: %s" % \
+                      (xdata.docinfo.URL, name, self.RenderXML(el))
+                if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE):
+                    self.logger.debug(msg)
+                else:
+                    self.LintError("xinclude-does-not-exist", msg)
+
+            parent = el.getparent()
+            parent.remove(el)
+            for extra in extras:
+                if extra != xdata.docinfo.URL:
+                    lxml.etree.SubElement(parent, xinclude, href=extra)
+
     def validate(self, filename, schemafile, schema=None):
         """ Validate a file against the given schema.
 
diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
index 04151d764..6383a3c99 100644
--- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
+++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py
@@ -10,7 +10,9 @@ import Bcfg2.Server.Lint
 
 try:
     import json
-except ImportError:
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
+except (ImportError, AttributeError):
     import simplejson as json
 
 
diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py
index 294963669..724b34d8d 100644
--- a/src/lib/Bcfg2/Server/MultiprocessingCore.py
+++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py
@@ -275,6 +275,7 @@ class ChildCore(Core):
     @exposed
     def GetConfig(self, client):
         """ Render the configuration for a client """
+        self.metadata.update_client_list()
         self.logger.debug("%s: Building configuration for %s" %
                           (self.name, client))
         return lxml.etree.tostring(self.BuildConfiguration(client))
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index aa8db2bc0..407e9df46 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -275,7 +275,8 @@ class PluginDatabaseModel(object):
     inherit from.  This is just a mixin; models must also inherit from
     django.db.models.Model to be valid Django models."""
 
-    class Meta:  # pylint: disable=C0111,W0232
+    class Meta(object):  # pylint: disable=W0232
+        """ Model metadata options """
         app_label = "Server"
 
 
diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py
index 622b69c79..c45d6fa84 100644
--- a/src/lib/Bcfg2/Server/Plugin/interfaces.py
+++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py
@@ -216,6 +216,10 @@ class Metadata(object):
         """
         raise NotImplementedError
 
+    def update_client_list(self):
+        """ Re-read the cached list of clients """
+        raise NotImplementedError
+
 
 class Connector(object):
     """ Connector plugins augment client metadata instances with
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 6ff256147..bf51ff678 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -686,7 +686,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 client = MetadataClientModel(hostname=client_name)
                 # pylint: enable=E1102
                 client.save()
-            self.clients = self.list_clients()
+            self.update_client_list()
             return client
         else:
             try:
@@ -739,7 +739,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                                       attribs, alias=True)
 
     def list_clients(self):
-        """ List all clients in client database """
+        """ List all clients in client database.
+
+        Making ``self.clients`` a property and reading the client list
+        dynamically from the database on every call to
+        ``self.clients`` can result in very high rates of database
+        reads, so we cache the ``list_clients()`` results to reduce
+        the database load.  When the database is in use, the client
+        list is reread periodically with
+        :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """
         if self._use_db:
             return set([c.hostname for c in MetadataClientModel.objects.all()])
         else:
@@ -790,7 +798,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 self.logger.warning(msg)
                 raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
             client.delete()
-            self.clients = self.list_clients()
+            self.update_client_list()
         else:
             return self._remove_xdata(self.clients_xml, "Client", client_name)
 
@@ -859,8 +867,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
             except KeyError:
                 self.clientgroups[clname] = [profile]
         self.states['clients.xml'] = True
-        if self._use_db:
-            self.clients = self.list_clients()
+        self.update_client_list()
 
     def _get_condition(self, element):
         """ Return a predicate that returns True if a client meets
@@ -1452,6 +1459,30 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         return True
     # pylint: enable=R0911,R0912
 
+    def update_client_list(self):
+        """ Re-read the client list from the database (if the database is in
+        use) """
+        if self._use_db:
+            self.logger.debug("Metadata: Re-reading client list from database")
+            old = set(self.clients)
+            self.clients = self.list_clients()
+            new = set(self.clients)
+            added = new - old
+            removed = old - new
+            self.logger.debug("Metadata: Added %s clients: %s" %
+                              (len(added), added))
+            self.logger.debug("Metadata: Removed %s clients: %s" %
+                              (len(removed), removed))
+            # we could do this with set.symmetric_difference(), but we
+            # want detailed numbers of added/removed clients for
+            # logging
+            for client in added.union(removed):
+                self.expire_cache(client)
+
+    def start_client_run(self, metadata):
+        """ Hook to reread client list if the database is in use """
+        self.update_client_list()
+
     def end_statistics(self, metadata):
         """ Hook to toggle clients in bootstrap mode """
         if self.auth.get(metadata.hostname,
diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py
index ba7baab11..c5fb46c97 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ohai.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py
@@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin
 
 try:
     import json
-except ImportError:
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
+except (ImportError, AttributeError):
     import simplejson as json
 
 PROBECODE = """#!/bin/sh
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index dba56eed2..c1d53a78e 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -72,6 +72,7 @@ class AptSource(Source):
     def read_files(self):
         bdeps = dict()
         bprov = dict()
+        self.essentialpkgs = set()
         depfnames = ['Depends', 'Pre-Depends']
         if self.recommended:
             depfnames.append('Recommends')
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 6a493c19d..3cfda9e9c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -90,7 +90,9 @@ try:
     import yum
     try:
         import json
-    except ImportError:
+        # py2.4 json library is structured differently
+        json.loads  # pylint: disable=W0104
+    except (ImportError, AttributeError):
         import simplejson as json
     HAS_YUM = True
 except ImportError:
@@ -354,8 +356,8 @@ class YumCollection(Collection):
                 self.__class__._helper = find_executable('bcfg2-yum-helper')
                 if not self.__class__._helper:
                     self.__class__._helper = "/usr/sbin/bcfg2-yum-helper"
-            # pylint: enable=W0212
-        return self._helper
+        return self.__class__._helper
+        # pylint: enable=W0212
 
     @property
     def use_yum(self):
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 553c16202..21d50ace6 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -51,8 +51,10 @@ def load_django_models():
 
 try:
     import json
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
     HAS_JSON = True
-except ImportError:
+except (ImportError, AttributeError):
     try:
         import simplejson as json
         HAS_JSON = True
diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py
index 87cee7029..04314218c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Properties.py
+++ b/src/lib/Bcfg2/Server/Plugins/Properties.py
@@ -13,8 +13,10 @@ from Bcfg2.Server.Plugin import PluginExecutionError
 
 try:
     import json
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
     HAS_JSON = True
-except ImportError:
+except (ImportError, AttributeError):
     try:
         import simplejson as json
         HAS_JSON = True
diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py
index 35d4cfa0a..ae82724f3 100644
--- a/src/lib/Bcfg2/version.py
+++ b/src/lib/Bcfg2/version.py
@@ -2,7 +2,7 @@
 
 import re
 
-__version__ = "1.3.3"
+__version__ = "1.3.4"
 
 
 class Bcfg2VersionInfo(tuple):  # pylint: disable=E0012,R0924