Merge branch 'maint' into master

Signed-off-by: Sol Jerome <sol.jerome@gmail.com>

Conflicts:
	doc/appendix/guides/import-existing-ssh-keys.txt
	misc/bcfg2.spec
	src/lib/Bcfg2/Client/Tools/VCS.py
	src/lib/Bcfg2/Client/Tools/YUM.py
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Reporting/Collector.py
	src/lib/Bcfg2/Reporting/Storage/DjangoORM.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/__init__.py
	src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
	src/lib/Bcfg2/Server/Plugin/helpers.py
	src/lib/Bcfg2/Server/Plugins/Metadata.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
	src/lib/Bcfg2/settings.py
	src/sbin/bcfg2-crypt
	src/sbin/bcfg2-reports
	src/sbin/bcfg2-yum-helper
	testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py

6 files changed, 51 insertions, 11 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 6ff256147..bf51ff678 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -686,7 +686,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 client = MetadataClientModel(hostname=client_name)
                 # pylint: enable=E1102
                 client.save()
-            self.clients = self.list_clients()
+            self.update_client_list()
             return client
         else:
             try:
@@ -739,7 +739,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                                       attribs, alias=True)
 
     def list_clients(self):
-        """ List all clients in client database """
+        """ List all clients in client database.
+
+        Making ``self.clients`` a property and reading the client list
+        dynamically from the database on every call to
+        ``self.clients`` can result in very high rates of database
+        reads, so we cache the ``list_clients()`` results to reduce
+        the database load.  When the database is in use, the client
+        list is reread periodically with
+        :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """
         if self._use_db:
             return set([c.hostname for c in MetadataClientModel.objects.all()])
         else:
@@ -790,7 +798,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 self.logger.warning(msg)
                 raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
             client.delete()
-            self.clients = self.list_clients()
+            self.update_client_list()
         else:
             return self._remove_xdata(self.clients_xml, "Client", client_name)
 
@@ -859,8 +867,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
             except KeyError:
                 self.clientgroups[clname] = [profile]
         self.states['clients.xml'] = True
-        if self._use_db:
-            self.clients = self.list_clients()
+        self.update_client_list()
 
     def _get_condition(self, element):
         """ Return a predicate that returns True if a client meets
@@ -1452,6 +1459,30 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         return True
     # pylint: enable=R0911,R0912
 
+    def update_client_list(self):
+        """ Re-read the client list from the database (if the database is in
+        use) """
+        if self._use_db:
+            self.logger.debug("Metadata: Re-reading client list from database")
+            old = set(self.clients)
+            self.clients = self.list_clients()
+            new = set(self.clients)
+            added = new - old
+            removed = old - new
+            self.logger.debug("Metadata: Added %s clients: %s" %
+                              (len(added), added))
+            self.logger.debug("Metadata: Removed %s clients: %s" %
+                              (len(removed), removed))
+            # we could do this with set.symmetric_difference(), but we
+            # want detailed numbers of added/removed clients for
+            # logging
+            for client in added.union(removed):
+                self.expire_cache(client)
+
+    def start_client_run(self, metadata):
+        """ Hook to reread client list if the database is in use """
+        self.update_client_list()
+
     def end_statistics(self, metadata):
         """ Hook to toggle clients in bootstrap mode """
         if self.auth.get(metadata.hostname,
diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py
index ba7baab11..c5fb46c97 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ohai.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py
@@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin
 
 try:
     import json
-except ImportError:
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
+except (ImportError, AttributeError):
     import simplejson as json
 
 PROBECODE = """#!/bin/sh
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index dba56eed2..c1d53a78e 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -72,6 +72,7 @@ class AptSource(Source):
     def read_files(self):
         bdeps = dict()
         bprov = dict()
+        self.essentialpkgs = set()
         depfnames = ['Depends', 'Pre-Depends']
         if self.recommended:
             depfnames.append('Recommends')
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 6a493c19d..3cfda9e9c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -90,7 +90,9 @@ try:
     import yum
     try:
         import json
-    except ImportError:
+        # py2.4 json library is structured differently
+        json.loads  # pylint: disable=W0104
+    except (ImportError, AttributeError):
         import simplejson as json
     HAS_YUM = True
 except ImportError:
@@ -354,8 +356,8 @@ class YumCollection(Collection):
                 self.__class__._helper = find_executable('bcfg2-yum-helper')
                 if not self.__class__._helper:
                     self.__class__._helper = "/usr/sbin/bcfg2-yum-helper"
-            # pylint: enable=W0212
-        return self._helper
+        return self.__class__._helper
+        # pylint: enable=W0212
 
     @property
     def use_yum(self):
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 553c16202..21d50ace6 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -51,8 +51,10 @@ def load_django_models():
 
 try:
     import json
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
     HAS_JSON = True
-except ImportError:
+except (ImportError, AttributeError):
     try:
         import simplejson as json
         HAS_JSON = True
diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py
index 87cee7029..04314218c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Properties.py
+++ b/src/lib/Bcfg2/Server/Plugins/Properties.py
@@ -13,8 +13,10 @@ from Bcfg2.Server.Plugin import PluginExecutionError
 
 try:
     import json
+    # py2.4 json library is structured differently
+    json.loads  # pylint: disable=W0104
     HAS_JSON = True
-except ImportError:
+except (ImportError, AttributeError):
     try:
         import simplejson as json
         HAS_JSON = True