added server-side client metadata object caching

4 files changed, 47 insertions, 8 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
index 837f47279..955a46c6c 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -81,14 +81,22 @@ class PatternMap(object):
 class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked):
     __identifier__ = None
 
-    def __init__(self, filename, fam=None):
+    def __init__(self, filename, core=None):
+        try:
+            fam = core.fam
+        except AttributeError:
+            fam = None
         Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam,
                                                    should_monitor=True)
+        self.core = core
         self.patterns = []
         self.logger = logging.getLogger(self.__class__.__name__)
 
     def Index(self):
         Bcfg2.Server.Plugin.XMLFileBacked.Index(self)
+        if (self.core and
+            self.core.metadata_cache_mode in ['cautious', 'aggressive']):
+            self.core.metadata_cache.expire()
         self.patterns = []
         for entry in self.xdata.xpath('//GroupPattern'):
             try:
@@ -125,7 +133,7 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin,
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Connector.__init__(self)
         self.config = PatternFile(os.path.join(self.data, 'config.xml'),
-                                  fam=core.fam)
+                                  core=core)
 
     def get_additional_groups(self, metadata):
         return self.config.process_patterns(metadata.hostname)
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 774c6b1ef..5d0b35835 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -729,6 +729,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         for hdlr in self.handlers:
             aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(event.filename))
             if hdlr(event):
+                # clear the entire cache when we get an event for any
+                # metadata file
+                self.core.metadata_cache.expire()
                 try:
                     proc = getattr(self, "_handle_%s_event" % aname)
                 except AttributeError:
@@ -750,7 +753,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                         self.debug_log("Group %s set as nonexistent group %s" %
                                        (gname, group))
 
-
     def set_profile(self, client, profile, addresspair, force=False):
         """Set group parameter for provided client."""
         self.logger.info("Asserting client %s profile to %s" %
@@ -888,6 +890,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         if False in list(self.states.values()):
             raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet")
         client = client.lower()
+
+        if client in self.core.metadata_cache:
+            return self.core.metadata_cache[client]
+
         if client in self.aliases:
             client = self.aliases[client]
         
@@ -967,9 +973,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
             if len(profiles) >= 1:
                 profile = profiles[0]
 
-        return ClientMetadata(client, profile, groups, bundles, aliases,
-                              addresses, categories, uuid, password, version,
-                              self.query)
+        rv = ClientMetadata(client, profile, groups, bundles, aliases,
+                            addresses, categories, uuid, password, version,
+                            self.query)
+        if self.core.metadata_cache_mode == 'initial':
+            self.core.metadata_cache[client] = rv
+        return rv
 
     def get_all_group_names(self):
         all_groups = set()
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 7f300ebe0..056521ce7 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -2,10 +2,12 @@ import re
 import os
 import sys
 import time
+import copy
 import operator
 import lxml.etree
 import Bcfg2.Server
 import Bcfg2.Server.Plugin
+from Bcfg2.Compat import any
 
 try:
     from django.db import models
@@ -35,8 +37,6 @@ except ImportError:
     except ImportError:
         has_yaml = False
 
-import Bcfg2.Server.Plugin
-
 if has_django:
     class ProbesDataModel(models.Model,
                           Bcfg2.Server.Plugin.PluginDatabaseModel):
@@ -266,10 +266,22 @@ class Probes(Bcfg2.Server.Plugin.Probing,
         return self.probes.get_probe_data(meta)
 
     def ReceiveData(self, client, datalist):
+        if self.core.metadata_cache_mode in ['cautious', 'aggressive']:
+            if client.hostname in self.probedata:
+                olddata = copy.copy(self.probedata[client.hostname])
+            else:
+                olddata = ClientProbeDataSet()
+
         self.cgroups[client.hostname] = []
         self.probedata[client.hostname] = ClientProbeDataSet()
         for data in datalist:
             self.ReceiveDataItem(client, data)
+
+        if (self.core.metadata_cache_mode in ['cautious', 'aggressive'] and
+            (olddata.keys() != self.probedata[client.hostname].keys() or
+             any(olddata[p] != self.probedata[client.hostname][p]
+                 for p in olddata.keys()))):
+            self.core.metadata_cache.expire(client.hostname)
         self.write_data(client)
 
     def ReceiveDataItem(self, client, data):
diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py
index 46182e9a2..341d63118 100644
--- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py
+++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py
@@ -112,6 +112,16 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin,
         separately; and b) when a single client's metadata is
         generated multiple times by separate templates """
         self.cache = dict()
+        if self.core.metadata_cache_mode == 'aggressive':
+            # clear the metadata client cache if we're in aggressive
+            # mode, and produce a warning.  PuppetENC really isn't
+            # compatible with aggressive mode, since we don't know
+            # when the output from a given ENC has changed, and thus
+            # can't invalidate the cache sanely.
+            self.logger.warning("PuppetENC is incompatible with aggressive "
+                                "client metadata caching, try 'cautious' or "
+                                "'initial' instead")
+            self.core.cache.expire()
 
     def end_statistics(self, metadata):
         self.end_client_run(self, metadata)