From cd7b0b3d40a5a340d5b47819f94a21c9faf23120 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 5 Sep 2012 14:36:15 -0400 Subject: added server-side client metadata object caching --- src/lib/Bcfg2/Server/Plugins/GroupPatterns.py | 12 ++++++++++-- src/lib/Bcfg2/Server/Plugins/Metadata.py | 17 +++++++++++++---- src/lib/Bcfg2/Server/Plugins/Probes.py | 16 ++++++++++++++-- src/lib/Bcfg2/Server/Plugins/PuppetENC.py | 10 ++++++++++ 4 files changed, 47 insertions(+), 8 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 837f47279..955a46c6c 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -81,14 +81,22 @@ class PatternMap(object): class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): __identifier__ = None - def __init__(self, filename, fam=None): + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, should_monitor=True) + self.core = core self.patterns = [] self.logger = logging.getLogger(self.__class__.__name__) def Index(self): Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() self.patterns = [] for entry in self.xdata.xpath('//GroupPattern'): try: @@ -125,7 +133,7 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = PatternFile(os.path.join(self.data, 'config.xml'), - fam=core.fam) + core=core) def get_additional_groups(self, metadata): return self.config.process_patterns(metadata.hostname) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 774c6b1ef..5d0b35835 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -729,6 +729,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for hdlr in self.handlers: aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(event.filename)) if hdlr(event): + # clear the entire cache when we get an event for any + # metadata file + self.core.metadata_cache.expire() try: proc = getattr(self, "_handle_%s_event" % aname) except AttributeError: @@ -750,7 +753,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.debug_log("Group %s set as nonexistent group %s" % (gname, group)) - def set_profile(self, client, profile, addresspair, force=False): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % @@ -888,6 +890,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet") client = client.lower() + + if client in self.core.metadata_cache: + return self.core.metadata_cache[client] + if client in self.aliases: client = self.aliases[client] @@ -967,9 +973,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if len(profiles) >= 1: profile = profiles[0] - return ClientMetadata(client, profile, groups, bundles, aliases, - addresses, categories, uuid, password, version, - self.query) + rv = ClientMetadata(client, profile, groups, bundles, aliases, + addresses, categories, uuid, password, version, + self.query) + if self.core.metadata_cache_mode == 'initial': + self.core.metadata_cache[client] = rv + return rv def get_all_group_names(self): all_groups = set() diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7f300ebe0..056521ce7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -2,10 +2,12 @@ import re import os import sys import time +import copy import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin +from Bcfg2.Compat import any try: from django.db import models @@ -35,8 +37,6 @@ except ImportError: except ImportError: has_yaml = False -import Bcfg2.Server.Plugin - if has_django: class ProbesDataModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): @@ -266,10 +266,22 @@ class Probes(Bcfg2.Server.Plugin.Probing, return self.probes.get_probe_data(meta) def ReceiveData(self, client, datalist): + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + if client.hostname in self.probedata: + olddata = copy.copy(self.probedata[client.hostname]) + else: + olddata = ClientProbeDataSet() + self.cgroups[client.hostname] = [] self.probedata[client.hostname] = ClientProbeDataSet() for data in datalist: self.ReceiveDataItem(client, data) + + if (self.core.metadata_cache_mode in ['cautious', 'aggressive'] and + (olddata.keys() != self.probedata[client.hostname].keys() or + any(olddata[p] != self.probedata[client.hostname][p] + for p in olddata.keys()))): + self.core.metadata_cache.expire(client.hostname) self.write_data(client) def ReceiveDataItem(self, client, data): diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 46182e9a2..341d63118 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -112,6 +112,16 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, separately; and b) when a single client's metadata is generated multiple times by separate templates """ self.cache = dict() + if self.core.metadata_cache_mode == 'aggressive': + # clear the metadata client cache if we're in aggressive + # mode, and produce a warning. PuppetENC really isn't + # compatible with aggressive mode, since we don't know + # when the output from a given ENC has changed, and thus + # can't invalidate the cache sanely. + self.logger.warning("PuppetENC is incompatible with aggressive " + "client metadata caching, try 'cautious' or " + "'initial' instead") + self.core.cache.expire() def end_statistics(self, metadata): self.end_client_run(self, metadata) -- cgit v1.2.3-1-g7c22