From cb8b6e95e010ae322cb6be9b79e1a89009f45948 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 5 Aug 2013 14:50:23 -0400 Subject: Multiprocessing: proxy RecvProbeData calls This proxies RecvProbeData calls to child cores to expire the probe cache. The probe data itself is not relayed, just the fact that there was probe data received from a given client. Fixes #129. --- src/lib/Bcfg2/Server/Plugins/Probes.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Probes.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 407cfc2d4..b58fbf715 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -266,12 +266,14 @@ class Probes(Bcfg2.Server.Plugin.Probing, hostname=client.hostname).exclude( group__in=self.cgroups[client.hostname]).delete() - def load_data(self): + def load_data(self, client=None): """ Load probe data from the appropriate backend (probed.xml or the database) """ if self._use_db: - return self._load_data_db() + return self._load_data_db(client) else: + # the XML backend doesn't support loading data for single + # clients, so it reloads all data return self._load_data_xml() def _load_data_xml(self): @@ -296,16 +298,23 @@ class Probes(Bcfg2.Server.Plugin.Probing, elif pdata.tag == 'Group': self.cgroups[client.get('name')].append(pdata.get('name')) - def _load_data_db(self): + def _load_data_db(self, client=None): """ Load probe data from the database """ self.probedata = {} self.cgroups = {} - for pdata in ProbesDataModel.objects.all(): + if client is None: + probedata = ProbesDataModel.objects.all() + groupdata = ProbesGroupsModel.objects.all() + else: + probedata = ProbesDataModel.objects.filter(hostname=client) + groupdata = ProbesGroupsModel.objects.filter(hostname=client) + + for pdata in probedata: if pdata.hostname not in self.probedata: self.probedata[pdata.hostname] = ClientProbeDataSet( timestamp=time.mktime(pdata.timestamp.timetuple())) self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in ProbesGroupsModel.objects.all(): + for pgroup in groupdata: if pgroup.hostname not in self.cgroups: self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) -- cgit v1.2.3-1-g7c22 From 6c86b70c58d05bd6d1e6e9630d9490e36dccfe92 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 5 Aug 2013 15:13:39 -0400 Subject: Probes: fixed unit test --- src/lib/Bcfg2/Server/Plugins/Probes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Probes.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index b58fbf715..8c552a90b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -270,7 +270,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, """ Load probe data from the appropriate backend (probed.xml or the database) """ if self._use_db: - return self._load_data_db(client) + return self._load_data_db(client=client) else: # the XML backend doesn't support loading data for single # clients, so it reloads all data -- cgit v1.2.3-1-g7c22 From 159b152fcaecbbd69ad3665f8dd00c37d81af4e4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 6 Aug 2013 14:26:44 -0400 Subject: Probes: expire metadata cache after loading data --- src/lib/Bcfg2/Server/Plugins/Probes.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins/Probes.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 8c552a90b..87688a804 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -298,6 +298,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, elif pdata.tag == 'Group': self.cgroups[client.get('name')].append(pdata.get('name')) + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + def _load_data_db(self, client=None): """ Load probe data from the database """ self.probedata = {} @@ -319,6 +322,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire(client) + @Bcfg2.Server.Plugin.track_statistics() def GetProbes(self, meta): return self.probes.get_probe_data(meta) -- cgit v1.2.3-1-g7c22 From 99c680e94132d5bf3110bd14bfabc9e407b1dae9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 6 Aug 2013 14:50:37 -0400 Subject: Probes: properly clear cache When reloading probe groups/data for a single client from the database, only clear data for that client, not for all clients. --- src/lib/Bcfg2/Server/Plugins/Probes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Probes.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 87688a804..bf59809f7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -303,12 +303,14 @@ class Probes(Bcfg2.Server.Plugin.Probing, def _load_data_db(self, client=None): """ Load probe data from the database """ - self.probedata = {} - self.cgroups = {} if client is None: + self.probedata = {} + self.cgroups = {} probedata = ProbesDataModel.objects.all() groupdata = ProbesGroupsModel.objects.all() else: + self.probedata.pop(client, None) + self.cgroups.pop(client, None) probedata = ProbesDataModel.objects.filter(hostname=client) groupdata = ProbesGroupsModel.objects.filter(hostname=client) -- cgit v1.2.3-1-g7c22 From 7e9787c947e99b68317f5420951a296cea858daa Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 7 Aug 2013 11:37:38 -0400 Subject: Plugin: added new Caching interface This gives a single unified interface for expiring caches, no matter the plugin. This will be particularly useful with the MultiprocessingCore, as certain calls must be dispatched to child processes to expire their caches. --- src/lib/Bcfg2/Server/Plugins/Probes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Probes.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index bf59809f7..b9f93052a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -181,14 +181,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: @@ -266,6 +268,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, hostname=client.hostname).exclude( group__in=self.cgroups[client.hostname]).delete() + def expire_cache(self, key=None): + self.load_data(client=key) + def load_data(self, client=None): """ Load probe data from the appropriate backend (probed.xml or the database) """ @@ -299,7 +304,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[client.get('name')].append(pdata.get('name')) if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.metadata_cache.expire() + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def _load_data_db(self, client=None): """ Load probe data from the database """ @@ -325,7 +330,8 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[pgroup.hostname].append(pgroup.group) if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.metadata_cache.expire(client) + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) @Bcfg2.Server.Plugin.track_statistics() def GetProbes(self, meta): -- cgit v1.2.3-1-g7c22