From 7c274b35552e631d7b5359b13192e7463eace645 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 31 Jul 2013 09:11:08 -0400 Subject: Yum: record helper location in class variable so it's only detected once --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 48c5b1f65..9ec7ac122 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -262,6 +262,8 @@ class YumCollection(Collection): .. private-include: _add_gpg_instances, _get_pulp_consumer """ + _helper = None + #: Options that are included in the [packages:yum] section of the #: config but that should not be included in the temporary #: yum.conf we write out @@ -280,7 +282,6 @@ class YumCollection(Collection): #: external commands self.cmd = Executor() - self._helper = None if self.use_yum: #: Define a unique cache file for this collection to use #: for cached yum metadata -- cgit v1.2.3-1-g7c22 From c3b296c7d29c8c6cb0a81a5ad3ac44eab5ee33f1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 31 Jul 2013 15:42:21 -0400 Subject: Packages: reduce source_from_xml() calls on startup --- src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 332f0bbab..c47e18201 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -88,13 +88,12 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :type event: Bcfg2.Server.FileMonitor.Event :returns: None """ - Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if event and event.filename != self.name: for fpath in self.extras: if fpath == os.path.abspath(event.filename): self.parsed.add(fpath) break - + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if self.loaded: self.logger.info("Reloading Packages plugin") self.pkg_obj.Reload() @@ -111,10 +110,11 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] - for xsource in self.xdata.findall('.//Source'): - source = self.source_from_xml(xsource) - if source is not None: - self.entries.append(source) + if self.loaded: + for xsource in self.xdata.findall('.//Source'): + source = self.source_from_xml(xsource) + if source is not None: + self.entries.append(source) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` -- cgit v1.2.3-1-g7c22 From 8b0ee3a67064bf810b3e6621ce128853eaf8a6f0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 1 Aug 2013 08:51:34 -0400 Subject: Yum: Fixed class-scope variable assignments --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 9ec7ac122..b26fb6870 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -317,7 +317,8 @@ class YumCollection(Collection): self.logger.error("Could not create Pulp consumer " "cert directory at %s: %s" % (certdir, err)) - self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) + self.__class__.pulp_cert_set = PulpCertificateSet(certdir, + self.fam) @property def disableMetaData(self): @@ -353,15 +354,16 @@ class YumCollection(Collection): the default location. """ if not self._helper: try: - self._helper = self.setup.cfp.get("packages:yum", "helper") + self.__class__._helper = self.setup.cfp.get("packages:yum", + "helper") except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") self.cmd.run(['bcfg2-yum-helper']) - self._helper = 'bcfg2-yum-helper' + self.__class__._helper = 'bcfg2-yum-helper' except OSError: - self._helper = "/usr/sbin/bcfg2-yum-helper" + self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" return self._helper @property -- cgit v1.2.3-1-g7c22 From 9c86ddc5cdd166ef416b7fa2fab6f883f70e4a66 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 1 Aug 2013 09:15:08 -0400 Subject: Yum: suppress warning about class variable assignment --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index b26fb6870..4187f2812 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -353,6 +353,7 @@ class YumCollection(Collection): forking, but apparently not); finally we check in /usr/sbin, the default location. """ if not self._helper: + # pylint: disable=W0212 try: self.__class__._helper = self.setup.cfp.get("packages:yum", "helper") @@ -364,6 +365,7 @@ class YumCollection(Collection): self.__class__._helper = 'bcfg2-yum-helper' except OSError: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" + # pylint: enable=W0212 return self._helper @property -- cgit v1.2.3-1-g7c22 From e1f045ff3c56b09ff06e11e6d4f9677bf63d051f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 5 Aug 2013 13:44:57 -0400 Subject: Yum: better errors when yum helper output isn't valid JSON --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4187f2812..66f8e9dbe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -946,9 +946,14 @@ class YumCollection(Collection): try: return json.loads(result.stdout) except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: error reading bcfg2-yum-helper " - "output: %s" % err) + if result.stdout: + err = sys.exc_info()[1] + self.logger.error("Packages: Error reading bcfg2-yum-helper " + "output: %s" % err) + self.logger.error("Packages: bcfg2-yum-helper output: %s" % + result.stdout) + else: + self.logger.error("Packages: No bcfg2-yum-helper output") raise def setup_data(self, force_update=False): -- cgit v1.2.3-1-g7c22 From cb8b6e95e010ae322cb6be9b79e1a89009f45948 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 5 Aug 2013 14:50:23 -0400 Subject: Multiprocessing: proxy RecvProbeData calls This proxies RecvProbeData calls to child cores to expire the probe cache. The probe data itself is not relayed, just the fact that there was probe data received from a given client. Fixes #129. --- src/lib/Bcfg2/Server/Plugins/Probes.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 407cfc2d4..b58fbf715 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -266,12 +266,14 @@ class Probes(Bcfg2.Server.Plugin.Probing, hostname=client.hostname).exclude( group__in=self.cgroups[client.hostname]).delete() - def load_data(self): + def load_data(self, client=None): """ Load probe data from the appropriate backend (probed.xml or the database) """ if self._use_db: - return self._load_data_db() + return self._load_data_db(client) else: + # the XML backend doesn't support loading data for single + # clients, so it reloads all data return self._load_data_xml() def _load_data_xml(self): @@ -296,16 +298,23 @@ class Probes(Bcfg2.Server.Plugin.Probing, elif pdata.tag == 'Group': self.cgroups[client.get('name')].append(pdata.get('name')) - def _load_data_db(self): + def _load_data_db(self, client=None): """ Load probe data from the database """ self.probedata = {} self.cgroups = {} - for pdata in ProbesDataModel.objects.all(): + if client is None: + probedata = ProbesDataModel.objects.all() + groupdata = ProbesGroupsModel.objects.all() + else: + probedata = ProbesDataModel.objects.filter(hostname=client) + groupdata = ProbesGroupsModel.objects.filter(hostname=client) + + for pdata in probedata: if pdata.hostname not in self.probedata: self.probedata[pdata.hostname] = ClientProbeDataSet( timestamp=time.mktime(pdata.timestamp.timetuple())) self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in ProbesGroupsModel.objects.all(): + for pgroup in groupdata: if pgroup.hostname not in self.cgroups: self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) -- cgit v1.2.3-1-g7c22 From 6c86b70c58d05bd6d1e6e9630d9490e36dccfe92 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 5 Aug 2013 15:13:39 -0400 Subject: Probes: fixed unit test --- src/lib/Bcfg2/Server/Plugins/Probes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index b58fbf715..8c552a90b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -270,7 +270,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, """ Load probe data from the appropriate backend (probed.xml or the database) """ if self._use_db: - return self._load_data_db(client) + return self._load_data_db(client=client) else: # the XML backend doesn't support loading data for single # clients, so it reloads all data -- cgit v1.2.3-1-g7c22 From 54d86177513c26f52e4aa7947e5bbc8ba91969ff Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 5 Aug 2013 19:15:37 -0400 Subject: Metadata: Don't update XML on gratuitous profile update Check to see if the profile that is being set by set_profile exactly matches the existing profile list. If it does, then avoid writing out a new clients.xml. This simple optimization reduces the amount of clients.xml rewriting that occurs if you have a bunch of clients running bcfg2 -p at the same time (for example, during a cluster rebuild). --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index e8962d707..973acb89a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -980,9 +980,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, profiles = [g for g in self.clientgroups[client] if g in self.groups and self.groups[g].is_profile] - self.logger.info("Changing %s profile from %s to %s" % + if profiles != [profile]: + self.logger.info("Changing %s profile from %s to %s" % (client, profiles, profile)) - self.update_client(client, dict(profile=profile)) + self.update_client(client, dict(profile=profile)) if client in self.clientgroups: for prof in profiles: self.clientgroups[client].remove(prof) -- cgit v1.2.3-1-g7c22 From 8b6a550d6dfa086935bf695ded1cdad89a383ff4 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 5 Aug 2013 19:29:25 -0400 Subject: Make pylint happy --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 973acb89a..512090ac5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -982,7 +982,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if g in self.groups and self.groups[g].is_profile] if profiles != [profile]: self.logger.info("Changing %s profile from %s to %s" % - (client, profiles, profile)) + (client, profiles, profile)) self.update_client(client, dict(profile=profile)) if client in self.clientgroups: for prof in profiles: -- cgit v1.2.3-1-g7c22 From a96a3301d5a2b0630795709c00e80f938fed10a4 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 5 Aug 2013 19:55:56 -0400 Subject: Found a stray write that should be part of the new client case --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 512090ac5..0e2277239 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1005,8 +1005,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.add_client(client, dict(profile=profile)) self.clients.append(client) self.clientgroups[client] = [profile] - if not self._use_db: - self.clients_xml.write() + if not self._use_db: + self.clients_xml.write() def set_version(self, client, version): """Set version for provided client.""" -- cgit v1.2.3-1-g7c22 From 936b8aa2b6329ddf45c1032171c16d0eaea99b37 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 5 Aug 2013 21:19:41 -0400 Subject: Make updating the structures conditional on profile changing Based on the expectations of the tests, I am reasonably confident that updating the in memory structures is logically part of changing the client's profile so I put it in the if block --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 0e2277239..e62b3f77f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -984,12 +984,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.info("Changing %s profile from %s to %s" % (client, profiles, profile)) self.update_client(client, dict(profile=profile)) - if client in self.clientgroups: - for prof in profiles: - self.clientgroups[client].remove(prof) - self.clientgroups[client].append(profile) - else: - self.clientgroups[client] = [profile] + if client in self.clientgroups: + for prof in profiles: + self.clientgroups[client].remove(prof) + self.clientgroups[client].append(profile) + else: + self.clientgroups[client] = [profile] else: self.logger.info("Creating new client: %s, profile %s" % (client, profile)) -- cgit v1.2.3-1-g7c22 From 159b152fcaecbbd69ad3665f8dd00c37d81af4e4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 6 Aug 2013 14:26:44 -0400 Subject: Probes: expire metadata cache after loading data --- src/lib/Bcfg2/Server/Plugins/Probes.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 8c552a90b..87688a804 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -298,6 +298,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, elif pdata.tag == 'Group': self.cgroups[client.get('name')].append(pdata.get('name')) + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + def _load_data_db(self, client=None): """ Load probe data from the database """ self.probedata = {} @@ -319,6 +322,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire(client) + @Bcfg2.Server.Plugin.track_statistics() def GetProbes(self, meta): return self.probes.get_probe_data(meta) -- cgit v1.2.3-1-g7c22 From 99c680e94132d5bf3110bd14bfabc9e407b1dae9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 6 Aug 2013 14:50:37 -0400 Subject: Probes: properly clear cache When reloading probe groups/data for a single client from the database, only clear data for that client, not for all clients. --- src/lib/Bcfg2/Server/Plugins/Probes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 87688a804..bf59809f7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -303,12 +303,14 @@ class Probes(Bcfg2.Server.Plugin.Probing, def _load_data_db(self, client=None): """ Load probe data from the database """ - self.probedata = {} - self.cgroups = {} if client is None: + self.probedata = {} + self.cgroups = {} probedata = ProbesDataModel.objects.all() groupdata = ProbesGroupsModel.objects.all() else: + self.probedata.pop(client, None) + self.cgroups.pop(client, None) probedata = ProbesDataModel.objects.filter(hostname=client) groupdata = ProbesGroupsModel.objects.filter(hostname=client) -- cgit v1.2.3-1-g7c22 From 4bf1c82868357e5c357e0b656419af37a99a8545 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Tue, 6 Aug 2013 16:01:12 -0400 Subject: Get profile by building metadata instead of guessing Rather than doing some ad-hoc lookups of internal data structures stpierre suggested that it'd be better to use the normal metadata build procedures. This implements that and adjusts the tests. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index e62b3f77f..cc0456334 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -978,18 +978,21 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - profiles = [g for g in self.clientgroups[client] - if g in self.groups and self.groups[g].is_profile] - if profiles != [profile]: + metadata = self.core.build_metadata(client) + if metadata.profile != profile: self.logger.info("Changing %s profile from %s to %s" % - (client, profiles, profile)) + (client, metadata.profile, profile)) self.update_client(client, dict(profile=profile)) if client in self.clientgroups: - for prof in profiles: - self.clientgroups[client].remove(prof) + if metadata.profile in self.clientgroups[client]: + self.clientgroups[client].remove(metadata.profile) self.clientgroups[client].append(profile) else: self.clientgroups[client] = [profile] + else: + self.logger.debug( + "Ignoring %s request to change profile from %s to %s" + % (client, metadata.profile, profile)) else: self.logger.info("Creating new client: %s, profile %s" % (client, profile)) -- cgit v1.2.3-1-g7c22 From 441a96457ba55529c0316a7459dcb295988824b0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 7 Aug 2013 11:23:03 -0400 Subject: Packages: cache group and package selections --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 37 ++++++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index cd1bb70b0..470a52bbc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -157,8 +157,21 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() - # pylint: enable=C0301 + #: groupcache caches group lookups. It maps Collections (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of package groups, and thence to the packages + #: indicated by those groups. + self.groupcache = dict() + + #: pkgcache caches complete package sets. It maps Collections + #: (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of initial packages, and thence to the final + #: (complete) package selections resolved from the initial + #: packages + self.pkgcache = dict() + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def set_debug(self, debug): @@ -402,14 +415,24 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for el in to_remove: el.getparent().remove(el) - gpkgs = collection.get_groups(groups) - for pkgs in gpkgs.values(): + groups.sort() + # check for this set of groups in the group cache + gkey = hash(tuple(groups)) + if gkey not in self.groupcache[collection.cachekey]: + self.groupcache[collection.cachekey][gkey] = \ + collection.get_groups(groups) + for pkgs in self.groupcache[collection.cachekey][gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) - packages, unknown = collection.complete(base) + # check for this set of packages in the package cache + pkey = hash(tuple(base)) + if pkey not in self.pkgcache[collection.cachekey]: + self.pkgcache[collection.cachekey][pkey] = \ + collection.complete(base) + packages, unknown = self.pkgcache[collection.cachekey][pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -462,9 +485,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.disableMetaData: collection.setup_data(force_update) - # clear Collection caches + # clear Collection and package caches self.clients = dict() self.collections = dict() + self.groupcache = dict() + self.pkgcache = dict() for source in self.sources.entries: cachefiles.add(source.cachefile) @@ -578,6 +603,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): -- cgit v1.2.3-1-g7c22 From 7e9787c947e99b68317f5420951a296cea858daa Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 7 Aug 2013 11:37:38 -0400 Subject: Plugin: added new Caching interface This gives a single unified interface for expiring caches, no matter the plugin. This will be particularly useful with the MultiprocessingCore, as certain calls must be dispatched to child processes to expire their caches. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 7 ++++++- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 5 +++++ src/lib/Bcfg2/Server/Plugins/Probes.py | 12 +++++++++--- src/lib/Bcfg2/Server/Plugins/PuppetENC.py | 2 +- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 5 +++++ 5 files changed, 26 insertions(+), 5 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index cc0456334..03323d64b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -487,6 +487,7 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" @@ -495,6 +496,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) self.watch_clients = watch_clients @@ -934,13 +936,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[gname] self.states['groups.xml'] = True + def expire_cache(self, key=None): + self.core.metadata_cache.expire(key) + def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.core.metadata_cache.expire() + self.expire_cache() event_handler(event) if False not in list(self.states.values()) and self.debug_flag: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 470a52bbc..3e4fb33ec 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -70,6 +70,7 @@ class OnDemandDict(MutableMapping): class Packages(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -94,6 +95,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -458,6 +460,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True + def expire_cache(self, _=None): + self.Reload() + def _load_config(self, force_update=False): """ Load the configuration data and setup sources diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index bf59809f7..b9f93052a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -181,14 +181,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: @@ -266,6 +268,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, hostname=client.hostname).exclude( group__in=self.cgroups[client.hostname]).delete() + def expire_cache(self, key=None): + self.load_data(client=key) + def load_data(self, client=None): """ Load probe data from the appropriate backend (probed.xml or the database) """ @@ -299,7 +304,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[client.get('name')].append(pdata.get('name')) if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.metadata_cache.expire() + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def _load_data_db(self, client=None): """ Load probe data from the database """ @@ -325,7 +330,8 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[pgroup.hostname].append(pgroup.group) if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.metadata_cache.expire(client) + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) @Bcfg2.Server.Plugin.track_statistics() def GetProbes(self, meta): diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 801e7006d..072f3f7e7 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -127,7 +127,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.warning("PuppetENC is incompatible with aggressive " "client metadata caching, try 'cautious' or " "'initial' instead") - self.core.cache.expire() + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def end_statistics(self, metadata): self.end_client_run(self, metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index d8b3104b7..2deea5f07 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -92,6 +92,7 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -125,6 +126,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -149,6 +151,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, HostKeyEntrySet(keypattern, self.data) self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk + def expire_cache(self, key=None): + self.__skn = False + def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: -- cgit v1.2.3-1-g7c22 From 4b09a72355d6fea244ad6b8dcfb2fd151a5ada6b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 7 Aug 2013 13:22:14 -0400 Subject: MultiprocessingCore: added a way to dispatch RMI calls to child processes --- src/lib/Bcfg2/Server/Plugins/Guppy.py | 1 + src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 3 +++ 2 files changed, 4 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index 4f2601f15..3c9b8a459 100644 --- a/src/lib/Bcfg2/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -37,6 +37,7 @@ class Guppy(Bcfg2.Server.Plugin.Plugin): experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable', 'Disable'] + __child_rmi__ = __rmi__[:] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 3e4fb33ec..4096153a3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -93,6 +93,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] + __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ + [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Caching.__init__(self) -- cgit v1.2.3-1-g7c22