From 2217fa6295070f137988006c4bb00d25dfc0cb5e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 15 Jul 2013 15:32:21 -0400 Subject: Read-only yum cache This makes the yum cache read-only so that bcfg2-yum-helper cannot update the cache on the fly, which should help avoid locking issues with the yum caches that can cause client runs to fail. It also makes the Packages plugin behave more consistently, since use of yum libraries won't cause the cache to be refreshed at random times on the fly, but rather more predictably as with the Apt cache or the yum cache without using yum libraries. Unlike those two cases, though, the caches will not all be downloaded initially, but rather opportunistically as needed. In order for this to work, the Bcfg2 server must not run as root. Root ignores the 'w' permissions bit, so the cache cannot be made read-only. --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 85 ++++++++++++++++++++--- 1 file changed, 75 insertions(+), 10 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/__init__.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f93bd0932..aea447520 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,8 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ + MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -22,6 +23,52 @@ APT_CONFIG_DEFAULT = \ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +class OnDemandDict(MutableMapping): + """ This maps a set of keys to a set of value-getting functions; + the values are populated on-the-fly by the functions as the values + are needed (and not before). This is used by + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`; + see the docstring for that function for details on why. + + Unlike a dict, you should not specify values for for the righthand + side of this mapping, but functions that get values. E.g.: + + .. code-block:: python + + d = OnDemandDict(foo=load_foo, + bar=lambda: "bar"); + """ + + def __init__(self, **getters): + self._values = dict() + self._getters = dict(**getters) + + def __getitem__(self, key): + if key not in self._values: + self._values[key] = self._getters[key]() + return self._values[key] + + def __setitem__(self, key, getter): + self._getters[key] = getter + + def __delitem__(self, key): + del self._values[key] + del self._getters[key] + + def __len__(self): + return len(self._getters) + + def __iter__(self): + return iter(self._getters.keys()) + + def __str__(self): + rv = dict(self._values) + for key in self._getters.keys(): + if key not in rv: + rv[key] = 'unknown' + return str(rv) + + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -535,20 +582,38 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def get_additional_data(self, metadata): """ Return additional data for the given client. This will be - a dict containing a single key, ``sources``, whose value is a - list of data returned from - :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, - namely, a list of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` - data. + an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` + containing two keys: + + * ``sources``, whose value is a list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data; and + * ``get_config``, whose value is the + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` + function, which can be used to get the Packages config for + other systems. + + This uses an OnDemandDict instead of just a normal dict + because loading a source collection can be a fairly + time-consuming process, particularly for the first time. As a + result, when all metadata objects are built at once (such as + after the server is restarted, or far more frequently if + Metadata caching is disabled), this function would be a major + bottleneck if we tried to build all collections at the same + time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ - collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - get_config=self.get_config) + def get_sources(): + return self.get_collection(metadata).get_additional_data + + return OnDemandDict( + sources=get_sources, + get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in -- cgit v1.2.3-1-g7c22 From 4b0934821864538205dabb97974172903f68ba3f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 15 Jul 2013 16:16:19 -0400 Subject: Packages: added missing docstring --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/__init__.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index aea447520..a021cda13 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -609,6 +609,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ def get_sources(): + """ getter for the 'sources' key of the OnDemandDict + returned by this function. This delays calling + get_collection() until it's absolutely necessary. """ return self.get_collection(metadata).get_additional_data return OnDemandDict( -- cgit v1.2.3-1-g7c22 From 7dca37efeaff708fd6db30f498228b94bfafbe8e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 08:55:47 -0400 Subject: Packages: better stringification/representation of OnDemandDict --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/__init__.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index a021cda13..cd1bb70b0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -61,7 +61,7 @@ class OnDemandDict(MutableMapping): def __iter__(self): return iter(self._getters.keys()) - def __str__(self): + def __repr__(self): rv = dict(self._values) for key in self._getters.keys(): if key not in rv: -- cgit v1.2.3-1-g7c22