diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins/Packages')
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 4 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 12 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 169 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 144 |
4 files changed, 249 insertions, 80 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index b25cb0fc4..39c51f351 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -614,6 +614,10 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.filter_unknown(unknown) return packages, unknown + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, + list.__repr__(self)) + def get_collection_class(source_type): """ Given a source type, determine the class of Collection object diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 332f0bbab..c47e18201 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -88,13 +88,12 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :type event: Bcfg2.Server.FileMonitor.Event :returns: None """ - Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if event and event.filename != self.name: for fpath in self.extras: if fpath == os.path.abspath(event.filename): self.parsed.add(fpath) break - + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if self.loaded: self.logger.info("Reloading Packages plugin") self.pkg_obj.Reload() @@ -111,10 +110,11 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] - for xsource in self.xdata.findall('.//Source'): - source = self.source_from_xml(xsource) - if source is not None: - self.entries.append(source) + if self.loaded: + for xsource in self.xdata.findall('.//Source'): + source = self.source_from_xml(xsource) + if source is not None: + self.entries.append(source) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index bb7caab0d..f038ec9c0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,13 +53,15 @@ The Yum Backend import os import re import sys +import time import copy import errno import socket import logging import lxml.etree -from subprocess import Popen, PIPE import Bcfg2.Server.Plugin +from lockfile import FileLock +from Bcfg2.Utils import Executor # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -85,7 +87,9 @@ try: import yum try: import json - except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 + except (ImportError, AttributeError): import simplejson as json HAS_YUM = True except ImportError: @@ -102,9 +106,6 @@ FL = '{http://linux.duke.edu/metadata/filelists}' PULPSERVER = None PULPCONFIG = None -#: The path to bcfg2-yum-helper -HELPER = None - def _setup_pulp(setup): """ Connect to a Pulp server and pass authentication credentials. @@ -263,6 +264,8 @@ class YumCollection(Collection): .. private-include: _add_gpg_instances, _get_pulp_consumer """ + _helper = None + #: Options that are included in the [packages:yum] section of the #: config but that should not be included in the temporary #: yum.conf we write out @@ -277,18 +280,25 @@ class YumCollection(Collection): debug=debug) self.keypath = os.path.join(self.cachepath, "keys") + #: A :class:`Bcfg2.Utils.Executor` object to use to run + #: external commands + self.cmd = Executor() + if self.use_yum: #: Define a unique cache file for this collection to use #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) #: The path to the server-side config file used when #: resolving packages with the Python yum libraries self.cfgfile = os.path.join(self.cachefile, "yum.conf") - self.write_config() + + if not os.path.exists(self.cachefile): + self.debug_log("Creating common cache %s" % self.cachefile) + os.mkdir(self.cachefile) + if not self.disableMetaData: + self.setup_data() else: self.cachefile = None @@ -309,7 +319,28 @@ class YumCollection(Collection): self.logger.error("Could not create Pulp consumer " "cert directory at %s: %s" % (certdir, err)) - self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) + self.__class__.pulp_cert_set = PulpCertificateSet(certdir, + self.fam) + + @property + def disableMetaData(self): + """ Report whether or not metadata processing is enabled. + This duplicates code in Packages/__init__.py, and can probably + be removed in Bcfg2 1.4 when we have a module-level setup + object. """ + if self.setup is None: + return True + try: + return not self.setup.cfp.getboolean("packages", "resolver") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return False + except ValueError: + # for historical reasons we also accept "enabled" and + # "disabled" + return self.setup.cfp.get( + "packages", + "metadata", + default="enabled").lower() == "disabled" @property def __package_groups__(self): @@ -323,20 +354,21 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ - global HELPER - if not HELPER: + if not self._helper: + # pylint: disable=W0212 try: - HELPER = self.setup.cfp.get("packages:yum", "helper") + self.__class__._helper = self.setup.cfp.get("packages:yum", + "helper") except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") - Popen(['bcfg2-yum-helper'], - stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() - HELPER = 'bcfg2-yum-helper' + self.cmd.run(['bcfg2-yum-helper']) + self.__class__._helper = 'bcfg2-yum-helper' except OSError: - HELPER = "/usr/sbin/bcfg2-yum-helper" - return HELPER + self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" + # pylint: enable=W0212 + return self._helper @property def use_yum(self): @@ -374,6 +406,7 @@ class YumCollection(Collection): # the rpmdb is so hopelessly intertwined with yum that we # have to totally reinvent the dependency resolver. mainopts = dict(cachedir='/', + persistdir='/', installroot=self.cachefile, keepcache="0", debuglevel="0", @@ -675,7 +708,10 @@ class YumCollection(Collection): gdicts.append(dict(group=group, type=ptype)) if self.use_yum: - return self.call_helper("get_groups", inputdata=gdicts) + try: + return self.call_helper("get_groups", inputdata=gdicts) + except ValueError: + return dict() else: pkgs = dict() for gdict in gdicts: @@ -837,13 +873,25 @@ class YumCollection(Collection): if not self.use_yum: return Collection.complete(self, packagelist) + lock = FileLock(os.path.join(self.cachefile, "lock")) + slept = 0 + while lock.is_locked(): + if slept > 30: + self.logger.warning("Packages: Timeout waiting for yum cache " + "to release its lock") + return set(), set() + self.logger.debug("Packages: Yum cache is locked, waiting...") + time.sleep(3) + slept += 3 + if packagelist: - result = \ - self.call_helper("complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) - if not result: - # some sort of error, reported by call_helper() + try: + result = self.call_helper( + "complete", + dict(packages=list(packagelist), + groups=list(self.get_relevant_groups()))) + except ValueError: + # error reported by call_helper() return set(), packagelist # json doesn't understand sets or tuples, so we get back a # lists of lists (packages) and a list of unicode strings @@ -874,38 +922,41 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - verbose = self.debug_flag or self.setup['verbose'] - if verbose: + if self.setup['verbose']: + cmd.append("-v") + if self.debug_flag: + if not self.setup['verbose']: + # ensure that running in debug gets -vv, even if + # verbose is not enabled + cmd.append("-v") cmd.append("-v") cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose) - try: - helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to execute %s: %s" % - (" ".join(cmd), err)) - return None + self.debug_log("Packages: running %s" % " ".join(cmd)) if inputdata: - idata = json.dumps(inputdata) - (stdout, stderr) = helper.communicate(idata) - else: - (stdout, stderr) = helper.communicate() - rv = helper.wait() - if rv: - self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, stderr)) + result = self.cmd.run(cmd, timeout=self.setup['client_timeout'], + inputdata=json.dumps(inputdata)) else: + result = self.cmd.run(cmd, timeout=self.setup['client_timeout']) + if not result.success: + self.logger.error("Packages: error running bcfg2-yum-helper: %s" % + result.error) + elif result.stderr: self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr, flag=verbose) + result.stderr) + try: - return json.loads(stdout) + return json.loads(result.stdout) except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: error reading bcfg2-yum-helper " - "output: %s" % err) - return None + if result.stdout: + err = sys.exc_info()[1] + self.logger.error("Packages: Error reading bcfg2-yum-helper " + "output: %s" % err) + self.logger.error("Packages: bcfg2-yum-helper output: %s" % + result.stdout) + else: + self.logger.error("Packages: No bcfg2-yum-helper output") + raise def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called @@ -917,8 +968,7 @@ class YumCollection(Collection): If using the yum Python libraries, this cleans up cached yum metadata, regenerates the server-side yum config (in order to catch any new sources that have been added to this server), - and then cleans up cached yum metadata again, in case the new - config has any preexisting cache. + then regenerates the yum cache. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -929,15 +979,22 @@ class YumCollection(Collection): return Collection.setup_data(self, force_update) if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config - self.call_helper("clean") + # clean up data from the old config + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass - os.unlink(self.cfgfile) + if os.path.exists(self.cfgfile): + os.unlink(self.cfgfile) self.write_config() - if force_update: - self.call_helper("clean") + try: + self.call_helper("makecache") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f82b8a392..479138ef1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,8 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ + MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -22,7 +23,54 @@ APT_CONFIG_DEFAULT = \ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +class OnDemandDict(MutableMapping): + """ This maps a set of keys to a set of value-getting functions; + the values are populated on-the-fly by the functions as the values + are needed (and not before). This is used by + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`; + see the docstring for that function for details on why. + + Unlike a dict, you should not specify values for for the righthand + side of this mapping, but functions that get values. E.g.: + + .. code-block:: python + + d = OnDemandDict(foo=load_foo, + bar=lambda: "bar"); + """ + + def __init__(self, **getters): + self._values = dict() + self._getters = dict(**getters) + + def __getitem__(self, key): + if key not in self._values: + self._values[key] = self._getters[key]() + return self._values[key] + + def __setitem__(self, key, getter): + self._getters[key] = getter + + def __delitem__(self, key): + del self._values[key] + del self._getters[key] + + def __len__(self): + return len(self._getters) + + def __iter__(self): + return iter(self._getters.keys()) + + def __repr__(self): + rv = dict(self._values) + for key in self._getters.keys(): + if key not in rv: + rv[key] = 'unknown' + return str(rv) + + class Packages(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -45,8 +93,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] + __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ + [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -110,8 +162,21 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() - # pylint: enable=C0301 + #: groupcache caches group lookups. It maps Collections (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of package groups, and thence to the packages + #: indicated by those groups. + self.groupcache = dict() + + #: pkgcache caches complete package sets. It maps Collections + #: (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of initial packages, and thence to the final + #: (complete) package selections resolved from the initial + #: packages + self.pkgcache = dict() + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def set_debug(self, debug): @@ -355,14 +420,24 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for el in to_remove: el.getparent().remove(el) - gpkgs = collection.get_groups(groups) - for pkgs in gpkgs.values(): + groups.sort() + # check for this set of groups in the group cache + gkey = hash(tuple(groups)) + if gkey not in self.groupcache[collection.cachekey]: + self.groupcache[collection.cachekey][gkey] = \ + collection.get_groups(groups) + for pkgs in self.groupcache[collection.cachekey][gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) - packages, unknown = collection.complete(base) + # check for this set of packages in the package cache + pkey = hash(tuple(base)) + if pkey not in self.pkgcache[collection.cachekey]: + self.pkgcache[collection.cachekey][pkey] = \ + collection.complete(base) + packages, unknown = self.pkgcache[collection.cachekey][pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -388,6 +463,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True + def expire_cache(self, _=None): + self.Reload() + def _load_config(self, force_update=False): """ Load the configuration data and setup sources @@ -415,9 +493,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.disableMetaData: collection.setup_data(force_update) - # clear Collection caches + # clear Collection and package caches self.clients = dict() self.collections = dict() + self.groupcache = dict() + self.pkgcache = dict() for source in self.sources.entries: cachefiles.add(source.cachefile) @@ -493,8 +573,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - return Collection(metadata, [], self.cachepath, self.data, - self.core.fam) + collection = Collection(metadata, [], self.cachepath, self.data, + self.core.fam) + ckey = collection.cachekey + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) + return collection if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -510,7 +594,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if len(sclasses) > 1: self.logger.warning("Packages: Multiple source types found for " "%s: %s" % - ",".join([s.__name__ for s in sclasses])) + (metadata.hostname, + ",".join([s.__name__ for s in sclasses]))) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % @@ -530,24 +615,47 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): """ Return additional data for the given client. This will be - a dict containing a single key, ``sources``, whose value is a - list of data returned from - :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, - namely, a list of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` - data. + an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` + containing two keys: + + * ``sources``, whose value is a list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data; and + * ``get_config``, whose value is the + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` + function, which can be used to get the Packages config for + other systems. + + This uses an OnDemandDict instead of just a normal dict + because loading a source collection can be a fairly + time-consuming process, particularly for the first time. As a + result, when all metadata objects are built at once (such as + after the server is restarted, or far more frequently if + Metadata caching is disabled), this function would be a major + bottleneck if we tried to build all collections at the same + time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ - collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - get_config=self.get_config) + def get_sources(): + """ getter for the 'sources' key of the OnDemandDict + returned by this function. This delays calling + get_collection() until it's absolutely necessary. """ + return self.get_collection(metadata).get_additional_data() + + return OnDemandDict( + sources=get_sources, + get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in |