diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins/Packages')
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 9 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 85 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Pac.py | 9 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 41 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 58 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 160 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py | 399 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 231 |
8 files changed, 613 insertions, 379 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index a82a183d8..dba56eed2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -13,8 +13,7 @@ class AptCollection(Collection): overrides nothing, and defers all operations to :class:`PacSource` """ - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -22,7 +21,7 @@ class AptCollection(Collection): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] @@ -53,10 +52,6 @@ class AptCollection(Collection): class AptSource(Source): """ Handle APT sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``AptSource`` are "apt", "debian", "ubuntu", and "nexenta" - basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] - #: AptSource sets the ``type`` on Package entries to "deb" ptype = 'deb' diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 39c51f351..8b20df58a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -73,17 +73,17 @@ The Collection Module --------------------- """ -import sys import copy -import logging import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Server.FileMonitor import get_fam +from Bcfg2.Server.Statistics import track_statistics -LOGGER = logging.getLogger(__name__) - -class Collection(list, Bcfg2.Server.Plugin.Debuggable): +class Collection(list, Debuggable): """ ``Collection`` objects represent the set of :class:`Bcfg2.Server.Plugins.Packages.Source` objects that apply to a given client, and can be used to query all software @@ -93,8 +93,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): #: Whether or not this Packages backend supports package groups __package_groups__ = False - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): """ :param metadata: The client metadata for this collection :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata @@ -111,29 +110,24 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): directory, where more permanent data can be stored :type basepath: string - :param fam: A file monitor object to use if this Collection - needs to monitor for file activity - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :param debug: Enable debugging output :type debug: bool .. ----- .. autoattribute:: __package_groups__ """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) list.__init__(self, sources) - self.debug_flag = debug + self.debug_flag = self.debug_flag or debug self.metadata = metadata self.basepath = basepath self.cachepath = cachepath self.virt_pkgs = dict() - self.fam = fam + self.fam = get_fam() try: - self.setup = sources[0].setup self.ptype = sources[0].ptype except IndexError: - self.setup = None self.ptype = "unknown" @property @@ -204,19 +198,6 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return sorted(list(set(groups))) @property - def basegroups(self): - """ Get a list of group names used by this Collection type in - resolution of - :ref:`server-plugins-generators-packages-magic-groups`. - - The base implementation simply aggregates the results of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.basegroups`.""" - groups = set() - for source in self: - groups.update(source.basegroups) - return list(groups) - - @property def cachefiles(self): """ A list of the full path to all cachefiles used by this collection. @@ -229,7 +210,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): cachefiles.add(source.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ Given a list of package group names, return a dict of ``<group name>: <list of packages>``. This method is provided @@ -250,7 +231,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): rv[group] = self.get_group(group, ptype) return rv - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_group(self, group, ptype=None): """ Get the list of packages of the given type in a package group. @@ -386,20 +367,6 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): for source in self: source.filter_unknown(unknown) - def magic_groups_match(self): - """ Returns True if the client's - :ref:`server-plugins-generators-packages-magic-groups` match - the magic groups for any of the sources contained in this - Collection. - - The base implementation returns True if any source - :func:`Bcfg2.Server.Plugins.Packages.Source.Source.magic_groups_match` - returns True. - - :returns: bool - """ - return any(s.magic_groups_match(self.metadata) for s in self) - def build_extra_structures(self, independent): """ Add additional entries to the ``<Independent/>`` section of the final configuration. This can be used to handle, e.g., @@ -476,9 +443,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ for pkg in pkglist: lxml.etree.SubElement(entry, 'BoundPackage', name=pkg, - version=self.setup.cfp.get("packages", - "version", - default="auto"), + version=Bcfg2.Options.setup.packages_version, type=self.ptype, origin='Packages') def get_new_packages(self, initial, complete): @@ -499,7 +464,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ return list(complete.difference(initial)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. @@ -630,22 +595,8 @@ def get_collection_class(source_type): :type source_type: string :returns: type - the Collection subclass that should be used to instantiate an object to contain sources of the given type. """ - modname = "Bcfg2.Server.Plugins.Packages.%s" % source_type.title() - try: - module = sys.modules[modname] - except KeyError: - try: - module = __import__(modname).Server.Plugins.Packages - except ImportError: - msg = "Packages: Unknown source type %s" % source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - try: - cclass = getattr(module, source_type.title() + "Collection") - except AttributeError: - msg = "Packages: No collection class found for %s sources" % \ - source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - return cclass + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % source_type.title()): + return getattr(mod, "%sCollection" % source_type.title()) + raise Bcfg2.Server.Plugin.PluginExecutionError( + "Packages: No collection class found for %s sources" % source_type) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 99aed5ce5..5f4d2ea41 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -12,8 +12,7 @@ class PacCollection(Collection): overrides nothing, and defers all operations to :class:`PacSource` """ - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): + def __init__(self, metadata, sources, cachepath, basepath, debug=False): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -21,7 +20,7 @@ class PacCollection(Collection): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] @@ -29,10 +28,6 @@ class PacCollection(Collection): class PacSource(Source): """ Handle Pacman sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``PacSource`` are "arch" and "parabola" - basegroups = ['arch', 'parabola'] - #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index c47e18201..1af046ec0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -4,12 +4,12 @@ import os import sys import Bcfg2.Server.Plugin +from Bcfg2.Server.Statistics import track_statistics from Bcfg2.Server.Plugins.Packages.Source import SourceInitError # pylint: disable=E0012,R0924 -class PackagesSources(Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): +class PackagesSources(Bcfg2.Server.Plugin.StructFile): """ PackagesSources handles parsing of the :mod:`Bcfg2.Server.Plugins.Packages` ``sources.xml`` file, and the creation of the appropriate @@ -19,7 +19,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, __identifier__ = None create = "Sources" - def __init__(self, filename, cachepath, fam, packages, setup): + def __init__(self, filename, cachepath, packages): """ :param filename: The full path to ``sources.xml`` :type filename: string @@ -27,21 +27,15 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data will be cached :type cachepath: string - :param fam: The file access monitor to use to create watches - on ``sources.xml`` and any XIncluded files. - :type fam: Bcfg2.Server.FileMonitor.FileMonitor :param packages: The Packages plugin object ``sources.xml`` is being parsed on behalf of (i.e., the calling object) :type packages: Bcfg2.Server.Plugins.Packages.Packages - :param setup: A Bcfg2 options dict - :type setup: dict :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` - If ``sources.xml`` cannot be read """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, should_monitor=True) #: The full path to the directory where @@ -57,8 +51,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, err = sys.exc_info()[1] self.logger.error("Could not create Packages cache at %s: %s" % (self.cachepath, err)) - #: The Bcfg2 options dict - self.setup = setup #: The :class:`Bcfg2.Server.Plugins.Packages.Packages` that #: instantiated this ``PackagesSources`` object @@ -72,10 +64,9 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, self.parsed = set() def set_debug(self, debug): - Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + Bcfg2.Server.Plugin.StructFile.set_debug(self, debug) for source in self.entries: source.set_debug(debug) - set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -106,7 +97,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, load its data. """ return sorted(list(self.parsed)) == sorted(self.extras) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] @@ -120,7 +111,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, ``Index`` is responsible for calling :func:`source_from_xml` for each ``Source`` tag in each file. """ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def source_from_xml(self, xsource): """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass @@ -141,19 +132,17 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, xsource.get("url")))) return None - try: - module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cls = getattr(module, "%sSource" % stype.title()) - except (ImportError, AttributeError): - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown source type %s (%s)" % (stype, - err)) + cls = None + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % stype.title()): + cls = getattr(mod, "%sSource" % stype.title()) + break + else: + self.logger.error("Packages: Unknown source type %s" % stype) return None try: - source = cls(self.cachepath, xsource, self.setup) + source = cls(self.cachepath, xsource) except SourceInitError: err = sys.exc_info()[1] self.logger.error("Packages: %s" % err) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 22073493c..4b6130f72 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -27,7 +27,6 @@ in your ``Source`` subclass: * :func:`Source.urls` * :func:`Source.read_files` -* :attr:`Source.basegroups` Additionally, you may want to consider overriding the following methods and attributes: @@ -50,10 +49,11 @@ in your ``Source`` subclass. For an example of this kind of import os import re import sys -import Bcfg2.Server.Plugin +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ cPickle, md5 +from Bcfg2.Server.Statistics import track_statistics def fetch_url(url): @@ -92,7 +92,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 +class Source(Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single <Source> tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -106,28 +106,21 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 those features. """ - #: The list of - #: :ref:`server-plugins-generators-packages-magic-groups` that - #: make sources of this type available to clients. - basegroups = [] - #: The Package type handled by this Source class. The ``type`` #: attribute of Package entries will be set to the value ``ptype`` #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None - def __init__(self, basepath, xsource, setup): # pylint: disable=R0912 + def __init__(self, basepath, xsource): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache data for this source should be stored :type basepath: string :param xsource: The XML tag that describes this source :type source: lxml.etree._Element - :param setup: A Bcfg2 options dict - :type setup: dict :raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError` """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) #: The base filesystem path under which cache data for this #: source should be stored @@ -136,9 +129,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: The XML tag that describes this source self.xsource = xsource - #: A Bcfg2 options dict - self.setup = setup - #: A set of package names that are deemed "essential" by this #: source self.essentialpkgs = set() @@ -308,8 +298,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :return: list of strings - group names """ return sorted(list(set([g for g in metadata.groups - if (g in self.basegroups or - g in self.groups or + if (g in self.groups or g in self.arches)]))) def load_state(self): @@ -332,7 +321,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.essentialpkgs), cache, 2) cache.close() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def setup_data(self, force_update=False): """ Perform all data fetching and setup tasks. For most backends, this involves downloading all metadata from the @@ -636,16 +625,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 def applies(self, metadata): """ Return true if this source applies to the given client, - i.e., the client is in all necessary groups and - :ref:`server-plugins-generators-packages-magic-groups`. + i.e., the client is in all necessary groups. :param metadata: The client metadata to check to see if this source applies :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :returns: bool """ - # check base groups - if not self.magic_groups_match(metadata): + # check arch groups + if not self.arch_groups_match(metadata): return False # check Group/Client tags from sources.xml @@ -716,29 +704,13 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ return [] - def magic_groups_match(self, metadata): - """ Returns True if the client's - :ref:`server-plugins-generators-packages-magic-groups` match - the magic groups this source. Also returns True if magic - groups are off in the configuration and the client's - architecture matches (i.e., architecture groups are *always* - checked). + def arch_groups_match(self, metadata): + """ Returns True if the client is in an arch group that + matches the arch of this source. :returns: bool """ - found_arch = False for arch in self.arches: if arch in metadata.groups: - found_arch = True - break - if not found_arch: - return False - - if not self.setup.cfp.getboolean("packages", "magic_groups", - default=False): - return True - else: - for group in self.basegroups: - if group in metadata.groups: - return True - return False + return True + return False diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 66f8e9dbe..b98d3f419 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -60,6 +60,7 @@ import socket import logging import lxml.etree import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor from lockfile import FileLock from Bcfg2.Utils import Executor # pylint: disable=W0622 @@ -69,6 +70,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url +from Bcfg2.Server.Statistics import track_statistics LOGGER = logging.getLogger(__name__) @@ -105,13 +107,36 @@ PULPSERVER = None PULPCONFIG = None -def _setup_pulp(setup): +options = [ # pylint: disable=C0103 + Bcfg2.Options.Common.client_timeout, + Bcfg2.Options.PathOption( + cf=("packages:yum", "helper"), dest="yum_helper", + help="Path to the bcfg2-yum-helper executable"), + Bcfg2.Options.BooleanOption( + cf=("packages:yum", "use_yum_libraries"), + help="Use Python yum libraries"), + Bcfg2.Options.PathOption( + cf=("packages:yum", "gpg_keypath"), default="/etc/pki/rpm-gpg", + help="GPG key path on the client"), + Bcfg2.Options.Option( + cf=("packages:yum", "*"), dest="yum_options", + help="Other yum options to include in generated yum configs")] +if HAS_PULP: + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "username"), dest="pulp_username", + help="Username for Pulp authentication")) + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "password"), dest="pulp_password", + help="Password for Pulp authentication")) + + +def _setup_pulp(): """ Connect to a Pulp server and pass authentication credentials. This only needs to be called once, but multiple calls won't hurt anything. - :param setup: A Bcfg2 options dict - :type setup: dict :returns: :class:`pulp.client.api.server.PulpServer` """ global PULPSERVER, PULPCONFIG @@ -122,19 +147,6 @@ def _setup_pulp(setup): raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: - try: - username = setup.cfp.get("packages:pulp", "username") - password = setup.cfp.get("packages:pulp", "password") - except ConfigParser.NoSectionError: - msg = "Packages: No [pulp] section found in bcfg2.conf" - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - except ConfigParser.NoOptionError: - msg = "Packages: Required option not found in bcfg2.conf: %s" % \ - sys.exc_info()[1] - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server @@ -142,7 +154,9 @@ def _setup_pulp(setup): int(serveropts['port']), serveropts['scheme'], serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) + PULPSERVER.set_basic_auth_credentials( + Bcfg2.Options.setup.pulp_username, + Bcfg2.Options.setup.pulp_password) server.set_active_server(PULPSERVER) return PULPSERVER @@ -173,7 +187,7 @@ class PulpCertificateSet(Bcfg2.Server.Plugin.EntrySet): #: The path to certificates on consumer machines certpath = "/etc/pki/consumer/cert.pem" - def __init__(self, path, fam): + def __init__(self, path): """ :param path: The path to the directory where Pulp consumer certificates will be stored @@ -191,7 +205,7 @@ class PulpCertificateSet(Bcfg2.Server.Plugin.EntrySet): important='true', sensitive='true', paranoid=self.metadata['paranoid']) - self.fam = fam + self.fam = Bcfg2.Server.FileMonitor.get_fam() self.fam.AddMonitor(path, self) def HandleEvent(self, event): @@ -272,9 +286,8 @@ class YumCollection(Collection): #: :class:`PulpCertificateSet` object used to handle Pulp certs pulp_cert_set = None - def __init__(self, metadata, sources, cachepath, basepath, fam, - debug=False): - Collection.__init__(self, metadata, sources, cachepath, basepath, fam, + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + Collection.__init__(self, metadata, sources, cachepath, basepath, debug=debug) self.keypath = os.path.join(self.cachepath, "keys") @@ -295,13 +308,15 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): self.debug_log("Creating common cache %s" % self.cachefile) os.mkdir(self.cachefile) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: self.setup_data() + self.cmd = Executor() else: self.cachefile = None + self.cmd = None if HAS_PULP and self.has_pulp_sources: - _setup_pulp(self.setup) + _setup_pulp() if self.pulp_cert_set is None: certdir = os.path.join( self.basepath, @@ -317,28 +332,7 @@ class YumCollection(Collection): self.logger.error("Could not create Pulp consumer " "cert directory at %s: %s" % (certdir, err)) - self.__class__.pulp_cert_set = PulpCertificateSet(certdir, - self.fam) - - @property - def disableMetaData(self): - """ Report whether or not metadata processing is enabled. - This duplicates code in Packages/__init__.py, and can probably - be removed in Bcfg2 1.4 when we have a module-level setup - object. """ - if self.setup is None: - return True - try: - return not self.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" + self.__class__.pulp_cert_set = PulpCertificateSet(certdir) @property def __package_groups__(self): @@ -354,10 +348,8 @@ class YumCollection(Collection): the default location. """ if not self._helper: # pylint: disable=W0212 - try: - self.__class__._helper = self.setup.cfp.get("packages:yum", - "helper") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.__class__._helper = Bcfg2.Options.setup.yum_helper + if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") @@ -372,9 +364,7 @@ class YumCollection(Collection): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries @property def has_pulp_sources(self): @@ -391,7 +381,7 @@ class YumCollection(Collection): cachefiles.add(self.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def write_config(self): """ Write the server-side config file to :attr:`cfgfile` based on the data from :func:`get_config`""" @@ -410,15 +400,15 @@ class YumCollection(Collection): debuglevel="0", sslverify="0", reposdir="/dev/null") - if self.setup['debug']: + if Bcfg2.Options.setup.debug: mainopts['debuglevel'] = "5" - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: mainopts['debuglevel'] = "2" try: - for opt in self.setup.cfp.options("packages:yum"): + for opt, val in Bcfg2.Options.setup.yum_options.items(): if opt not in self.option_blacklist: - mainopts[opt] = self.setup.cfp.get("packages:yum", opt) + mainopts[opt] = val except ConfigParser.NoSectionError: pass @@ -494,7 +484,7 @@ class YumCollection(Collection): return "# This config was generated automatically by the Bcfg2 " \ "Packages plugin\n\n" + buf.getvalue() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def build_extra_structures(self, independent): """ Add additional entries to the ``<Independent/>`` section of the final configuration. This adds several kinds of @@ -540,8 +530,7 @@ class YumCollection(Collection): for key in needkeys: # figure out the path of the key on the client - keydir = self.setup.cfp.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") + keydir = Bcfg2.Options.setup.gpg_keypath remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() @@ -586,7 +575,7 @@ class YumCollection(Collection): # each pulp source can only have one arch, so we don't # have to check the arch in url_map if (source.pulp_id and - source.pulp_id not in consumer['repoids']): + source.pulp_id not in consumer['repoids']): try: consumerapi.bind(self.metadata.hostname, source.pulp_id) @@ -601,7 +590,7 @@ class YumCollection(Collection): name=self.pulp_cert_set.certpath) self.pulp_cert_set.bind_entry(crt, self.metadata) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _get_pulp_consumer(self, consumerapi=None): """ Get a Pulp consumer object for the client. @@ -630,7 +619,7 @@ class YumCollection(Collection): "%s" % err) return consumer - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _add_gpg_instances(self, keyentry, localkey, remotekey, keydata=None): """ Add GPG keys instances to a ``Package`` entry. This is called from :func:`build_extra_structures` to add GPG keys to @@ -673,7 +662,7 @@ class YumCollection(Collection): self.logger.error("Packages: Could not read GPG key %s: %s" % (localkey, err)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ If using the yum libraries, given a list of package group names, return a dict of ``<group name>: <list of packages>``. @@ -760,8 +749,7 @@ class YumCollection(Collection): """ Given a package tuple, return a dict of attributes suitable for applying to either a Package or an Instance tag """ - attrs = dict(version=self.setup.cfp.get("packages", "version", - default="auto")) + attrs = dict(version=Bcfg2.Options.setup.packages_version) if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): return attrs @@ -850,7 +838,7 @@ class YumCollection(Collection): new.append(pkg) return new - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): """ Build a complete list of all packages and their dependencies. @@ -902,7 +890,7 @@ class YumCollection(Collection): else: return set(), set() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def call_helper(self, command, inputdata=None): """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have horrific memory leaks, so apparently the right way to get @@ -920,22 +908,20 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - if self.setup['verbose']: + if Bcfg2.Options.setup.verbose: cmd.append("-v") if self.debug_flag: - if not self.setup['verbose']: - # ensure that running in debug gets -vv, even if - # verbose is not enabled - cmd.append("-v") - cmd.append("-v") + cmd.append("-d") cmd.append(command) self.debug_log("Packages: running %s" % " ".join(cmd)) if inputdata: - result = self.cmd.run(cmd, timeout=self.setup['client_timeout'], + result = self.cmd.run(cmd, + timeout=Bcfg2.Options.setup.client_timeout, inputdata=json.dumps(inputdata)) else: - result = self.cmd.run(cmd, timeout=self.setup['client_timeout']) + result = self.cmd.run(cmd, + timeout=Bcfg2.Options.setup.client_timeout) if not result.success: self.logger.error("Packages: error running bcfg2-yum-helper: %s" % result.error) @@ -998,20 +984,16 @@ class YumCollection(Collection): class YumSource(Source): """ Handle yum sources """ - #: :ref:`server-plugins-generators-packages-magic-groups` for - #: ``YumSource`` are "yum", "redhat", "centos", and "fedora" - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - #: YumSource sets the ``type`` on Package entries to "yum" ptype = 'yum' - def __init__(self, basepath, xsource, setup): - Source.__init__(self, basepath, xsource, setup) + def __init__(self, basepath, xsource): + Source.__init__(self, basepath, xsource) self.pulp_id = None if HAS_PULP and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") - _setup_pulp(self.setup) + _setup_pulp() repoapi = RepositoryAPI() try: self.repo = repoapi.repository(self.pulp_id) @@ -1054,9 +1036,7 @@ class YumSource(Source): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries def save_state(self): """ If using the builtin yum parser, save state to @@ -1138,7 +1118,7 @@ class YumSource(Source): self.file_to_arch[self.escape_url(fullurl)] = arch return urls - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def read_files(self): """ When using the builtin yum parser, read and parse locally downloaded metadata files. This diverges from the stock @@ -1186,7 +1166,7 @@ class YumSource(Source): self.packages[key].difference(self.packages['global']) self.save_state() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_filelist(self, data, arch): """ parse filelists.xml.gz data """ if arch not in self.filemap: @@ -1200,7 +1180,7 @@ class YumSource(Source): self.filemap[arch][fentry.text] = \ set([pkg.get('name')]) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_primary(self, data, arch): """ parse primary.xml.gz data """ if arch not in self.packages: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py new file mode 100644 index 000000000..48304d26e --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -0,0 +1,399 @@ +""" Libraries for bcfg2-yum-helper plugin, used if yum library support +is enabled. The yum libs have horrific memory leaks, so apparently +the right way to get around that in long-running processes it to have +a short-lived helper. No, seriously -- check out the yum-updatesd +code. It's pure madness. """ + +import os +import sys +import yum +import logging +import Bcfg2.Options +import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout +try: + import json +except ImportError: + import simplejson as json + + +def pkg_to_tuple(package): + """ json doesn't distinguish between tuples and lists, but yum + does, so we convert a package in list format to one in tuple + format """ + if isinstance(package, list): + return tuple(package) + else: + return package + + +def pkgtup_to_string(package): + """ given a package tuple, return a human-readable string + describing the package """ + if package[3] in ['auto', 'any']: + return package[0] + + rv = [package[0], "-"] + if package[2]: + rv.extend([package[2], ':']) + rv.extend([package[3], '-', package[4]]) + if package[1]: + rv.extend(['.', package[1]]) + return ''.join(str(e) for e in rv) + + +class YumHelper(object): + """ Yum helper base object """ + + def __init__(self, cfgfile, verbose=1): + self.cfgfile = cfgfile + self.yumbase = yum.YumBase() + # pylint: disable=E1121,W0212 + try: + self.yumbase.preconf.debuglevel = verbose + self.yumbase.preconf.fn = cfgfile + self.yumbase._getConfig() + except AttributeError: + self.yumbase._getConfig(cfgfile, debuglevel=verbose) + # pylint: enable=E1121,W0212 + self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 + self._groups = None + + def get_groups(self): + """ getter for the groups property """ + if self._groups is not None: + return self._groups + else: + return ["noarch"] + + def set_groups(self, groups): + """ setter for the groups property """ + self._groups = set(groups).union(["noarch"]) + + groups = property(get_groups, set_groups) + + def get_package_object(self, pkgtup, silent=False): + """ given a package tuple, get a yum package object """ + try: + matches = yum.packageSack.packagesNewestByName( + self.yumbase.pkgSack.searchPkgTuple(pkgtup)) + except yum.Errors.PackageSackError: + if not silent: + self.logger.warning("Package '%s' not found" % + self.get_package_name(pkgtup)) + matches = [] + except yum.Errors.RepoError: + err = sys.exc_info()[1] + self.logger.error("Temporary failure loading metadata for %s: %s" % + (self.get_package_name(pkgtup), err)) + matches = [] + + pkgs = self._filter_arch(matches) + if pkgs: + return pkgs[0] + else: + return None + + def get_group(self, group, ptype="default"): + """ Resolve a package group name into a list of packages """ + if group.startswith("@"): + group = group[1:] + + try: + if self.yumbase.comps.has_group(group): + group = self.yumbase.comps.return_group(group) + else: + self.logger.error("%s is not a valid group" % group) + return [] + except yum.Errors.GroupsError: + err = sys.exc_info()[1] + self.logger.warning(err) + return [] + + if ptype == "default": + return [p + for p, d in list(group.default_packages.items()) + if d] + elif ptype == "mandatory": + return [p + for p, m in list(group.mandatory_packages.items()) + if m] + elif ptype == "optional" or ptype == "all": + return group.packages + else: + self.logger.warning("Unknown group package type '%s'" % ptype) + return [] + + def _filter_arch(self, packages): + """ filter packages in the given list that do not have an + architecture in the list of groups for this client """ + matching = [] + for pkg in packages: + if pkg.arch in self.groups: + matching.append(pkg) + else: + self.logger.debug("%s has non-matching architecture (%s)" % + (pkg, pkg.arch)) + if matching: + return matching + else: + # no packages match architecture; we'll assume that the + # user knows what s/he is doing and this is a multiarch + # box. + return packages + + def get_package_name(self, package): + """ get the name of a package or virtual package from the + internal representation used by this Collection class """ + if isinstance(package, tuple): + if len(package) == 3: + return yum.misc.prco_tuple_to_string(package) + else: + return pkgtup_to_string(package) + else: + return str(package) + + def complete(self, packagelist): + """ resolve dependencies and generate a complete package list + from the given list of initial packages """ + packages = set() + unknown = set() + for pkg in packagelist: + if isinstance(pkg, tuple): + pkgtup = pkg + else: + pkgtup = (pkg, None, None, None, None) + pkgobj = self.get_package_object(pkgtup) + if not pkgobj: + self.logger.debug("Unknown package %s" % + self.get_package_name(pkg)) + unknown.add(pkg) + else: + if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): + self.logger.debug("%s added to transaction multiple times" + % pkgobj) + else: + self.logger.debug("Adding %s to transaction" % pkgobj) + self.yumbase.tsInfo.addInstall(pkgobj) + self.yumbase.resolveDeps() + + for txmbr in self.yumbase.tsInfo: + packages.add(txmbr.pkgtup) + return list(packages), list(unknown) + + +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock + def clean_cache(self): + """ clean the yum cache """ + for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", + "ExpireCache"]: + # for reasons that are entirely obvious, all of the yum + # API clean* methods return a tuple of 0 (zero, always + # zero) and a list containing a single message about how + # many files were deleted. so useful. thanks, yum. + msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] + if not msg.startswith("0 "): + self.logger.info(msg) + + @acquire_lock + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + try: + repo.getGroups() + except yum.Errors.RepoMDError: + pass # this repo has no groups + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + # this does something with the groups cache as a side effect + self.yumbase.comps # pylint: disable=W0104 + + +class HelperSubcommand(Bcfg2.Options.Subcommand): + """ Base class for all yum helper subcommands """ + + # the value to JSON encode and print out if the command fails + fallback = None + + # whether or not this command accepts input on stdin + accept_input = True + + def __init__(self): + Bcfg2.Options.Subcommand.__init__(self) + self.verbosity = 0 + if Bcfg2.Options.setup.debug: + self.verbosity = 5 + elif Bcfg2.Options.setup.verbose: + self.verbosity = 1 + + def run(self, setup): + try: + data = json.loads(sys.stdin.read()) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error decoding JSON input: %s" % + sys.exc_info()[1]) + print(json.dumps(self.fallback)) + return 2 + + try: + print(json.dumps(self._run(setup, data))) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error running %s: %s" % + self.__class__.__name__.lower(), + sys.exc_info()[1], exc_info=1) + print(json.dumps(self.fallback)) + return 2 + return 0 + + def _run(self, setup, data): + """ Actually run the command """ + raise NotImplementedError + + +class DepSolverSubcommand(HelperSubcommand): # pylint: disable=W0223 + """ Base class for helper commands that use the depsolver (i.e., + only resolve dependencies, don't modify the cache) """ + + def __init__(self): + HelperSubcommand.__init__(self) + self.depsolver = DepSolver(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class CacheManagerSubcommand(HelperSubcommand): # pylint: disable=W0223 + """ Base class for helper commands that use the cachemanager + (i.e., modify the cache) """ + fallback = False + accept_input = False + + def __init__(self): + HelperSubcommand.__init__(self) + self.cachemgr = CacheManager(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class Clean(CacheManagerSubcommand): + """ Clean the cache """ + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.clean_cache() + return True + + +class MakeCache(CacheManagerSubcommand): + """ Update the on-disk cache """ + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.populate_cache() + return True + + +class Complete(DepSolverSubcommand): + """ Given an initial set of packages, get a complete set of + packages with all dependencies resolved """ + fallback = dict(packages=[], unknown=[]) + + def _run(self, _, data): + self.depsolver.groups = data['groups'] + self.fallback['unknown'] = data['packages'] + (packages, unknown) = self.depsolver.complete( + [pkg_to_tuple(p) for p in data['packages']]) + return dict(packages=list(packages), unknown=list(unknown)) + + +class GetGroups(DepSolverSubcommand): + """ Resolve the given package groups """ + def _run(self, _, data): + rv = dict() + for gdata in data: + if "type" in gdata: + packages = self.depsolver.get_group(gdata['group'], + ptype=gdata['type']) + else: + packages = self.depsolver.get_group(gdata['group']) + rv[gdata['group']] = list(packages) + return rv + + +Get_Groups = GetGroups # pylint: disable=C0103 + + +class CLI(Bcfg2.Options.CommandRegistry): + """ The bcfg2-yum-helper CLI """ + options = [ + Bcfg2.Options.PathOption( + "-c", "--yum-config", help="Yum config file"), + Bcfg2.Options.PositionalArgument( + "command", help="Yum helper command", + choices=['clean', 'complete', 'get_groups'])] + + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=HelperSubcommand) + parser = Bcfg2.Options.get_parser("Bcfg2 yum helper", + components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + """ Run bcfg2-yum-helper """ + if not os.path.exists(Bcfg2.Options.setup.yum_config): + self.logger.error("Config file %s not found" % + Bcfg2.Options.setup.yum_config) + return 1 + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 479138ef1..49f64bdf3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -7,20 +7,32 @@ import sys import glob import shutil import lxml.etree -import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ - MutableMapping +from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Statistics import track_statistics -#: The default path for generated yum configs -YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" -#: The default path for generated apt configs -APT_CONFIG_DEFAULT = \ - "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +def packages_boolean(value): + """ For historical reasons, the Packages booleans 'resolver' and + 'metadata' both accept "enabled" in addition to the normal boolean + values. """ + if value == 'disabled': + return False + elif value == 'enabled': + return True + else: + return value + + +class PackagesBackendAction(Bcfg2.Options.ComponentAction): + """ ComponentAction to load Packages backends """ + bases = ['Bcfg2.Server.Plugins.Packages'] + module = True class OnDemandDict(MutableMapping): @@ -70,7 +82,6 @@ class OnDemandDict(MutableMapping): class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -85,6 +96,38 @@ class Packages(Bcfg2.Server.Plugin.Plugin, .. private-include: _build_packages""" + options = [ + Bcfg2.Options.Option( + cf=("packages", "backends"), dest="packages_backends", + help="Packages backends to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), + Bcfg2.Options.PathOption( + cf=("packages", "cache"), dest="packages_cache", + help="Path to the Packages cache", + default='<repository>/Packages/cache'), + Bcfg2.Options.Option( + cf=("packages", "resolver"), dest="packages_resolver", + help="Disable the Packages resolver", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "metadata"), dest="packages_metadata", + help="Disable all Packages metadata processing", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "version"), dest="packages_version", + help="Set default Package entry version", default="auto", + choices=["auto", "any"]), + Bcfg2.Options.PathOption( + cf=("packages", "yum_config"), + help="The default path for generated yum configs", + default="/etc/yum.repos.d/bcfg2.repo"), + Bcfg2.Options.PathOption( + cf=("packages", "apt_config"), + help="The default path for generated apt configs", + default= + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list")] + #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] @@ -93,12 +136,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ - [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -107,9 +146,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. - self.cachepath = \ - self.core.setup.cfp.get("packages", "cache", - default=os.path.join(self.data, 'cache')) + self.cachepath = Bcfg2.Options.setup.packages_cache #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') @@ -117,14 +154,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # create key directory if needed os.makedirs(self.keypath) - # warn about deprecated magic groups - if self.core.setup.cfp.getboolean("packages", "magic_groups", - default=False): - self.logger.warning("Packages: Magic groups are deprecated and " - "will be removed in a future release") - self.logger.warning("You can disable magic groups by setting " - "magic_groups=0 in [packages] in bcfg2.conf") - # pylint: disable=C0301 #: The #: :class:`Bcfg2.Server.Plugins.Packages.PackagesSources.PackagesSources` @@ -132,8 +161,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects for #: this plugin. self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), - self.cachepath, core.fam, self, - self.core.setup) + self.cachepath, self) #: We cache #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` @@ -153,7 +181,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. - self.collections = dict() + self.collections = Bcfg2.Server.Cache.Cache("Packages", "collections") #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` @@ -161,21 +189,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. - self.clients = dict() - - #: groupcache caches group lookups. It maps Collections (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of package groups, and thence to the packages - #: indicated by those groups. - self.groupcache = dict() - - #: pkgcache caches complete package sets. It maps Collections - #: (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of initial packages, and thence to the final - #: (complete) package selections resolved from the initial - #: packages - self.pkgcache = dict() + self.clients = Bcfg2.Server.Cache.Cache("Packages", "cache") + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ @@ -187,48 +202,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return rv set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ - @property - def disableResolver(self): - """ Report the state of the resolver. This can be disabled in - the configuration. Note that disabling metadata (see - :attr:`disableMetaData`) implies disabling the resolver. - - This property cannot be set. """ - if self.disableMetaData: - # disabling metadata without disabling the resolver Breaks - # Things - return True - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled", which are not handled according to the - # Python docs but appear to be handled properly by - # ConfigParser in at least some versions - return self.core.setup.cfp.get( - "packages", - "resolver", - default="enabled").lower() == "disabled" - - @property - def disableMetaData(self): - """ Report whether or not metadata processing is enabled. - - This property cannot be set. """ - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.core.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" - def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. @@ -276,9 +249,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if entry.tag == 'Package': collection = self.get_collection(metadata) - entry.set('version', self.core.setup.cfp.get("packages", - "version", - default="auto")) + entry.set('version', Bcfg2.Options.setup.packages_version) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) @@ -304,27 +275,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` """ if entry.tag == 'Package': - if self.core.setup.cfp.getboolean("packages", "magic_groups", - default=False): - collection = self.get_collection(metadata) - if collection.magic_groups_match(): - return True - else: - return True + return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): + if entry.get("name") in [Bcfg2.Options.setup.apt_config, + Bcfg2.Options.setup.yum_config]: return True return False - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: @@ -353,13 +312,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :returns: None """ collection = self.get_collection(metadata) - indep = lxml.etree.Element('Independent') + indep = lxml.etree.Element('Independent', name=self.__class__.__name__) self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) structures.append(indep) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _build_packages(self, metadata, independent, structures, collection=None): """ Perform dependency resolution and build the complete list @@ -382,8 +341,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :func:`get_collection` :type collection: Bcfg2.Server.Plugins.Packages.Collection.Collection """ - if self.disableResolver: - # Config requests no resolver + if (not Bcfg2.Options.setup.packages_metadata or + not Bcfg2.Options.setup.packages_resolver): + # Config requests no resolver. Note that disabling + # metadata implies disabling the resolver. for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("group"): @@ -422,11 +383,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups.sort() # check for this set of groups in the group cache + gcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_groups", + collection.cachekey) gkey = hash(tuple(groups)) - if gkey not in self.groupcache[collection.cachekey]: - self.groupcache[collection.cachekey][gkey] = \ - collection.get_groups(groups) - for pkgs in self.groupcache[collection.cachekey][gkey].values(): + if gkey not in gcache: + gcache[gkey] = collection.get_groups(groups) + for pkgs in gcache[gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution @@ -434,10 +396,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # check for this set of packages in the package cache pkey = hash(tuple(base)) - if pkey not in self.pkgcache[collection.cachekey]: - self.pkgcache[collection.cachekey][pkey] = \ - collection.complete(base) - packages, unknown = self.pkgcache[collection.cachekey][pkey] + pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", + collection.cachekey) + if pkey not in pcache: + pcache[pkey] = collection.complete(base) + packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -447,7 +410,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, newpkgs.sort() collection.packages_to_entry(newpkgs, independent) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Refresh(self): """ Packages.Refresh() => True|False @@ -455,7 +418,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config(force_update=True) return True - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Reload(self): """ Packages.Refresh() => True|False @@ -463,7 +426,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True - def expire_cache(self, _=None): + def child_reload(self, _=None): + """ Reload the Packages configuration on a child process. """ self.Reload() def _load_config(self, force_update=False): @@ -490,18 +454,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for collection in list(self.collections.values()): cachefiles.update(collection.cachefiles) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: collection.setup_data(force_update) # clear Collection and package caches - self.clients = dict() - self.collections = dict() - self.groupcache = dict() - self.pkgcache = dict() + Bcfg2.Server.Cache.expire("Packages") for source in self.sources.entries: cachefiles.add(source.cachefile) - if not self.disableMetaData: + if Bcfg2.Options.setup.packages_metadata: source.setup_data(force_update) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): @@ -533,7 +494,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if localfile not in keyfiles: keyfiles.append(localfile) if ((force_update and key not in keys) or - not os.path.exists(localfile)): + not os.path.exists(localfile)): self.logger.info("Packages: Downloading and parsing %s" % key) try: @@ -556,7 +517,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if kfile not in keyfiles: os.unlink(kfile) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` @@ -573,12 +534,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - collection = Collection(metadata, [], self.cachepath, self.data, - self.core.fam) - ckey = collection.cachekey - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) - return collection + return Collection(metadata, [], self.cachepath, self.data) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -610,13 +566,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, "for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, self.cachepath, self.data, - self.core.fam, debug=self.debug_flag) + debug=self.debug_flag) ckey = collection.cachekey if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): @@ -665,8 +619,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ - if metadata.hostname in self.clients: - del self.clients[metadata.hostname] + self.clients.expire(metadata.hostname) def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` |