summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--doc/conf.py1
-rw-r--r--doc/development/packages.txt11
-rw-r--r--doc/server/plugins/generators/packages.txt2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py20
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Collection.py17
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Pac.py16
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py21
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Yum.py476
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py4
-rwxr-xr-xsrc/sbin/bcfg2-yum-helper106
10 files changed, 452 insertions, 222 deletions
diff --git a/doc/conf.py b/doc/conf.py
index 31a7960fe..7fc10ad61 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -19,7 +19,6 @@ import time
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.append(os.path.abspath('../src/lib'))
sys.path.insert(0, os.path.abspath('../src/lib'))
# -- General configuration -----------------------------------------------------
diff --git a/doc/development/packages.txt b/doc/development/packages.txt
index c310805a7..d6147e844 100644
--- a/doc/development/packages.txt
+++ b/doc/development/packages.txt
@@ -46,6 +46,17 @@ Packages Source Description
Existing Packages Backends
==========================
+Yum
+---
+
.. automodule:: Bcfg2.Server.Plugins.Packages.Yum
+
+APT
+---
+
.. automodule:: Bcfg2.Server.Plugins.Packages.Apt
+
+Pacman
+------
+
.. automodule:: Bcfg2.Server.Plugins.Packages.Pac
diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt
index d2c425f1d..e97742ca7 100644
--- a/doc/server/plugins/generators/packages.txt
+++ b/doc/server/plugins/generators/packages.txt
@@ -691,8 +691,6 @@ It understands the following directives:
+-------------+------------------------------------------------------+----------+-----------------------------+
| cache | Path where Packages will store its cache | String | <repo>/Packages/cache |
+-------------+------------------------------------------------------+----------+-----------------------------+
-| keycache | Path where Packages will cache downloaded GPG keys | String | <repo>/Packages/keys |
-+-------------+------------------------------------------------------+----------+-----------------------------+
[packages:yum] section
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 5e3d86f02..11355e117 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -1,3 +1,5 @@
+""" APT backend for :mod:`Bcfg2.Server.Plugins.Packages` """
+
import re
import gzip
from Bcfg2.Server.Plugins.Packages.Collection import Collection
@@ -5,7 +7,16 @@ from Bcfg2.Server.Plugins.Packages.Source import Source
class AptCollection(Collection):
+ """ Handle collections of APT sources. This is a no-op object
+ that simply inherits from
+ :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`,
+ overrides nothing, and defers all operations to :class:`PacSource`
+ """
+
def get_config(self):
+ """ Get an APT configuration file (i.e., ``sources.list``).
+
+ :returns: string """
lines = ["# This config was generated automatically by the Bcfg2 " \
"Packages plugin", '']
@@ -22,11 +33,19 @@ class AptCollection(Collection):
class AptSource(Source):
+ """ Handle APT sources """
+
+ #: :ref:`server-plugins-generators-packages-magic-groups` for
+ #: ``AptSource`` are "apt", "debian", "ubuntu", and "nexenta"
basegroups = ['apt', 'debian', 'ubuntu', 'nexenta']
+
+ #: AptSource sets the ``type`` on Package entries to "deb"
ptype = 'deb'
@property
def urls(self):
+ """ A list of URLs to the base metadata file for each
+ repository described by this source. """
if not self.rawurl:
rv = []
for part in self.components:
@@ -92,3 +111,4 @@ class AptSource(Source):
bprov[barch][dname] = set()
bprov[barch][dname].add(pkgname)
self.process_files(bdeps, bprov)
+ read_files.__doc__ = Source.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index f5c035e00..d0e4a3665 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -109,6 +109,8 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
:type basepath: string
:param debug: Enable debugging output
:type debug: bool
+
+ .. autoattribute:: __package_groups__
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
list.__init__(self, sources)
@@ -119,15 +121,11 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
try:
self.setup = sources[0].setup
- self.cachepath = sources[0].basepath
self.ptype = sources[0].ptype
except IndexError:
self.setup = None
- self.cachepath = None
self.ptype = "unknown"
- self.cachefile = None
-
@property
def cachekey(self):
""" A unique identifier for the set of sources contained in
@@ -141,8 +139,9 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
source type. This should be a config appropriate for use on
either the server (to resolve dependencies) or the client.
- Subclasses must override this method. By default it logs an
- error and returns the empty string.
+ Subclasses must override this method in order to be able to
+ generate configs. By default it logs an error and returns the
+ empty string.
:returns: string """
self.logger.error("Packages: Cannot generate config for host %s with "
@@ -188,7 +187,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
get_relevant_groups.__doc__ = Source.get_relevant_groups.__doc__ + """
The base implementation simply aggregates the results of
- :func:`Bcfg2.Server.Plugins.Packages.Source.Source.get_relevant_groups`.
+ :func:`Bcfg2.Server.Plugins.Packages.Source.Source.get_relevant_groups`
"""
@property
@@ -212,7 +211,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
The base implementation simply aggregates
:attr:`Bcfg2.Server.Plugins.Packages.Source.Source.cachefile`
attributes."""
- cachefiles = set([self.cachepath])
+ cachefiles = set()
for source in self:
cachefiles.add(source.cachefile)
return list(cachefiles)
@@ -393,7 +392,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
handled for a complete client configuration.
:param independent: The XML tag to add extra entries to. This
- should be modified in place.
+ is modified in place.
:type independent: lxml.etree._Element
"""
pass
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
index 13090cd9f..ac58f4c99 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
@@ -1,18 +1,33 @@
+""" Pacman backend for :mod:`Bcfg2.Server.Plugins.Packages` """
+
import tarfile
from Bcfg2.Server.Plugins.Packages.Collection import Collection
from Bcfg2.Server.Plugins.Packages.Source import Source
class PacCollection(Collection):
+ """ Handle collections of Pacman sources. This is a no-op object
+ that simply inherits from
+ :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`,
+ overrides nothing, and defers all operations to :class:`PacSource`
+ """
pass
class PacSource(Source):
+ """ Handle Pacman sources """
+
+ #: :ref:`server-plugins-generators-packages-magic-groups` for
+ #: ``PacSource`` are "arch" and "parabola"
basegroups = ['arch', 'parabola']
+
+ #: PacSource sets the ``type`` on Package entries to "pacman"
ptype = 'pacman'
@property
def urls(self):
+ """ A list of URLs to the base metadata file for each
+ repository described by this source. """
if not self.rawurl:
rv = []
for part in self.components:
@@ -56,3 +71,4 @@ class PacSource(Source):
tarinfo.name.rsplit("-", 2)[0])
tar.close()
self.process_files(bdeps, bprov)
+ read_files.__doc__ = Source.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 3269f8c08..7188394b3 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -119,6 +119,11 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
#: default, excludes any package whose name starts with "choice"
unknown_filter = lambda p: p.startswith("choice")
+ #: The Package type handled by this Source class. The ``type``
+ #: attribute of Package entries will be set to the value ``ptype``
+ #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`.
+ ptype = None
+
def __init__(self, basepath, xsource, setup):
"""
:param basepath: The base filesystem path under which cache
@@ -390,18 +395,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
#. First, if the map contains a ``component`` key, use that as
the name.
-
#. If not, then try to match the repository URL against
:attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`. If
that succeeds, use the first matched group; additionally,
if the Source tag that describes this repo is contained in
a ``<Group>`` tag, prepend that to the name.
-
#. If :attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`
does not match the repository, and the Source tag that
describes this repo is contained in a ``<Group>`` tag, use
the name of the group.
-
#. Failing that, use the full URL to this repository, with the
protocol and trailing slash stripped off if possible.
@@ -512,9 +514,12 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
return os.path.join(self.basepath, url.replace('/', '@'))
def read_files(self):
- """ Read and parse locally downloaded metadata files. This
- function should call :func:`process_files` as its final
- step. This should also populate :attr:`pkgnames`. """
+ """ Read and parse locally downloaded metadata files and
+ populates
+ :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.pkgnames`. Should
+ call
+ :func:`Bcfg2.Server.Plugins.Packages.Source.Source.process_files`
+ as its final step."""
pass
def process_files(self, deps, prov):
@@ -672,10 +677,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
def is_package(self, metadata, package): # pylint: disable=W0613
""" Return True if a package is a package, False otherwise.
- The base implementation returns True if any Source object's
- :func:`Bcfg2.Server.Plugins.Packages.Source.Source.is_package`
- returns True.
-
:param package: The name of the package
:type package: string
:returns: bool
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index c1dd80689..1aa0128c4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -1,3 +1,55 @@
+""" Yum backend for :mod:`Bcfg2.Server.Plugins.Packages`. This module
+is the most complex backend because it has to handle Yum sources
+without yum Python libraries, with yum Python libraries, and Pulp
+sources. (See :ref:`native-yum-libraries` for details on using the
+yum Python libraries and :ref:`pulp-source-support` for details on
+Pulp sources.)
+
+.. _bcfg2-yum-helper:
+
+bcfg2-yum-helper
+~~~~~~~~~~~~~~~~
+
+If using the yum Python libraries, :class:`YumCollection` makes shell
+calls to an external command, ``bcfg2-yum-helper``, which performs the
+actual yum API calls. This is done because the yum libs have horrific
+memory leaks, and apparently the right way to get around that in
+long-running processes it to have a short-lived helper. This is how
+it's done by yum itself in ``yum-updatesd``, which is a long-running
+daemon that checks for and applies updates.
+
+.. _yum-pkg-objects:
+
+Package Objects
+~~~~~~~~~~~~~~~
+
+:class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` objects
+have the option to translate from some backend-specific representation
+of packages to XML entries; see :ref:`pkg-objects` for more
+information on this. If you are using the Python yum libraries,
+:class:`Bcfg2.Server.Plugins.Packages.Yum.YumCollection` opts to do
+this, using the yum tuple representation of packages, which is::
+
+ (<name>, <arch>, <epoch>, <version>, <release>)
+
+For shorthand this is occasionally abbrevated "naevr". Any datum that
+is not defined is ``None``. So a normal package entry that can be any
+version would be passed to :ref:`bcfg2-yum-helper` as::
+
+ ("somepackage", None, None, None, None)
+
+A package returned from the helper might look more like this::
+
+ ("somepackage", "x86_64", None, "1.2.3", "1.el6")
+
+We translate between this representation and the XML representation of
+packages with :func:`YumCollection.packages_from_entry` and
+:func:`YumCollection.packages_to_entry`.
+
+The Yum Backend
+~~~~~~~~~~~~~~~
+"""
+
import os
import re
import sys
@@ -8,29 +60,31 @@ import lxml.etree
from subprocess import Popen, PIPE
import Bcfg2.Server.Plugin
from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \
- ConfigParser, json
+ ConfigParser, json, any
from Bcfg2.Server.Plugins.Packages.Collection import Collection
from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \
fetch_url
logger = logging.getLogger(__name__)
+# pylint: disable=E0611
try:
from pulp.client.consumer.config import ConsumerConfig
from pulp.client.api.repository import RepositoryAPI
from pulp.client.api.consumer import ConsumerAPI
from pulp.client.api import server
- has_pulp = True
+ HAS_PULP = True
except ImportError:
- has_pulp = False
+ HAS_PULP = False
try:
import yum
- has_yum = True
+ HAS_YUM = True
except ImportError:
- has_yum = False
+ HAS_YUM = False
logger.info("Packages: No yum libraries found; forcing use of internal "
"dependency resolver")
+# pylint: enable=E0611
XP = '{http://linux.duke.edu/metadata/common}'
RP = '{http://linux.duke.edu/metadata/rpm}'
@@ -42,9 +96,18 @@ PULPCONFIG = None
def _setup_pulp(setup):
+ """ Connect to a Pulp server and pass authentication credentials.
+ This only needs to be called once, but multiple calls won't hurt
+ anything.
+
+ :param setup: A Bcfg2 options dict
+ :type setup: dict
+ :returns: :class:`pulp.client.api.server.PulpServer`
+ """
global PULPSERVER, PULPCONFIG
- if not has_pulp:
- msg = "Packages: Cannot create Pulp collection: Pulp libraries not found"
+ if not HAS_PULP:
+ msg = "Packages: Cannot create Pulp collection: Pulp libraries " + \
+ "not found"
logger.error(msg)
raise Bcfg2.Server.Plugin.PluginInitError(msg)
@@ -53,11 +116,12 @@ def _setup_pulp(setup):
username = setup.cfp.get("packages:pulp", "username")
password = setup.cfp.get("packages:pulp", "password")
except ConfigParser.NoSectionError:
- msg = "Packages: No [pulp] section found in Packages/packages.conf"
+ msg = "Packages: No [pulp] section found in bcfg2.conf"
logger.error(msg)
raise Bcfg2.Server.Plugin.PluginInitError(msg)
except ConfigParser.NoOptionError:
- msg = "Packages: Required option not found in Packages/packages.conf: %s" % sys.exc_info()[1]
+ msg = "Packages: Required option not found in bcfg2.conf: %s" % \
+ sys.exc_info()[1]
logger.error(msg)
raise Bcfg2.Server.Plugin.PluginInitError(msg)
@@ -74,8 +138,14 @@ def _setup_pulp(setup):
class YumCollection(Collection):
- #: YumCollections support package groups
- __package_groups__ = True
+ """ Handle collections of Yum sources. If we're using the yum
+ Python libraries, then this becomes a very full-featured
+ :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
+ object; if not, then it defers to the :class:`YumSource`
+ object.
+
+ .. private-include: _add_gpg_instances, _get_pulp_consumer
+ """
#: Options that are included in the [packages:yum] section of the
#: config but that should not be included in the temporary
@@ -87,23 +157,45 @@ class YumCollection(Collection):
self.keypath = os.path.join(self.basepath, "keys")
if self.use_yum:
- self.cachefile = os.path.join(self.cachepath,
+ #: Define a unique cache file for this collection to use
+ #: for cached yum metadata
+ self.cachefile = os.path.join(self.basepath,
"cache-%s" % self.cachekey)
if not os.path.exists(self.cachefile):
os.mkdir(self.cachefile)
+ #: The path to the server-side config file used when
+ #: resolving packages with the Python yum libraries
self.cfgfile = os.path.join(self.cachefile, "yum.conf")
self.write_config()
- if has_pulp and self.has_pulp_sources:
+ else:
+ self.cachefile = None
+
+ if HAS_PULP and self.has_pulp_sources:
_setup_pulp(self.setup)
self._helper = None
@property
+ def __package_groups__(self):
+ """ YumCollections support package groups only if
+ :attr:`use_yum` is True """
+ if self.use_yum:
+ return True
+ else:
+ return False
+
+ @property
def helper(self):
+ """ The full path to :file:`bcfg2-yum-helper`. First, we
+ check in the config file to see if it has been explicitly
+ specified; next we see if it's in $PATH (which we do by making
+ a call to it; I wish there was a way to do this without
+ forking, but apparently not); finally we check in /usr/sbin,
+ the default location. """
try:
return self.setup.cfp.get("packages:yum", "helper")
- except:
+ except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
pass
if not self._helper:
@@ -118,19 +210,30 @@ class YumCollection(Collection):
@property
def use_yum(self):
- return has_yum and self.setup.cfp.getboolean("packages:yum",
+ """ True if we should use the yum Python libraries, False
+ otherwise """
+ return HAS_YUM and self.setup.cfp.getboolean("packages:yum",
"use_yum_libraries",
default=False)
@property
def has_pulp_sources(self):
- """ see if there are any pulp sources to handle """
- for source in self:
- if source.pulp_id:
- return True
- return False
+ """ True if there are any Pulp sources to handle, False
+ otherwise """
+ return any(s.pulp_id for s in self)
+
+ @property
+ def cachefiles(self):
+ """ A list of the full path to all cachefiles used by this
+ collection."""
+ cachefiles = set(Collection.cachefiles(self))
+ if self.cachefile:
+ cachefiles.add(self.cachefile)
+ return list(cachefiles)
def write_config(self):
+ """ Write the server-side config file to :attr:`cfgfile` based
+ on the data from :func:`get_config`"""
if not os.path.exists(self.cfgfile):
yumconf = self.get_config(raw=True)
yumconf.add_section("main")
@@ -163,6 +266,17 @@ class YumCollection(Collection):
yumconf.write(open(self.cfgfile, 'w'))
def get_config(self, raw=False):
+ """ Get the yum configuration for this collection.
+
+ :param raw: Return a :class:`ConfigParser.SafeConfigParser`
+ object representing the configuration instead of a
+ string. This is useful if you need to modify the
+ config before writing it (as :func:`write_config`
+ does in order to produce a server-specific
+ configuration).
+ :type raw: bool
+ :returns: string or ConfigParser.SafeConfigParser """
+
config = ConfigParser.SafeConfigParser()
for source in self:
for url_map in source.url_map:
@@ -219,28 +333,46 @@ class YumCollection(Collection):
"Packages plugin\n\n" + buf.getvalue()
def build_extra_structures(self, independent):
- """ build list of gpg keys to be added to the specification by
- validate_structures() """
+ """ Add additional entries to the ``<Independent/>`` section
+ of the final configuration. This adds several kinds of
+ entries:
+
+ * For GPG keys, adds a ``Package`` entry that describes the
+ version and release of all expected ``gpg-pubkey`` packages;
+ and ``Path`` entries to copy all of the GPG keys to the
+ appropriate place on the client filesystem. Calls
+ :func:`_add_gpg_instances`.
+
+ * For Pulp Sources, adds a ``Path`` entry for the consumer
+ certificate; and ``Action`` entries to update the
+ consumer-side Pulp config if the consumer is newly
+ registered. Creates a new Pulp consumer from the Bcfg2
+ server as necessary.
+
+ :param independent: The XML tag to add extra entries to. This
+ is modified in place.
+ :type independent: lxml.etree._Element
+ """
needkeys = set()
for source in self:
for key in source.gpgkeys:
needkeys.add(key)
if len(needkeys):
- if has_yum:
- # this must be be has_yum, not use_yum, because
+ if HAS_YUM:
+ # this must be be HAS_YUM, not use_yum, because
# regardless of whether the user wants to use the yum
# resolver we want to include gpg key data
keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey",
type=self.ptype, origin='Packages')
else:
- self.logger.warning("GPGKeys were specified for yum sources in "
- "sources.xml, but no yum libraries were "
- "found")
+ self.logger.warning("GPGKeys were specified for yum sources "
+ "in sources.xml, but no yum libraries "
+ "were found")
self.logger.warning("GPG key version/release data cannot be "
"determined automatically")
- self.logger.warning("Install yum libraries, or manage GPG keys "
- "manually")
+ self.logger.warning("Install yum libraries, or manage GPG "
+ "keys manually")
keypkg = None
for key in needkeys:
@@ -260,7 +392,8 @@ class YumCollection(Collection):
keypath.text = kdata
# hook to add version/release info if possible
- self._add_gpg_instances(keypkg, kdata, localkey, remotekey)
+ self._add_gpg_instances(keypkg, localkey, remotekey,
+ keydata=kdata)
independent.append(keypath)
if keypkg is not None:
independent.append(keypkg)
@@ -290,6 +423,15 @@ class YumCollection(Collection):
crt.text = consumerapi.certificate(self.metadata.hostname)
def _get_pulp_consumer(self, consumerapi=None):
+ """ Get a Pulp consumer object for the client.
+
+ :param consumerapi: A Pulp ConsumerAPI object. If none is
+ passed, one will be instantiated.
+ :type consumerapi: pulp.client.api.consumer.ConsumerAPI
+ :returns: dict - the consumer. Returns None on failure
+ (including if there is no existing Pulp consumer for
+ this client.
+ """
if consumerapi is None:
consumerapi = ConsumerAPI()
consumer = None
@@ -304,19 +446,38 @@ class YumCollection(Collection):
err)
except:
err = sys.exc_info()[1]
- self.logger.error("Packages: Unknown error querying Pulp server: %s"
- % err)
+ self.logger.error("Packages: Unknown error querying Pulp server: "
+ "%s" % err)
return consumer
- def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey):
- """ add gpg keys to the specification to ensure they get
- installed """
- # this must be be has_yum, not use_yum, because regardless of
+ def _add_gpg_instances(self, keyentry, localkey, remotekey, keydata=None):
+ """ Add GPG keys instances to a ``Package`` entry. This is
+ called from :func:`build_extra_structures` to add GPG keys to
+ the specification.
+
+ :param keyentry: The ``Package`` entry to add key instances
+ to. This will be modified in place.
+ :type keyentry: lxml.etree._Element
+ :param localkey: The full path to the key file on the Bcfg2 server
+ :type localkey: string
+ :param remotekey: The full path to the key file on the client.
+ (If they key is not yet on the client, this
+ will be the full path to where the key file
+ will go eventually.)
+ :type remotekey: string
+ :param keydata: The contents of the key file. If this is not
+ provided, read the data from ``localkey``.
+ :type keydata: string
+ """
+ # this must be be HAS_YUM, not use_yum, because regardless of
# whether the user wants to use the yum resolver we want to
# include gpg key data
- if not has_yum:
+ if not HAS_YUM:
return
+ if keydata is None:
+ keydata = open(localkey).read()
+
try:
kinfo = yum.misc.getgpgkeyinfo(keydata)
version = yum.misc.keyIdToRPMVer(kinfo['keyid'])
@@ -331,47 +492,29 @@ class YumCollection(Collection):
self.logger.error("Packages: Could not read GPG key %s: %s" %
(localkey, err))
- def is_package(self, package):
- if not self.use_yum:
- return Collection.is_package(self, package)
- elif isinstance(package, tuple):
- if package[1] is None and package[2] == (None, None, None):
- package = package[0]
- else:
- return None
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("is_package", package)
-
- def is_virtual_package(self, package):
- if not self.use_yum:
- return Collection.is_virtual_package(self, package)
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("is_virtual_package", package)
-
- def get_deps(self, package):
- if not self.use_yum:
- return Collection.get_deps(self, package)
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("get_deps", package)
-
- def get_provides(self, required, all=False, silent=False):
- if not self.use_yum:
- return Collection.get_provides(self, required)
- else:
- # this should really never get called; it's just provided
- # for API completeness
- return self.call_helper("get_provides", required)
-
def get_groups(self, grouplist):
+ """ If using the yum libraries, given a list of package group
+ names, return a dict of ``<group name>: <list of packages>``.
+ This is much faster than implementing
+ :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_group`,
+ since we have to make a call to the bcfg2 Yum helper, and each
+ time we do that we make another call to yum, which means we
+ set up yum metadata from the cache (hopefully) each time. So
+ resolving ten groups once is much faster than resolving one
+ group ten times.
+
+ If you are using the builtin yum parser, this raises a warning
+ and returns an empty dict.
+
+ :param grouplist: The list of groups to query
+ :type grouplist: list of strings - group names
+ :returns: dict of ``<group name>: <list of packages>``
+
+ In this implementation the packages may be strings or tuples.
+ See :ref:`yum-pkg-objects` for more information. """
if not self.use_yum:
- self.logger.warning("Packages: Package groups are not supported by "
- "Bcfg2's internal Yum dependency generator")
+ self.logger.warning("Packages: Package groups are not supported "
+ "by Bcfg2's internal Yum dependency generator")
return dict()
if not grouplist:
@@ -387,18 +530,15 @@ class YumCollection(Collection):
return self.call_helper("get_groups", gdicts)
- def get_group(self, group, ptype="default"):
- if not self.use_yum:
- self.logger.warning("Packages: Package groups are not supported by "
- "Bcfg2's internal Yum dependency generator")
- return []
-
- if group.startswith("@"):
- group = group[1:]
-
- return self.call_helper("get_group", dict(group=group, type=ptype))
-
def packages_from_entry(self, entry):
+ """ When using the Python yum libraries, convert a Package
+ entry to a list of package tuples. See :ref:`yum-pkg-objects`
+ and :ref:`pkg-objects` for more information on this process.
+
+ :param entry: The Package entry to convert
+ :type entry: lxml.etree._Element
+ :returns: list of tuples
+ """
rv = set()
name = entry.get("name")
@@ -425,6 +565,24 @@ class YumCollection(Collection):
return list(rv)
def packages_to_entry(self, pkglist, entry):
+ """ When using the Python yum libraries, convert a list of
+ package tuples to a Package entry. See :ref:`yum-pkg-objects`
+ and :ref:`pkg-objects` for more information on this process.
+
+ If pkglist contains only one package, then its data is
+ converted to a single ``BoundPackage`` entry that is added as
+ a subelement of ``entry``. If pkglist contains more than one
+ package, then a parent ``BoundPackage`` entry is created and
+ child ``Instance`` entries are added to it.
+
+ :param pkglist: A list of package tuples to convert to an XML
+ Package entry
+ :type pkglist: list of tuples
+ :param entry: The base XML entry to add Package entries to.
+ This is modified in place.
+ :type entry: lxml.etree._Element
+ :returns: None
+ """
def _get_entry_attrs(pkgtup):
attrs = dict(version=self.setup.cfp.get("packages",
"version",
@@ -464,6 +622,18 @@ class YumCollection(Collection):
lxml.etree.SubElement(entry, 'BoundPackage', **attrs)
def get_new_packages(self, initial, complete):
+ """ Compute the difference between the complete package list
+ (as returned by :func:`complete`) and the initial package list
+ computed from the specification, allowing for package tuples.
+ See :ref:`yum-pkg-objects` and :ref:`pkg-objects` for more
+ information on this process.
+
+ :param initial: The initial package list
+ :type initial: set of strings, but see :ref:`pkg-objects`
+ :param complete: The final package list
+ :type complete: set of strings, but see :ref:`pkg-objects`
+ :return: set of tuples
+ """
initial_names = []
for pkg in initial:
if isinstance(pkg, tuple):
@@ -477,6 +647,22 @@ class YumCollection(Collection):
return new
def complete(self, packagelist):
+ """ Build a complete list of all packages and their dependencies.
+
+ When using the Python yum libraries, this defers to the
+ :ref:`bcfg2-yum-helper`; when using the builtin yum parser,
+ this defers to
+ :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.complete`.
+
+ :param packagelist: Set of initial packages computed from the
+ specification.
+ :type packagelist: set of strings, but see :ref:`pkg-objects`
+ :returns: tuple of sets - The first element contains a set of
+ strings (but see :ref:`pkg-objects`) describing the
+ complete package list, and the second element is a
+ set of symbols whose dependencies could not be
+ resolved.
+ """
if not self.use_yum:
return Collection.complete(self, packagelist)
@@ -500,11 +686,21 @@ class YumCollection(Collection):
return set(), set()
def call_helper(self, command, input=None):
- """ Make a call to bcfg2-yum-helper. The yum libs have
+ """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have
horrific memory leaks, so apparently the right way to get
around that in long-running processes it to have a short-lived
helper. No, seriously -- check out the yum-updatesd code.
- It's pure madness. """
+ It's pure madness.
+
+ :param command: The :ref:`bcfg2-yum-helper` command to call.
+ :type command: string
+ :param input: The input to pass to ``bcfg2-yum-helper`` on
+ stdin. If this is None, no input will be given
+ at all.
+ :type input: Any JSON-encodable data structure.
+ :returns: Varies depending on the return value of the
+ ``bcfg2-yum-helper`` command.
+ """
cmd = [self.helper, "-c", self.cfgfile]
verbose = self.debug_flag or self.setup['verbose']
if verbose:
@@ -540,6 +736,23 @@ class YumCollection(Collection):
return None
def setup_data(self, force_update=False):
+ """ Do any collection-level data setup tasks. This is called
+ when sources are loaded or reloaded by
+ :class:`Bcfg2.Server.Plugins.Packages.Packages`.
+
+ If the builtin yum parsers are in use, this defers to
+ :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.setup_data`.
+ If using the yum Python libraries, this cleans up cached yum
+ metadata, regenerates the server-side yum config (in order to
+ catch any new sources that have been added to this server),
+ and then cleans up cached yum metadata again, in case the new
+ config has any preexisting cache.
+
+ :param force_update: Ignore all local cache and setup data
+ from its original upstream sources (i.e.,
+ the package repositories)
+ :type force_update: bool
+ """
if not self.use_yum:
return Collection.setup_data(self, force_update)
@@ -556,14 +769,29 @@ class YumCollection(Collection):
class YumSource(Source):
+ """ Handle yum sources """
+
+ #: :ref:`server-plugins-generators-packages-magic-groups` for
+ #: ``YumSource`` are "yum", "redhat", "centos", and "fedora"
basegroups = ['yum', 'redhat', 'centos', 'fedora']
+
+ #: YumSource sets the ``type`` on Package entries to "yum"
ptype = 'yum'
+
+ #: By default,
+ #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` filters
+ #: out unknown packages that start with "choice", but that doesn't
+ #: mean anything to Yum or RPM. Instead, we filter out unknown
+ #: packages that start with "rpmlib", although this is likely
+ #: legacy behavior; that would seem to indicate that a package
+ #: required some RPM feature that isn't provided, which is a bad
+ #: thing. This should probably go away at some point.
unknown_filter = lambda u: u.startswith("rpmlib")
def __init__(self, basepath, xsource, setup):
Source.__init__(self, basepath, xsource, setup)
self.pulp_id = None
- if has_pulp and xsource.get("pulp_id"):
+ if HAS_PULP and xsource.get("pulp_id"):
self.pulp_id = xsource.get("pulp_id")
_setup_pulp(self.setup)
@@ -585,11 +813,12 @@ class YumSource(Source):
raise SourceInitError(msg)
except socket.error:
err = sys.exc_info()[1]
- raise SourceInitError("Could not contact Pulp server: %s" % err)
+ raise SourceInitError("Could not contact Pulp server: %s" %
+ err)
except:
err = sys.exc_info()[1]
- raise SourceInitError("Unknown error querying Pulp server: %s" %
- err)
+ raise SourceInitError("Unknown error querying Pulp server: %s"
+ % err)
self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'],
self.repo['relative_path'])
self.arches = [self.repo['arch']]
@@ -601,14 +830,20 @@ class YumSource(Source):
for x in ['global'] + self.arches])
self.needed_paths = set()
self.file_to_arch = dict()
+ __init__.__doc__ = Source.__init__.__doc__
@property
def use_yum(self):
- return has_yum and self.setup.cfp.getboolean("packages:yum",
+ """ True if we should use the yum Python libraries, False
+ otherwise """
+ return HAS_YUM and self.setup.cfp.getboolean("packages:yum",
"use_yum_libraries",
default=False)
def save_state(self):
+ """ If using the builtin yum parser, save state to
+ :attr:`cachefile`. If using the Python yum libraries, yum
+ handles caching and state and this method is a no-op."""
if not self.use_yum:
cache = open(self.cachefile, 'wb')
cPickle.dump((self.packages, self.deps, self.provides,
@@ -616,6 +851,9 @@ class YumSource(Source):
cache.close()
def load_state(self):
+ """ If using the builtin yum parser, load saved state from
+ :attr:`cachefile`. If using the Python yum libraries, yum
+ handles caching and state and this method is a no-op."""
if not self.use_yum:
data = open(self.cachefile)
(self.packages, self.deps, self.provides,
@@ -623,10 +861,26 @@ class YumSource(Source):
@property
def urls(self):
+ """ A list of URLs to the base metadata file for each
+ repository described by this source. """
return [self._get_urls_from_repodata(m['url'], m['arch'])
for m in self.url_map]
def _get_urls_from_repodata(self, url, arch):
+ """ When using the builtin yum parser, given the base URL of a
+ repository, return the URLs of the various repo metadata files
+ needed to get package data from the repo.
+
+ If using the yum Python libraries, this just returns ``url``
+ as it was passed in, but should realistically not be called.
+
+ :param url: The base URL to the repository (i.e., the
+ directory that contains the ``repodata/`` directory)
+ :type url: string
+ :param arch: The architecture of the directory.
+ :type arch: string
+ :return: list of strings - URLs to metadata files
+ """
if self.use_yum:
return [url]
@@ -650,8 +904,8 @@ class YumSource(Source):
xdata = lxml.etree.XML(repomd)
except lxml.etree.XMLSyntaxError:
err = sys.exc_info()[1]
- self.logger.error("Packages: Failed to process metadata at %s: %s" %
- (rmdurl, err))
+ self.logger.error("Packages: Failed to process metadata at %s: %s"
+ % (rmdurl, err))
return []
urls = []
@@ -664,6 +918,11 @@ class YumSource(Source):
return urls
def read_files(self):
+ """ When using the builtin yum parser, read and parse locally
+ downloaded metadata files. This diverges from the stock
+ :func:`Bcfg2.Server.Plugins.Packages.Source.Source.read_files`
+ quite a bit. """
+
# we have to read primary.xml first, and filelists.xml afterwards;
primaries = list()
filelists = list()
@@ -740,14 +999,15 @@ class YumSource(Source):
self.provides[arch][prov] = list()
self.provides[arch][prov].append(pkgname)
- def is_package(self, metadata, item):
+ def is_package(self, metadata, package):
arch = [a for a in self.arches if a in metadata.groups]
if not arch:
return False
- return ((item in self.packages['global'] or
- item in self.packages[arch[0]]) and
- item not in self.blacklist and
- (len(self.whitelist) == 0 or item in self.whitelist))
+ return ((package in self.packages['global'] or
+ package in self.packages[arch[0]]) and
+ package not in self.blacklist and
+ (len(self.whitelist) == 0 or package in self.whitelist))
+ is_package.__doc__ = Source.is_package.__doc__
def get_vpkgs(self, metadata):
if self.use_yum:
@@ -760,6 +1020,7 @@ class YumSource(Source):
for filename, pkgs in list(fmdata.items()):
rv[filename] = pkgs
return rv
+ get_vpkgs.__doc__ = Source.get_vpkgs.__doc__
def filter_unknown(self, unknown):
if self.use_yum:
@@ -777,12 +1038,25 @@ class YumSource(Source):
unknown.difference_update(filtered)
else:
Source.filter_unknown(self, unknown)
+ filter_unknown.__doc__ = Source.filter_unknown.__doc__
def setup_data(self, force_update=False):
if not self.use_yum:
Source.setup_data(self, force_update=force_update)
+ setup_data.__doc__ = \
+ "``setup_data`` is only used by the builtin yum parser. " + \
+ Source.setup_data.__doc__
def get_repo_name(self, url_map):
+ """ Try to find a sensible name for a repository. First use a
+ repository's Pulp ID, if it has one; if not, then defer to
+ :class:`Bcfg2.Server.Plugins.Packages.Source.Source.get_repo_name`
+
+ :param url_map: A single :attr:`url_map` dict, i.e., any
+ single element of :attr:`url_map`.
+ :type url_map: dict
+ :returns: string - the name of the repository.
+ """
if self.pulp_id:
return self.pulp_id
else:
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index f7932dd75..c74aa77d9 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -55,9 +55,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
default=os.path.join(self.data, 'cache'))
#: Where Packages should store downloaded GPG key files
- self.keypath = \
- self.core.setup.cfp.get("packages", "keycache",
- default=os.path.join(self.data, 'keys'))
+ self.keypath = os.path.join(self.cachepath, 'keys')
if not os.path.exists(self.keypath):
# create key directory if needed
os.makedirs(self.keypath)
diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper
index 6d32f1efc..a9c620496 100755
--- a/src/sbin/bcfg2-yum-helper
+++ b/src/sbin/bcfg2-yum-helper
@@ -14,6 +14,7 @@ from Bcfg2.Compat import json
LOGGER = None
+
def get_logger(verbose=0):
""" set up logging according to the verbose level given on the
command line """
@@ -32,6 +33,7 @@ def get_logger(verbose=0):
LOGGER.addHandler(syslog)
return LOGGER
+
def pkg_to_tuple(package):
""" json doesn't distinguish between tuples and lists, but yum
does, so we convert a package in list format to one in tuple
@@ -41,6 +43,7 @@ def pkg_to_tuple(package):
else:
return package
+
def pkgtup_to_string(package):
if package[3] in ['auto', 'any']:
return package[0]
@@ -77,22 +80,10 @@ class DepSolver(object):
groups = property(get_groups, set_groups)
- def is_package(self, package):
- if isinstance(package, tuple):
- if package[1] is None and package[2] == (None, None, None):
- pkgtup = (package[0], None, None, None, None)
- elif len(package) == 5:
- pkgtup = package
- else:
- pkgtup = (package, None, None, None, None)
- return bool(self.get_package_object(pkgtup, silent=True))
-
- def is_virtual_package(self, package):
- return bool(self.get_provides(package, silent=True))
-
def get_package_object(self, pkgtup, silent=False):
try:
- matches = yum.packageSack.packagesNewestByName(self.yumbase.pkgSack.searchPkgTuple(pkgtup))
+ matches = yum.packageSack.packagesNewestByName(
+ self.yumbase.pkgSack.searchPkgTuple(pkgtup))
except yum.Errors.PackageSackError:
if not silent:
self.logger.warning("Package '%s' not found" %
@@ -110,39 +101,6 @@ class DepSolver(object):
else:
return None
- def get_deps(self, package):
- pkg = self.get_package_object(package)
- deps = []
- if pkg:
- deps = set(pkg.requires)
- # filter out things the package itself provides
- deps.difference_update([dep for dep in deps
- if pkg.checkPrco('provides', dep)])
- else:
- self.logger.error("No package available: %s" %
- self.get_package_name(package))
- return deps
-
- def get_provides(self, required, all=False, silent=False):
- if not isinstance(required, tuple):
- required = (required, None, (None, None, None))
-
- try:
- prov = \
- self.yumbase.whatProvides(*required).returnNewestByNameArch()
- except yum.Errors.NoMoreMirrorsRepoError:
- err = sys.exc_info()[1]
- self.logger.error("Temporary failure loading metadata for %s: %s" %
- (self.get_package_name(required), err))
- return []
-
- if prov and not all:
- prov = self._filter_provides(prov)
- elif not prov and not silent:
- self.logger.error("No package provides %s" %
- self.get_package_name(required))
- return prov
-
def get_group(self, group, ptype="default"):
if group.startswith("@"):
group = group[1:]
@@ -172,35 +130,6 @@ class DepSolver(object):
self.logger.warning("Unknown group package type '%s'" % ptype)
return []
- def _filter_provides(self, providers):
- providers = [pkg for pkg in self._filter_arch(providers)]
- if len(providers) > 1:
- # go through each provider and make sure it's the newest
- # package of its name available. If we have multiple
- # providers, avoid installing old packages.
- #
- # For instance: on Fedora 14,
- # perl-Sub-WrapPackages-2.0-2.fc14 erroneously provided
- # perl(lib), which should not have been provided;
- # perl(lib) is provided by the "perl" package. The bogus
- # provide was removed in perl-Sub-WrapPackages-2.0-4.fc14,
- # but if we just queried to resolve the "perl(lib)"
- # dependency, we'd get both packages. By performing this
- # check, we learn that there's a newer
- # perl-Sub-WrapPackages available, so it can't be the best
- # provider of perl(lib).
- rv = []
- for pkg in providers:
- found = self.get_package_object(pkg.pkgtup)
- if found == pkg or found.pkgtup == pkg.pkgtup:
- rv.append(pkg)
- else:
- self.logger.debug("Skipping %s, not newest (%s)" %
- (pkg, found))
- else:
- rv = providers
- return rv
-
def _filter_arch(self, packages):
matching = []
for pkg in packages:
@@ -243,8 +172,8 @@ class DepSolver(object):
unknown.add(pkg)
else:
if self.yumbase.tsInfo.exists(pkgtup=po.pkgtup):
- self.logger.debug("%s added to transaction multiple times" %
- po)
+ self.logger.debug("%s added to transaction multiple times"
+ % po)
else:
self.logger.debug("Adding %s to transaction" % po)
self.yumbase.tsInfo.addInstall(po)
@@ -269,7 +198,8 @@ class DepSolver(object):
def main():
parser = OptionParser()
parser.add_option("-c", "--config", help="Config file")
- parser.add_option("-v", "--verbose", help="Verbosity level", action="count")
+ parser.add_option("-v", "--verbose", help="Verbosity level",
+ action="count")
(options, args) = parser.parse_args()
logger = get_logger(options.verbose)
try:
@@ -293,19 +223,6 @@ def main():
for p in data['packages']])
print(json.dumps(dict(packages=list(packages),
unknown=list(unknown))))
- elif cmd == "is_virtual_package":
- package = pkg_to_tuple(json.loads(sys.stdin.read()))
- print(json.dumps(bool(depsolver.get_provides(package, silent=True))))
- elif cmd == "get_deps" or cmd == "get_provides":
- package = pkg_to_tuple(json.loads(sys.stdin.read()))
- print(json.dumps([p.name for p in getattr(depsolver, cmd)(package)]))
- elif cmd == "get_group":
- data = json.loads(sys.stdin.read())
- if "type" in data:
- packages = depsolver.get_group(data['group'], ptype=data['type'])
- else:
- packages = depsolver.get_group(data['group'])
- print(json.dumps(list(packages)))
elif cmd == "get_groups":
data = json.loads(sys.stdin.read())
rv = dict()
@@ -317,10 +234,7 @@ def main():
packages = depsolver.get_group(gdata['group'])
rv[gdata['group']] = list(packages)
print(json.dumps(rv))
- elif cmd == "is_package":
- package = pkg_to_tuple(json.loads(sys.stdin.read()))
- print(json.dumps(getattr(depsolver, cmd)(package)))
-
+
if __name__ == '__main__':
sys.exit(main())