From 92bc95f86a834b2853c77bbbfa1c0021213e1e17 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 19 Sep 2012 11:40:52 -0400 Subject: documented Packages base class --- .../Bcfg2/Server/Plugins/Packages/Collection.py | 32 ++- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 10 +- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 225 +++++++++++++++++---- 3 files changed, 210 insertions(+), 57 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 5074cc389..0460038c2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -428,7 +428,9 @@ class _Collection(list, Bcfg2.Server.Plugin.Debuggable): pass def get_additional_data(self): - """ Get additional Connector data to be supplied to + """ Get additional + :class:`Bcfg2.Server.Plugin.interfaces.Connector` data to be + supplied to :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data` (and thence to client metadata objects). @@ -462,9 +464,8 @@ class _Collection(list, Bcfg2.Server.Plugin.Debuggable): def packages_from_entry(self, entry): """ Given a Package or BoundPackage entry, get a list of the package(s) described by it in a format appropriate for passing - to :func:`Bcfg2.Server.Plugins.Packages.Packages.complete`. - By default, that's just the name; only the - :mod:`Bcfg2.Server.Plugins.Packages.Yum` backend supports + to :func:`complete`. By default, that's just the name; only + the :mod:`Bcfg2.Server.Plugins.Packages.Yum` backend supports versions or other extended data. See :ref:`pkg-objects` for more details. @@ -476,14 +477,13 @@ class _Collection(list, Bcfg2.Server.Plugin.Debuggable): def packages_to_entry(self, pkglist, entry): """ Given a list of package objects as returned by - :func:`packages_from_entry` or - :func:`Bcfg2.Server.Plugins.Packages.Packages.complete`, - return an XML tree describing the BoundPackage entries that - should be included in the client configuration. See - :ref:`pkg-objects` for more details. + :func:`packages_from_entry` or :func:`complete`, return an XML + tree describing the BoundPackage entries that should be + included in the client configuration. See :ref:`pkg-objects` + for more details. :param pkglist: A list of packages as returned by - :func:`Bcfg2.Server.Plugins.Packages.Packages.complete` + :func:`complete` :type pkglist: list of strings, but see :ref:`pkg-objects` :param entry: The base XML entry to add all of the Package entries to. This should be modified in place. @@ -498,13 +498,11 @@ class _Collection(list, Bcfg2.Server.Plugin.Debuggable): def get_new_packages(self, initial, complete): """ Compute the difference between the complete package list - (as returned by - :func:`Bcfg2.Server.Plugins.Packages.Packages.complete`) and - the initial package list computed from the specification. - This is necessary because the format may be different between - the two lists due to :func:`packages_to_entry` and - :func:`packages_from_entry`. See :ref:`pkg-objects` for more - details. + (as returned by :func:`complete`) and the initial package list + computed from the specification. This is necessary because + the format may be different between the two lists due to + :func:`packages_to_entry` and :func:`packages_from_entry`. See + :ref:`pkg-objects` for more details. :param initial: The initial package list :type initial: set of strings, but see :ref:`pkg-objects` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index d73a942c4..273f6cbb4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -569,10 +569,12 @@ class Source(Bcfg2.Server.Plugin.Debuggable): self.save_state() def filter_unknown(self, unknown): - """ After :func:`complete`, filter out packages that appear in - the list of unknown packages but should not be presented to - the user. :attr:`unknown_filter` is called to assess whether - or not a package is expected to be unknown. + """ After + :func:`Bcfg2.Server.Plugins.Packages.Collection._Collection.complete`, + filter out packages that appear in the list of unknown + packages but should not be presented to the user. + :attr:`unknown_filter` is called to assess whether or not a + package is expected to be unknown. :param unknown: A set of unknown packages. The set should be modified in place. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 06b1b78c0..083bcd5e8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,10 @@ from Bcfg2.Compat import ConfigParser, urlopen from Bcfg2.Server.Plugins.Packages import Collection from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +#: The default path for generated yum configs YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" + +#: The default path for generated apt configs APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2" @@ -18,8 +21,22 @@ class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.ClientRunHooks): - name = 'Packages' + """ Packages resolves Package entries on the Bcfg2 server in order + to present a complete list of Package entries to the client in + order to determine the completeness of the client configuration. + It does so by delegating control of package version information to + a number of backends, which may parse repository metadata directly + or defer to package manager libraries for truly dynamic + resolution. + + .. private-include: _build_packages, _get_collection""" + + #: Packages is an alternative to + #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] + + #: Packages exposes two additional XML-RPC calls, :func:`Refresh` + #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core, datastore): @@ -43,14 +60,21 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.core.setup) + __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def toggle_debug(self): rv = Bcfg2.Server.Plugin.Plugin.toggle_debug(self) self.sources.toggle_debug() return rv + toggle_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.toggle_debug.__doc__ @property def disableResolver(self): + """ Report the state of the resolver. This can be disabled in + the configuration. Note that disabling metadata (see + :attr:`disableMetaData`) implies disabling the resolver. + + This property cannot be set. """ if self.disableMetaData: # disabling metadata without disabling the resolver Breaks # Things @@ -64,11 +88,16 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # "disabled", which are not handled according to the # Python docs but appear to be handled properly by # ConfigParser in at least some versions - return self.core.setup.cfp.get("packages", "resolver", - default="enabled").lower() == "disabled" + return self.core.setup.cfp.get( + "packages", + "resolver", + default="enabled").lower() == "disabled" @property def disableMetaData(self): + """ Report whether or not metadata processing is enabled. + + This property cannot be set. """ try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): @@ -76,11 +105,20 @@ class Packages(Bcfg2.Server.Plugin.Plugin, except ValueError: # for historical reasons we also accept "enabled" and # "disabled" - return self.core.setup.cfp.get("packages", "metadata", - default="enabled").lower() == "disabled" + return self.core.setup.cfp.get( + "packages", + "metadata", + default="enabled").lower() == "disabled" def create_config(self, entry, metadata): - """ create yum/apt config for the specified host """ + """ Create yum/apt config for the specified client. + + :param entry: The base entry to bind. This will be modified + in place. + :type entry: lxml.etree._Element + :param metadata: The client to create the config for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ attrib = dict(encoding='ascii', owner='root', group='root', @@ -94,6 +132,21 @@ class Packages(Bcfg2.Server.Plugin.Plugin, entry.attrib.__setitem__(key, value) def HandleEntry(self, entry, metadata): + """ Bind configuration entries. ``HandleEntry`` handles + entries two different ways: + + * All ``Package`` entries have their ``version`` and ``type`` + attributes set according to the appropriate + :class:`Bcfg2.Server.Plugins.Packages.Collection._Collection` + object for this client. + * ``Path`` entries are delegated to :func:`create_config` + + :param entry: The entry to bind + :type entry: lxml.etree._Element + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: lxml.etree._Element - The fully bound entry + """ if entry.tag == 'Package': collection = self._get_collection(metadata) entry.set('version', self.core.setup.cfp.get("packages", @@ -101,17 +154,28 @@ class Packages(Bcfg2.Server.Plugin.Plugin, default="auto")) entry.set('type', collection.ptype) elif entry.tag == 'Path': - if (entry.get("name") == \ - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == \ - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): - self.create_config(entry, metadata) + self.create_config(entry, metadata) + return entry def HandlesEntry(self, entry, metadata): + """ Determine if the given entry can be handled. Packages + handles two kinds of entries: + + * ``Package`` entries are handled if the client has any + sources at all. + * ``Path`` entries are handled if they match the paths that + are handled by a backend that can produce client + configurations, e.g., :attr:`YUM_CONFIG_DEFAULT`, + :attr:`APT_CONFIG_DEFAULT`, or the overridden value of + either of those from the configuration. + + :param entry: The entry to bind + :type entry: lxml.etree._Element + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: bool - Whether or not this plugin can handle the entry + :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` + """ if entry.tag == 'Package': if self.core.setup.cfp.getboolean("packages", "magic_groups", default=True): @@ -134,12 +198,32 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return False def validate_structures(self, metadata, structures): - '''Ensure client configurations include all needed prerequisites - - Arguments: - metadata - client metadata instance - structures - a list of structure-stage entry combinations - ''' + """ Do the real work of Packages. This does two things: + + #. Given the full list of all packages that apply to this + client from the specification, calls + :func:`_build_packages` to resolve dependencies, determine + unknown packages (i.e., those that are not in any + repository that applies to this client), and build a + complete package list. + + #. Calls + :func:`Bcfg2.Server.Plugins.Packages.Collection._Collection.build_extra_structures` + to add any other extra data required by the backend (e.g., + GPG keys) + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param structures: A list of lxml.etree._Element objects + describing the structures (i.e., bundles) + for this client. This can be modified in + place. + :type structures: list of lxml.etree._Element objects + :returns: None + """ collection = self._get_collection(metadata) indep = lxml.etree.Element('Independent') self._build_packages(metadata, indep, structures, @@ -149,8 +233,26 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def _build_packages(self, metadata, independent, structures, collection=None): - """ build list of packages that need to be included in the - specification by validate_structures() """ + """ Perform dependency resolution and build the complete list + of packages that need to be included in the specification by + :func:`validate_structures`, based on the initial list of + packages. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param independent: The XML tag to add package entries + generated by dependency resolution to. + This will be modified in place. + :type independent: lxml.etree._Element + :param structures: A list of lxml.etree._Element objects + describing the structures (i.e., bundles) + for this client + :type structures: list of lxml.etree._Element objects + :param collection: The collection of sources for this client. + If none is given, one will be created with + :func:`_get_collection` + :type collection: Bcfg2.Server.Plugins.Packages.Collection._Collection + """ if self.disableResolver: # Config requests no resolver return @@ -182,7 +284,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, xml_declaration=False).decode('UTF-8')) gpkgs = collection.get_groups(groups) - for group, pkgs in gpkgs.items(): + for pkgs in gpkgs.values(): base.update(pkgs) base.update(initial | essential) @@ -200,29 +302,39 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection.packages_to_entry(newpkgs, independent) def Refresh(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and download sources\n''' + """ Packages.Refresh() => True|False + + Reload configuration specification and download sources """ self._load_config(force_update=True) return True def Reload(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and sources\n''' + """ Packages.Refresh() => True|False + + Reload configuration specification and sources """ self._load_config() return True def _load_config(self, force_update=False): - ''' + """ Load the configuration data and setup sources - Keyword args: - force_update Force downloading repo data - ''' + :param force_update: Ignore all locally cached and downloaded + data and fetch the metadata anew from the + upstream repository. + :type force_update: bool + """ self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): - """ Load sources from the config """ + """ Load sources from the config, downloading if necessary. + + :param force_update: Ignore all locally cached and downloaded + data and fetch the metadata anew from the + upstream repository. + :type force_update: bool + """ self.sentinels = set() cachefiles = set() @@ -252,7 +364,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): - """ Load gpg keys from the config """ + """ Load GPG keys from the config, downloading if necessary. + + :param force_update: Ignore all locally cached and downloaded + data and fetch the metadata anew from the + upstream repository. + :type force_update: bool + """ keyfiles = [] keys = [] for source in self.sources: @@ -274,18 +392,53 @@ class Packages(Bcfg2.Server.Plugin.Plugin, os.unlink(kfile) def _get_collection(self, metadata): + """ Get a + :class:`Bcfg2.Server.Plugins.Packages.Collection._Collection` + object for this client. + + :param metadata: The client metadata to get a Collection for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: Bcfg2.Server.Plugins.Packages.Collection._Collection + """ return Collection.Collection(metadata, self.sources, self.data, debug=self.debug_flag) def get_additional_data(self, metadata): + """ Return additional data for the given client. This will be + a dict containing a single key, ``sources``, whose value is a + list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection._Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :return: dict of lists of ``url_map`` data + """ collection = self._get_collection(metadata) return dict(sources=collection.get_additional_data()) def end_client_run(self, metadata): - """ clear the collection cache for this client, which must - persist only the duration of a client run""" + """ Hook to clear the cache for this client at + :attr:`Bcfg2.Server.Plugins.Packages.Collection.CLIENTS`, + which must persist only the duration of a client run. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ if metadata.hostname in Collection.CLIENTS: del Collection.CLIENTS[metadata.hostname] def end_statistics(self, metadata): + """ Hook to clear the cache for this client at + :attr:`Bcfg2.Server.Plugins.Packages.Collection.CLIENTS` once + statistics are processed to ensure that a stray cached + :class:`Bcfg2.Server.Plugins.Packages.Collection._Collection` + object is not built during statistics and preserved until a + subsequent client run. + + :param metadata: The client metadata + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ self.end_client_run(metadata) -- cgit v1.2.3-1-g7c22