From 208f29ee586b536b4e6019024899206bf4f7a3cc Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 13 Jun 2013 00:50:46 +0200 Subject: Plugins/Packages: ability to overwrite recommended flag per package --- schemas/pkgtype.xsd | 9 +++++ src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 21 +++++++---- .../Bcfg2/Server/Plugins/Packages/Collection.py | 13 ++++--- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 42 ++++++++++++++++++---- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 4 +-- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 7 +++- 6 files changed, 75 insertions(+), 21 deletions(-) diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd index 18eda88ab..e301f30b6 100644 --- a/schemas/pkgtype.xsd +++ b/schemas/pkgtype.xsd @@ -54,6 +54,15 @@ + + + + Whether also th recommended packages should be installed. + This is currently only used with the :ref:`APT + <client-tools-apt>` driver. + + + diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index a82a183d8..009395e8f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -76,10 +76,8 @@ class AptSource(Source): def read_files(self): bdeps = dict() + brecs = dict() bprov = dict() - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') for fname in self.files: if not self.rawurl: barch = [x @@ -91,6 +89,7 @@ class AptSource(Source): barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() + brecs[barch] = dict() bprov[barch] = dict() try: reader = gzip.GzipFile(fname) @@ -105,9 +104,10 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: self.essentialpkgs.add(pkgname) - elif words[0] in depfnames: + elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): if '|' in dep: @@ -118,17 +118,24 @@ class AptSource(Source): barch, vindex) vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) + + if words[0] == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) + if words[0] == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': for pkg in words[1].split(','): dname = pkg.rstrip().strip() if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index b25cb0fc4..d30b3dd95 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -308,7 +308,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package): + def get_deps(self, package, recs=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -318,9 +318,14 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): :type package: string :returns: list of strings, but see :ref:`pkg-objects` """ + recommended = None + if recs and package in recs: + recommended = recs[package] + for source in self: if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) + return source.get_deps(self.metadata, package, recommended) + return [] def get_essential(self): @@ -500,7 +505,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return list(complete.difference(initial)) @Bcfg2.Server.Plugin.track_statistics() - def complete(self, packagelist): # pylint: disable=R0912,R0914 + def complete(self, packagelist, recommended=None): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the @@ -564,7 +569,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current) + deps = self.get_deps(current, recommended) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 22073493c..55d12c400 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -256,6 +256,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + #: A dict of ```` -> ````. This will not necessarily be populated. + self.recommends = dict() + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) @@ -321,7 +325,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: cPickle.UnpicklingError - If the saved data is corrupt """ data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, - self.essentialpkgs) = cPickle.load(data) + self.essentialpkgs, self.recommends) = cPickle.load(data) def save_state(self): """ Save state to :attr:`cachefile`. If caching and @@ -329,7 +333,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 does not need to be implemented. """ cache = open(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides, - self.essentialpkgs), cache, 2) + self.essentialpkgs, self.recommends), cache, 2) cache.close() @Bcfg2.Server.Plugin.track_statistics() @@ -524,13 +528,13 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 as its final step.""" pass - def process_files(self, dependencies, provides): + def process_files(self, dependencies, provides, recommends=dict()): """ Given dicts of depends and provides generated by :func:`read_files`, this generates :attr:`deps` and :attr:`provides` and calls :func:`save_state` to save the cached data to disk. - Both arguments are dicts of dicts of lists. Keys are the + All arguments are dicts of dicts of lists. Keys are the arches of packages contained in this source; values are dicts whose keys are package names and values are lists of either dependencies for each package the symbols provided by each @@ -542,14 +546,20 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :param provides: A dict of symbols provided by packages in this repository. :type provides: dict; see above. + :param recommends: A dict of recommended dependencies + found for this source. + :type recommends: dict; see above. """ self.deps['global'] = dict() + self.recommends['global'] = dict() self.provides['global'] = dict() for barch in dependencies: self.deps[barch] = dict() + self.recommends[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() + rset = set() for barch in dependencies: if pkgname not in dependencies[barch]: dependencies[barch][pkgname] = [] @@ -559,6 +569,17 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 else: for barch in dependencies: self.deps[barch][pkgname] = dependencies[barch][pkgname] + + for barch in recommends: + if pkgname not in recommends[barch]: + recommends[barch][pkgname] = [] + rset.add(tuple(recommends[barch][pkgname])) + if len(rset) == 1: + self.recommends['global'][pkgname] = rset.pop() + else: + for barch in recommends: + self.recommends[barch][pkgname] = recommends[barch][pkgname] + provided = set() for bprovided in list(provides.values()): provided.update(set(bprovided)) @@ -667,17 +688,24 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ return ['global'] + [a for a in self.arches if a in metadata.groups] - def get_deps(self, metadata, package): + def get_deps(self, metadata, package, recommended=None): """ Get a list of the dependencies of the given package. :param package: The name of the symbol :type package: string :returns: list of strings """ + recs = [] + if ((recommended is None and self.recommended) or + (recommended and recommended.lower() == 'true')): + for arch in self.get_arches(metadata): + if package in self.recommends[arch]: + recs.extend(self.recommends[arch][package]) + for arch in self.get_arches(metadata): if package in self.deps[arch]: - return self.deps[arch][package] - return [] + recs.extend(self.deps[arch][package]) + return recs def get_provides(self, metadata, package): """ Get a list of all symbols provided by the given package. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4608bcca5..52ffa4388 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -820,7 +820,7 @@ class YumCollection(Collection): return new @Bcfg2.Server.Plugin.track_statistics() - def complete(self, packagelist): + def complete(self, packagelist, recommended=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -838,7 +838,7 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist) + return Collection.complete(self, packagelist, recommended) if packagelist: try: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f93bd0932..5f88b7a16 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -331,10 +331,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, initial = set() to_remove = [] groups = [] + recommended = dict() + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) + + if pkg.get("recommended"): + recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -362,7 +367,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) - packages, unknown = collection.complete(base) + packages, unknown = collection.complete(base, recommended) if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) -- cgit v1.2.3-1-g7c22