diff options
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 15 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 106 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 9 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-yum-helper | 188 |
4 files changed, 171 insertions, 147 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 1e269167b..d38a6e714 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -185,6 +185,21 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): """ do any collection-level data setup tasks """ pass + def packages_from_entry(self, entry): + """ given a Package or BoundPackage entry, get a list of the + package(s) described by it in a format appropriate for passing + to complete(). by default, that's just the name; only the Yum + backend supports getting versions""" + return [entry.get("name")] + + def packages_to_entry(self, pkglist, entry): + for pkg in pkglist: + lxml.etree.SubElement(entry, 'BoundPackage', name=pkg, + version=self.setup.cfp.get("packages", + "version", + default="auto"), + type=self.ptype, origin='Packages') + def complete(self, packagelist): '''Build the transitive closure of all package dependencies diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index eeff0c6eb..2f197443c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -139,11 +139,21 @@ class YumCollection(Collection): yumconf = self.get_config(raw=True) yumconf.add_section("main") - mainopts = dict(cachedir=self.cachefile, + # we set installroot to the cache directory so + # bcfg2-yum-helper works with an empty rpmdb. otherwise + # the rpmdb is so hopelessly intertwined with yum that we + # have to totally reinvent the dependency resolver. + mainopts = dict(cachedir='/', + installroot=self.cachefile, keepcache="0", - sslverify="0", debuglevel="0", + sslverify="0", reposdir="/dev/null") + if self.setup['debug']: + mainopts['debuglevel'] = "5" + elif self.setup['verbose']: + mainopts['debuglevel'] = "2" + try: for opt in self.setup.cfp.options("packages:yum"): if opt not in self.option_blacklist: @@ -369,6 +379,9 @@ class YumCollection(Collection): "Bcfg2's internal Yum dependency generator") return [] + if not grouplist: + return dict() + gdicts = [] for group, ptype in grouplist: if group.startswith("@"): @@ -390,29 +403,88 @@ class YumCollection(Collection): return self.call_helper("get_group", dict(group=group, type=ptype)) + def packages_from_entry(self, entry): + rv = set() + name = entry.get("name") + + def _tag_to_pkg(tag): + rv = (name, tag.get("arch"), tag.get("epoch"), + tag.get("version"), tag.get("release")) + # if a package requires no specific version, we just use + # the name, not the tuple. this limits the amount of JSON + # encoding/decoding that has to be done to pass the + # package list to bcfg2-yum-helper. + if rv[1:] == (None, None, None, None): + return name + else: + return rv + + for inst in entry.getchildren(): + if inst.tag != "Instance": + continue + rv.add(_tag_to_pkg(inst)) + if not rv: + rv.add(_tag_to_pkg(entry)) + return list(rv) + + def packages_to_entry(self, pkglist, entry): + def _get_entry_attrs(pkgtup): + attrs = dict(arch=pkgtup[1], + epoch=pkgtup[2], + version=pkgtup[3], + release=pkgtup[4]) + if attrs['version'] is None: + attrs['version'] = self.setup.cfp.get("packages", + "version", + default="auto"), + for k in attrs.keys()[:]: + if attrs[k] is None: + del attrs[k] + return attrs + + packages = dict() + for pkg in pkglist: + try: + packages[pkg[0]].append(pkg) + except KeyError: + packages[pkg[0]] = [pkg] + for name, instances in packages.items(): + pkgattrs = dict(type=self.ptype, + origin='Packages', + name=name) + if len(instances) > 1: + pkg_el = lxml.etree.SubElement(entry, 'BoundPackage', + **pkgattrs) + for inst in instances: + lxml.etree.SubElement(pkg_el, "Instance", + _get_entry_attrs(inst)) + else: + attrs = _get_entry_attrs(instances[0]) + attrs.update(pkgattrs) + lxml.etree.SubElement(entry, 'BoundPackage', **attrs) + def complete(self, packagelist): if not self.use_yum: return Collection.complete(self, packagelist) - packages = set() - unknown = set(packagelist) - - if unknown: + if packagelist: result = \ self.call_helper("complete", - dict(packages=list(unknown), + dict(packages=list(packagelist), groups=list(self.get_relevant_groups()))) - if result and "packages" in result and "unknown" in result: - # we stringify every package because it gets returned - # in unicode; set.update() doesn't work if some - # elements are unicode and other are strings. (I.e., - # u'foo' and 'foo' get treated as unique elements.) - packages.update([str(p) for p in result['packages']]) - unknown = set([str(p) for p in result['unknown']]) - + if not result: + # some sort of error, reported by call_helper() + return set(), packagelist + # json doesn't understand sets or tuples, so we get back a + # lists of lists (packages) and a list of unicode strings + # (unknown). turn those into a set of tuples and a set of + # strings, respectively. + unknown = set([str(u) for u in result['unknown']]) + packages = set([tuple(p) for p in result['packages']]) self.filter_unknown(unknown) - - return packages, unknown + return packages, unknown + else: + return set(), set() def call_helper(self, command, input=None): """ Make a call to bcfg2-yum-helper. The yum libs have diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 31670d3a3..3a7ec2920 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -171,7 +171,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): - initial.add(pkg.get("name")) + initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -196,12 +196,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.debug_log("Packages: %d initial, %d complete, %d new" % (len(initial), len(packages), len(newpkgs))) newpkgs.sort() - for pkg in newpkgs: - lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, - version=self.core.setup.cfp.get("packages", - "version", - default="auto"), - type=collection.ptype, origin='Packages') + collection.packages_to_entry(newpkgs, independent) def Refresh(self): '''Packages.Refresh() => True|False\nReload configuration diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 186eefe7a..53784518b 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -9,8 +9,7 @@ import os import sys import yum import logging -import Bcfg2.Logger -from optparse import OptionParser, OptionError +from optparse import OptionParser try: import json @@ -37,6 +36,24 @@ def get_logger(verbose=0): LOGGER.addHandler(syslog) return LOGGER +def pkg_to_tuple(package): + """ json doesn't distinguish between tuples and lists, but yum + does, so we convert a package in list format to one in tuple + format """ + if isinstance(package, list): + return tuple(package) + else: + return package + +def pkgtup_to_string(package): + rv = [package[0], "-"] + if package[2]: + rv.extend([package[2], ':']) + rv.extend([package[3], '-', package[4]]) + if package[1]: + rv.extend(['.', package[1]]) + return ''.join(str(e) for e in rv) + class DepSolver(object): def __init__(self, cfgfile, verbose=1): @@ -64,27 +81,28 @@ class DepSolver(object): def is_package(self, package): if isinstance(package, tuple): if package[1] is None and package[2] == (None, None, None): - package = package[0] - else: - return None - - return bool(self.get_package_object(package, silent=True)) + pkgtup = (package[0], None, None, None, None) + elif len(package) == 5: + pkgtup = package + else: + pkgtup = (package, None, None, None, None) + return bool(self.get_package_object(pkgtup, silent=True)) def is_virtual_package(self, package): return bool(self.get_provides(package, silent=True)) - def get_package_object(self, package, silent=False): + def get_package_object(self, pkgtup, silent=False): try: - matches = self.yumbase.pkgSack.returnNewestByName(name=package) + matches = yum.packageSack.packagesNewestByName(self.yumbase.pkgSack.searchPkgTuple(pkgtup)) except yum.Errors.PackageSackError: if not silent: self.logger.warning("Package '%s' not found" % - self.get_package_name(package)) + self.get_package_name(pkgtup)) matches = [] except yum.Errors.RepoError: err = sys.exc_info()[1] self.logger.error("Temporary failure loading metadata for %s: %s" % - (self.get_package_name(package), err)) + (self.get_package_name(pkgtup), err)) matches = [] pkgs = self._filter_arch(matches) @@ -100,7 +118,7 @@ class DepSolver(object): deps = set(pkg.requires) # filter out things the package itself provides deps.difference_update([dep for dep in deps - if pkg.checkPrco('provides', dep)]) + if pkg.checkPrco('provides', dep)]) else: self.logger.error("No package available: %s" % self.get_package_name(package)) @@ -120,7 +138,7 @@ class DepSolver(object): return [] if prov and not all: - prov = self._filter_provides(required, prov) + prov = self._filter_provides(prov) elif not prov and not silent: self.logger.error("No package provides %s" % self.get_package_name(required)) @@ -155,7 +173,7 @@ class DepSolver(object): self.logger.warning("Unknown group package type '%s'" % ptype) return [] - def _filter_provides(self, package, providers): + def _filter_provides(self, providers): providers = [pkg for pkg in self._filter_arch(providers)] if len(providers) > 1: # go through each provider and make sure it's the newest @@ -174,7 +192,7 @@ class DepSolver(object): # provider of perl(lib). rv = [] for pkg in providers: - found = self.get_package_object(pkg.name) + found = self.get_package_object(pkg.pkgtup) if found == pkg or found.pkgtup == pkg.pkgtup: rv.append(pkg) else: @@ -182,7 +200,7 @@ class DepSolver(object): (pkg, found)) else: rv = providers - return [p.name for p in rv] + return rv def _filter_arch(self, packages): matching = [] @@ -204,115 +222,38 @@ class DepSolver(object): """ get the name of a package or virtual package from the internal representation used by this Collection class """ if isinstance(package, tuple): - return yum.misc.prco_tuple_to_string(package) + if len(package) == 3: + return yum.misc.prco_tuple_to_string(package) + else: + return pkgtup_to_string(package) else: return str(package) def complete(self, packagelist): packages = set() - pkgs = set(packagelist) - requires = set() - satisfied = set() unknown = set() - final_pass = False - - while requires or pkgs: - # infinite loop protection - start_reqs = len(requires) - - while pkgs: - package = pkgs.pop() - if package in packages: - continue - - if not self.is_package(package): - # try this package out as a requirement - self.logger.debug("Adding requirement %s" % package) - requires.add((package, None, (None, None, None))) - continue - - packages.add(package) - reqs = set(self.get_deps(package)).difference(satisfied) - if reqs: - self.logger.debug("Adding requirements for %s: %s" % - (package, - ",".join([self.get_package_name(r) - for r in reqs]))) - requires.update(reqs) - - reqs_satisfied = set() - for req in requires: - if req in satisfied: - reqs_satisfied.add(req) - continue - - if req[1] is None and self.is_package(req[0]): - if req[0] not in packages: - pkgs.add(req[0]) - reqs_satisfied.add(req) - continue - - self.logger.debug("Handling requirement '%s'" % - self.get_package_name(req)) - providers = list(set(self.get_provides(req))) - if len(providers) > 1: - # hopefully one of the providing packages is already - # included - best = [p for p in providers if p in packages] - if best: - providers = best - else: - # pick a provider whose name matches the requirement - best = [p for p in providers if p == req[0]] - if len(best) == 1: - providers = best - elif not final_pass: - self.logger.debug("%s has multiple providers: %s" % - (self.get_package_name(req), - providers)) - self.logger.debug("No provider is obviously the " - "best; deferring") - providers = None - else: - # found no "best" package, but it's the - # final pass, so include them all - self.logger.debug("Found multiple providers for %s," - "including all" % - self.get_package_name(req)) - - if providers: - self.logger.debug("Requirement '%s' satisfied by %s" % - (self.get_package_name(req), - ",".join([self.get_package_name(p) - for p in providers]))) - newpkgs = set(providers).difference(packages) - if newpkgs: - for package in newpkgs: - if self.is_package(package): - pkgs.add(package) - else: - unknown.add(package) - reqs_satisfied.add(req) - elif providers is not None: - # nothing provided this requirement at all - self.logger.debug("Nothing provides %s" % - self.get_package_name(req)) - unknown.add(req) - reqs_satisfied.add(req) - # else, defer - requires.difference_update(reqs_satisfied) - - # infinite loop protection - if len(requires) == start_reqs and len(pkgs) == 0: - final_pass = True - - if final_pass and requires: - unknown.update(requires) - requires = set() - - unknown = [self.get_package_name(p) for p in unknown] + for pkg in packagelist: + if isinstance(pkg, tuple): + pkgtup = pkg + else: + pkgtup = (pkg, None, None, None, None) + po = self.get_package_object(pkgtup) + if not po: + self.logger.debug("Unknown package %s" % + self.get_package_name(pkg)) + unknown.add(pkg) + else: + if self.yumbase.tsInfo.exists(pkgtup=po.pkgtup): + self.logger.debug("%s added to transaction multiple times" % + po) + else: + self.logger.debug("Adding %s to transaction" % po) + self.yumbase.tsInfo.addInstall(po) + self.yumbase.resolveDeps() - return packages, unknown + for txmbr in self.yumbase.tsInfo: + packages.add(txmbr.pkgtup) + return list(packages), list(unknown) def clean_cache(self): for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", @@ -349,15 +290,16 @@ def main(): elif cmd == "complete": data = json.loads(sys.stdin.read()) depsolver.groups = data['groups'] - (packages, unknown) = depsolver.complete(data['packages']) + (packages, unknown) = depsolver.complete([pkg_to_tuple(p) + for p in data['packages']]) print json.dumps(dict(packages=list(packages), unknown=list(unknown))) elif cmd == "is_virtual_package": - package = json.loads(sys.stdin.read()) + package = pkg_to_tuple(json.loads(sys.stdin.read())) print json.dumps(bool(depsolver.get_provides(package, silent=True))) elif cmd == "get_deps" or cmd == "get_provides": - package = json.loads(sys.stdin.read()) - print json.dumps(list(getattr(depsolver, cmd)(package))) + package = pkg_to_tuple(json.loads(sys.stdin.read())) + print json.dumps([p.name for p in getattr(depsolver, cmd)(package)]) elif cmd == "get_group": data = json.loads(sys.stdin.read()) if "type" in data: @@ -377,7 +319,7 @@ def main(): rv[gdata['group']] = list(packages) print json.dumps(rv) elif cmd == "is_package": - package = json.loads(sys.stdin.read()) + package = pkg_to_tuple(json.loads(sys.stdin.read())) print json.dumps(getattr(depsolver, cmd)(package)) |