diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Yum.py')
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 211 |
1 files changed, 164 insertions, 47 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 53344e200..cba3373c1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1,17 +1,14 @@ import os +import re import sys -import time import copy -import glob import socket -import random import logging -import threading import lxml.etree -from UserDict import DictMixin -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file +from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, URLError, \ + ConfigParser from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url @@ -96,19 +93,29 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): os.mkdir(self.cachefile) - self.configdir = os.path.join(self.basepath, "yum") - if not os.path.exists(self.configdir): - os.mkdir(self.configdir) - self.cfgfile = os.path.join(self.configdir, - "%s-yum.conf" % self.cachekey) + self.cfgfile = os.path.join(self.cachefile, "yum.conf") self.write_config() if has_pulp and self.has_pulp_sources: _setup_pulp(self.setup) + self._helper = None + @property def helper(self): - return self.setup.cfp.get("packages:yum", "helper", - default="/usr/sbin/bcfg2-yum-helper") + try: + return self.setup.cfp.get("packages:yum", "helper") + except: + pass + + if not self._helper: + # first see if bcfg2-yum-helper is in PATH + try: + Popen(['bcfg2-yum-helper'], + stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() + self._helper = 'bcfg2-yum-helper' + except OSError: + self._helper = "/usr/sbin/bcfg2-yum-helper" + return self._helper @property def use_yum(self): @@ -129,11 +136,21 @@ class YumCollection(Collection): yumconf = self.get_config(raw=True) yumconf.add_section("main") - mainopts = dict(cachedir=self.cachefile, + # we set installroot to the cache directory so + # bcfg2-yum-helper works with an empty rpmdb. otherwise + # the rpmdb is so hopelessly intertwined with yum that we + # have to totally reinvent the dependency resolver. + mainopts = dict(cachedir='/', + installroot=self.cachefile, keepcache="0", - sslverify="0", debuglevel="0", + sslverify="0", reposdir="/dev/null") + if self.setup['debug']: + mainopts['debuglevel'] = "5" + elif self.setup['verbose']: + mainopts['debuglevel'] = "2" + try: for opt in self.setup.cfp.options("packages:yum"): if opt not in self.option_blacklist: @@ -162,7 +179,7 @@ class YumCollection(Collection): config.add_section(reponame) added = True except ConfigParser.DuplicateSectionError: - match = re.match("-(\d)", reponame) + match = re.search("-(\d+)", reponame) if match: rid = int(match.group(1)) + 1 else: @@ -186,6 +203,13 @@ class YumCollection(Collection): config.set(reponame, "includepkgs", " ".join(source.whitelist)) + if raw: + opts = source.server_options + else: + opts = source.client_options + for opt, val in opts.items(): + config.set(reponame, opt, val) + if raw: return config else: @@ -346,6 +370,25 @@ class YumCollection(Collection): # for API completeness return self.call_helper("get_provides", package) + def get_groups(self, grouplist): + if not self.use_yum: + self.logger.warning("Packages: Package groups are not supported by " + "Bcfg2's internal Yum dependency generator") + return [] + + if not grouplist: + return dict() + + gdicts = [] + for group, ptype in grouplist: + if group.startswith("@"): + group = group[1:] + if not ptype: + ptype = "default" + gdicts.append(dict(group=group, type=ptype)) + + return self.call_helper("get_groups", gdicts) + def get_group(self, group, ptype="default"): if not self.use_yum: self.logger.warning("Packages: Package groups are not supported by " @@ -355,32 +398,106 @@ class YumCollection(Collection): if group.startswith("@"): group = group[1:] - pkgs = self.call_helper("get_group", dict(group=group, type=ptype)) - return pkgs + return self.call_helper("get_group", dict(group=group, type=ptype)) + + def packages_from_entry(self, entry): + rv = set() + name = entry.get("name") + + def _tag_to_pkg(tag): + rv = (name, tag.get("arch"), tag.get("epoch"), + tag.get("version"), tag.get("release")) + # if a package requires no specific version, we just use + # the name, not the tuple. this limits the amount of JSON + # encoding/decoding that has to be done to pass the + # package list to bcfg2-yum-helper. + if rv[1:] == (None, None, None, None): + return name + else: + return rv + + for inst in entry.getchildren(): + if inst.tag != "Instance": + continue + rv.add(_tag_to_pkg(inst)) + if not rv: + rv.add(_tag_to_pkg(entry)) + return list(rv) + + def packages_to_entry(self, pkglist, entry): + def _get_entry_attrs(pkgtup): + attrs = dict(version=self.setup.cfp.get("packages", + "version", + default="auto")) + if attrs['version'] == 'any': + return attrs + + if pkgtup[1]: + attrs['arch'] = pkgtup[1] + if pkgtup[2]: + attrs['epoch'] = pkgtup[2] + if pkgtup[3]: + attrs['version'] = pkgtup[3] + if pkgtup[4]: + attrs['release'] = pkgtup[4] + return attrs + + packages = dict() + for pkg in pkglist: + try: + packages[pkg[0]].append(pkg) + except KeyError: + packages[pkg[0]] = [pkg] + for name, instances in packages.items(): + pkgattrs = dict(type=self.ptype, + origin='Packages', + name=name) + if len(instances) > 1: + pkg_el = lxml.etree.SubElement(entry, 'BoundPackage', + **pkgattrs) + for inst in instances: + lxml.etree.SubElement(pkg_el, "Instance", + _get_entry_attrs(inst)) + else: + attrs = _get_entry_attrs(instances[0]) + attrs.update(pkgattrs) + lxml.etree.SubElement(entry, 'BoundPackage', **attrs) + + def get_new_packages(self, initial, complete): + initial_names = [] + for pkg in initial: + if isinstance(pkg, tuple): + initial_names.append(pkg[0]) + else: + initial_names.append(pkg) + new = [] + for pkg in complete: + if pkg[0] not in initial_names: + new.append(pkg) + return new def complete(self, packagelist): if not self.use_yum: return Collection.complete(self, packagelist) - packages = set() - unknown = set(packagelist) - - if unknown: + if packagelist: result = \ self.call_helper("complete", - dict(packages=list(unknown), + dict(packages=list(packagelist), groups=list(self.get_relevant_groups()))) - if result and "packages" in result and "unknown" in result: - # we stringify every package because it gets returned - # in unicode; set.update() doesn't work if some - # elements are unicode and other are strings. (I.e., - # u'foo' and 'foo' get treated as unique elements.) - packages.update([str(p) for p in result['packages']]) - unknown = set([str(p) for p in result['unknown']]) - + if not result: + # some sort of error, reported by call_helper() + return set(), packagelist + # json doesn't understand sets or tuples, so we get back a + # lists of lists (packages) and a list of unicode strings + # (unknown). turn those into a set of tuples and a set of + # strings, respectively. + unknown = set([str(u) for u in result['unknown']]) + packages = set([tuple(p) for p in result['packages']]) self.filter_unknown(unknown) - - return packages, unknown + return packages, unknown + else: + return set(), set() def call_helper(self, command, input=None): """ Make a call to bcfg2-yum-helper. The yum libs have @@ -388,16 +505,12 @@ class YumCollection(Collection): around that in long-running processes it to have a short-lived helper. No, seriously -- check out the yum-updatesd code. It's pure madness. """ - # it'd be nice if we could change this to be more verbose if - # -v was given to bcfg2-server, but Collection objects don't - # get the 'setup' variable, so we don't know how verbose - # bcfg2-server is. It'd also be nice if we could tell yum to - # log to syslog. So would a unicorn. cmd = [self.helper, "-c", self.cfgfile] - if self.debug_flag: + verbose = self.debug_flag or self.setup['verbose'] + if verbose: cmd.append("-v") cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd)) + self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose) try: helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) except OSError: @@ -415,9 +528,9 @@ class YumCollection(Collection): if rv: self.logger.error("Packages: error running bcfg2-yum-helper " "(returned %d): %s" % (rv, stderr)) - elif self.debug_flag: + else: self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr) + stderr, flag=verbose) try: return json.loads(stdout) except ValueError: @@ -500,15 +613,14 @@ class YumSource(Source): def save_state(self): if not self.use_yum: - cache = file(self.cachefile, 'wb') + cache = open(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, self.filemap, self.url_map), cache, 2) cache.close() - def load_state(self): if not self.use_yum: - data = file(self.cachefile) + data = open(self.cachefile) (self.packages, self.deps, self.provides, self.filemap, self.url_map) = cPickle.load(data) @@ -520,7 +632,7 @@ class YumSource(Source): usettings = [{'version':self.version, 'component':comp, 'arch':arch} for comp in self.components] - else: # rawurl given + else: # rawurl given usettings = [{'version':self.version, 'component':None, 'arch':arch}] @@ -546,6 +658,11 @@ class YumSource(Source): except ValueError: self.logger.error("Packages: Bad url string %s" % rmdurl) return [] + except URLError: + err = sys.exc_info()[1] + self.logger.error("Packages: Failed to fetch url %s. %s" % + (rmdurl, err)) + return [] except HTTPError: err = sys.exc_info()[1] self.logger.error("Packages: Failed to fetch url %s. code=%s" % |