diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins/Packages')
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 17 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 75 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Pac.py | 7 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 32 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 45 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 211 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 99 |
7 files changed, 339 insertions, 147 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 49e9d417b..685cd5c1d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -2,13 +2,15 @@ import re import gzip from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source -from Bcfg2.Bcfg2Py3k import cPickle, file +from Bcfg2.Bcfg2Py3k import cPickle class AptCollection(Collection): def get_group(self, group): - self.logger.warning("Packages: Package groups are not supported by APT") + self.logger.warning("Packages: Package groups are not " + "supported by APT") return [] + class AptSource(Source): basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] ptype = 'deb' @@ -22,14 +24,15 @@ class AptSource(Source): 'components': self.components, 'arches': self.arches}] def save_state(self): - cache = file(self.cachefile, 'wb') - cPickle.dump((self.pkgnames, self.deps, self.provides), - cache, 2) + cache = open(self.cachefile, 'wb') + cPickle.dump((self.pkgnames, self.deps, self.provides, + self.essentialpkgs), cache, 2) cache.close() def load_state(self): - data = file(self.cachefile) - self.pkgnames, self.deps, self.provides = cPickle.load(data) + data = open(self.cachefile) + (self.pkgnames, self.deps, self.provides, + self.essentialpkgs) = cPickle.load(data) def filter_unknown(self, unknown): filtered = set([u for u in unknown if u.startswith('choice')]) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 3ea14ce75..b05a69d4a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -1,6 +1,7 @@ import sys import copy import logging +import lxml import Bcfg2.Server.Plugin logger = logging.getLogger(__name__) @@ -52,13 +53,40 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): @property def cachekey(self): - return md5(self.get_config()).hexdigest() + return md5(self.sourcelist()).hexdigest() def get_config(self): - self.logger.error("Packages: Cannot generate config for host with " - "multiple source types (%s)" % self.metadata.hostname) + self.logger.error("Packages: Cannot generate config for host %s with " + "no sources or multiple source types" % + self.metadata.hostname) return "" + def sourcelist(self): + srcs = [] + for source in self.sources: + # get_urls() loads url_map as a side-effect + source.get_urls() + for url_map in source.url_map: + if url_map['arch'] not in metadata.groups: + continue + reponame = source.get_repo_name(url_map) + srcs.append("Name: %s" % reponame) + srcs.append(" Type: %s" % source.ptype) + if url_map['url']: + srcs.append(" URL: %s" % url_map['url']) + elif url_map['rawurl']: + srcs.append(" RAWURL: %s" % url_map['rawurl']) + if source.gpgkeys: + srcs.append(" GPG Key(s): %s" % ", ".join(source.gpgkeys)) + else: + srcs.append(" GPG Key(s): None") + if len(source.blacklist): + srcs.append(" Blacklist: %s" % ", ".join(source.blacklist)) + if len(source.whitelist): + srcs.append(" Whitelist: %s" % ", ".join(source.whitelist)) + srcs.append("") + return "\n".join(srcs) + def get_relevant_groups(self): groups = [] for source in self.sources: @@ -79,6 +107,14 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): cachefiles.add(source.cachefile) return list(cachefiles) + def get_groups(self, grouplist): + """ provided since some backends may be able to query multiple + groups at once faster than serially """ + rv = dict() + for group, ptype in grouplist: + rv[group] = self.get_group(group, ptype) + return rv + def get_group(self, group, ptype=None): for source in self.sources: pkgs = source.get_group(self.metadata, group, ptype=ptype) @@ -152,6 +188,28 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): """ do any collection-level data setup tasks """ pass + def packages_from_entry(self, entry): + """ given a Package or BoundPackage entry, get a list of the + package(s) described by it in a format appropriate for passing + to complete(). by default, that's just the name; only the Yum + backend supports getting versions""" + return [entry.get("name")] + + def packages_to_entry(self, pkglist, entry): + for pkg in pkglist: + lxml.etree.SubElement(entry, 'BoundPackage', name=pkg, + version=self.setup.cfp.get("packages", + "version", + default="auto"), + type=self.ptype, origin='Packages') + + def get_new_packages(self, initial, complete): + """ compute the difference between the complete package list + and the initial package list. this is necessary because the + format may be different between the two lists due to + packages_{to,from}_entry() """ + return list(complete.difference(initial)) + def complete(self, packagelist): '''Build the transitive closure of all package dependencies @@ -350,15 +408,7 @@ def factory(metadata, sources, basepath, debug=False): ",".join([s.__name__ for s in sclasses])) cclass = Collection elif len(sclasses) == 0: - # you'd think this should be a warning, but it happens all the - # freaking time if you have a) machines in your clients.xml - # that do not have the proper groups set up yet (e.g., if you - # have multiple Bcfg2 servers and Packages-relevant groups set - # by probes); and b) templates that query all or multiple - # machines (e.g., with metadata.query.all_clients()) - if debug: - logger.error("Packages: No sources found for %s" % - metadata.hostname) + logger.error("Packages: No sources found for %s" % metadata.hostname) cclass = Collection else: cclass = get_collection_class(sclasses.pop().__name__.replace("Source", @@ -373,4 +423,3 @@ def factory(metadata, sources, basepath, debug=False): clients[metadata.hostname] = ckey collections[ckey] = collection return collection - diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 99a090739..34c7b42c1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -1,6 +1,6 @@ import gzip import tarfile -from Bcfg2.Bcfg2Py3k import cPickle, file +from Bcfg2.Bcfg2Py3k import cPickle from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source @@ -9,6 +9,7 @@ class PacCollection(Collection): self.logger.warning("Packages: Package groups are not supported by Pacman") return [] + class PacSource(Source): basegroups = ['arch', 'parabola'] ptype = 'pacman' @@ -22,13 +23,13 @@ class PacSource(Source): 'components': self.components, 'arches': self.arches}] def save_state(self): - cache = file(self.cachefile, 'wb') + cache = open(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides), cache, 2) cache.close() def load_state(self): - data = file(self.cachefile) + data = open(self.cachefile) self.pkgnames, self.deps, self.provides = cPickle.load(data) def filter_unknown(self, unknown): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 7796b9e34..0d565be31 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -4,17 +4,15 @@ import lxml.etree import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Packages.Source import SourceInitError -class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile, +class PackagesSources(Bcfg2.Server.Plugin.StructFile, Bcfg2.Server.Plugin.Debuggable): __identifier__ = None def __init__(self, filename, cachepath, fam, packages, setup): Bcfg2.Server.Plugin.Debuggable.__init__(self) try: - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, - filename, - fam) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, + should_monitor=True) except OSError: err = sys.exc_info()[1] msg = "Packages: Failed to read configuration file: %s" % err @@ -22,7 +20,6 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, msg += " Have you created it?" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) self.cachepath = cachepath self.setup = setup if not os.path.exists(self.cachepath): @@ -42,18 +39,11 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, source.toggle_debug() def HandleEvent(self, event=None): - Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event) + Bcfg2.Server.Plugin.XMLFileBacked.HandleEvent(self, event=event) if event and event.filename != self.name: - for fname in self.extras: - fpath = None - if fname.startswith("/"): - fpath = os.path.abspath(fname) - else: - fpath = \ - os.path.abspath(os.path.join(os.path.dirname(self.name), - fname)) + for fpath in self.extras: if fpath == os.path.abspath(event.filename): - self.parsed.add(fname) + self.parsed.add(fpath) break if self.loaded: @@ -65,7 +55,7 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, return sorted(list(self.parsed)) == sorted(self.extras) def Index(self): - Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) self.entries = [] for xsource in self.xdata.findall('.//Source'): source = self.source_from_xml(xsource) @@ -87,7 +77,8 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, stype.title()) cls = getattr(module, "%sSource" % stype.title()) except (ImportError, AttributeError): - self.logger.error("Packages: Unknown source type %s" % stype) + ex = sys.exc_info()[1] + self.logger.error("Packages: Unknown source type %s (%s)" % (stype, ex)) return None try: @@ -106,4 +97,7 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, return "PackagesSources: %s" % repr(self.entries) def __str__(self): - return "PackagesSources: %s" % str(self.entries) + return "PackagesSources: %s sources" % len(self.entries) + + def __len__(self): + return len(self.entries) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index edcdcd9f2..df3706fb1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -1,11 +1,10 @@ import os import re import sys -import base64 import Bcfg2.Server.Plugin from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ - urlopen, file, cPickle + urlopen, cPickle try: from hashlib import md5 @@ -51,7 +50,18 @@ class Source(Bcfg2.Server.Plugin.Debuggable): for key, tag in [('components', 'Component'), ('arches', 'Arch'), ('blacklist', 'Blacklist'), ('whitelist', 'Whitelist')]: - self.__dict__[key] = [item.text for item in xsource.findall(tag)] + setattr(self, key, [item.text for item in xsource.findall(tag)]) + self.server_options = dict() + self.client_options = dict() + opts = xsource.findall("Options") + for el in opts: + repoopts = dict([(k, v) + for k, v in el.attrib.items() + if k != "clientonly" and k != "serveronly"]) + if el.get("clientonly", "false").lower() == "false": + self.server_options.update(repoopts) + if el.get("serveronly", "false").lower() == "false": + self.client_options.update(repoopts) self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] @@ -137,9 +147,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): def get_repo_name(self, url_map): # try to find a sensible name for a repo - if 'components' in url_map and url_map['components']: - # use the first component as the name - rname = url_map['components'][0] + if 'component' in url_map and url_map['component']: + rname = url_map['component'] else: name = None for repo_re in (self.mrepo_re, @@ -149,12 +158,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable): if match: name = match.group(1) break - if name is None: - # couldn't figure out the name from the URL or URL map - # (which probably means its a screwy URL), so we just - # generate a random one - name = base64.b64encode(os.urandom(16))[:-2] - rname = "%s-%s" % (self.groups[0], name) + if name and self.groups: + rname = "%s-%s" % (self.groups[0], name) + elif self.groups: + rname = self.groups[0] + else: + # a global source with no reasonable name. just use + # the full url and let the regex below make it even + # uglier. + rname = url_map['url'] # see yum/__init__.py in the yum source, lines 441-449, for # the source of this regex. yum doesn't like anything but # string.ascii_letters, string.digits, and [-_.:]. There @@ -169,6 +181,9 @@ class Source(Bcfg2.Server.Plugin.Debuggable): else: return self.__class__.__name__ + def __repr__(self): + return str(self) + def get_urls(self): return [] urls = property(get_urls) @@ -182,6 +197,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): if a in metadata.groups] vdict = dict() for agrp in agroups: + if agrp not in self.provides: + self.logger.warning("%s provides no packages for %s" % + (self, agrp)) + continue for key, value in list(self.provides[agrp].items()): if key not in vdict: vdict[key] = set(value) @@ -213,7 +232,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): fname = self.escape_url(url) try: data = fetch_url(url) - file(fname, 'w').write(data) + open(fname, 'w').write(data) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 53344e200..cba3373c1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1,17 +1,14 @@ import os +import re import sys -import time import copy -import glob import socket -import random import logging -import threading import lxml.etree -from UserDict import DictMixin -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file +from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, URLError, \ + ConfigParser from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url @@ -96,19 +93,29 @@ class YumCollection(Collection): if not os.path.exists(self.cachefile): os.mkdir(self.cachefile) - self.configdir = os.path.join(self.basepath, "yum") - if not os.path.exists(self.configdir): - os.mkdir(self.configdir) - self.cfgfile = os.path.join(self.configdir, - "%s-yum.conf" % self.cachekey) + self.cfgfile = os.path.join(self.cachefile, "yum.conf") self.write_config() if has_pulp and self.has_pulp_sources: _setup_pulp(self.setup) + self._helper = None + @property def helper(self): - return self.setup.cfp.get("packages:yum", "helper", - default="/usr/sbin/bcfg2-yum-helper") + try: + return self.setup.cfp.get("packages:yum", "helper") + except: + pass + + if not self._helper: + # first see if bcfg2-yum-helper is in PATH + try: + Popen(['bcfg2-yum-helper'], + stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() + self._helper = 'bcfg2-yum-helper' + except OSError: + self._helper = "/usr/sbin/bcfg2-yum-helper" + return self._helper @property def use_yum(self): @@ -129,11 +136,21 @@ class YumCollection(Collection): yumconf = self.get_config(raw=True) yumconf.add_section("main") - mainopts = dict(cachedir=self.cachefile, + # we set installroot to the cache directory so + # bcfg2-yum-helper works with an empty rpmdb. otherwise + # the rpmdb is so hopelessly intertwined with yum that we + # have to totally reinvent the dependency resolver. + mainopts = dict(cachedir='/', + installroot=self.cachefile, keepcache="0", - sslverify="0", debuglevel="0", + sslverify="0", reposdir="/dev/null") + if self.setup['debug']: + mainopts['debuglevel'] = "5" + elif self.setup['verbose']: + mainopts['debuglevel'] = "2" + try: for opt in self.setup.cfp.options("packages:yum"): if opt not in self.option_blacklist: @@ -162,7 +179,7 @@ class YumCollection(Collection): config.add_section(reponame) added = True except ConfigParser.DuplicateSectionError: - match = re.match("-(\d)", reponame) + match = re.search("-(\d+)", reponame) if match: rid = int(match.group(1)) + 1 else: @@ -186,6 +203,13 @@ class YumCollection(Collection): config.set(reponame, "includepkgs", " ".join(source.whitelist)) + if raw: + opts = source.server_options + else: + opts = source.client_options + for opt, val in opts.items(): + config.set(reponame, opt, val) + if raw: return config else: @@ -346,6 +370,25 @@ class YumCollection(Collection): # for API completeness return self.call_helper("get_provides", package) + def get_groups(self, grouplist): + if not self.use_yum: + self.logger.warning("Packages: Package groups are not supported by " + "Bcfg2's internal Yum dependency generator") + return [] + + if not grouplist: + return dict() + + gdicts = [] + for group, ptype in grouplist: + if group.startswith("@"): + group = group[1:] + if not ptype: + ptype = "default" + gdicts.append(dict(group=group, type=ptype)) + + return self.call_helper("get_groups", gdicts) + def get_group(self, group, ptype="default"): if not self.use_yum: self.logger.warning("Packages: Package groups are not supported by " @@ -355,32 +398,106 @@ class YumCollection(Collection): if group.startswith("@"): group = group[1:] - pkgs = self.call_helper("get_group", dict(group=group, type=ptype)) - return pkgs + return self.call_helper("get_group", dict(group=group, type=ptype)) + + def packages_from_entry(self, entry): + rv = set() + name = entry.get("name") + + def _tag_to_pkg(tag): + rv = (name, tag.get("arch"), tag.get("epoch"), + tag.get("version"), tag.get("release")) + # if a package requires no specific version, we just use + # the name, not the tuple. this limits the amount of JSON + # encoding/decoding that has to be done to pass the + # package list to bcfg2-yum-helper. + if rv[1:] == (None, None, None, None): + return name + else: + return rv + + for inst in entry.getchildren(): + if inst.tag != "Instance": + continue + rv.add(_tag_to_pkg(inst)) + if not rv: + rv.add(_tag_to_pkg(entry)) + return list(rv) + + def packages_to_entry(self, pkglist, entry): + def _get_entry_attrs(pkgtup): + attrs = dict(version=self.setup.cfp.get("packages", + "version", + default="auto")) + if attrs['version'] == 'any': + return attrs + + if pkgtup[1]: + attrs['arch'] = pkgtup[1] + if pkgtup[2]: + attrs['epoch'] = pkgtup[2] + if pkgtup[3]: + attrs['version'] = pkgtup[3] + if pkgtup[4]: + attrs['release'] = pkgtup[4] + return attrs + + packages = dict() + for pkg in pkglist: + try: + packages[pkg[0]].append(pkg) + except KeyError: + packages[pkg[0]] = [pkg] + for name, instances in packages.items(): + pkgattrs = dict(type=self.ptype, + origin='Packages', + name=name) + if len(instances) > 1: + pkg_el = lxml.etree.SubElement(entry, 'BoundPackage', + **pkgattrs) + for inst in instances: + lxml.etree.SubElement(pkg_el, "Instance", + _get_entry_attrs(inst)) + else: + attrs = _get_entry_attrs(instances[0]) + attrs.update(pkgattrs) + lxml.etree.SubElement(entry, 'BoundPackage', **attrs) + + def get_new_packages(self, initial, complete): + initial_names = [] + for pkg in initial: + if isinstance(pkg, tuple): + initial_names.append(pkg[0]) + else: + initial_names.append(pkg) + new = [] + for pkg in complete: + if pkg[0] not in initial_names: + new.append(pkg) + return new def complete(self, packagelist): if not self.use_yum: return Collection.complete(self, packagelist) - packages = set() - unknown = set(packagelist) - - if unknown: + if packagelist: result = \ self.call_helper("complete", - dict(packages=list(unknown), + dict(packages=list(packagelist), groups=list(self.get_relevant_groups()))) - if result and "packages" in result and "unknown" in result: - # we stringify every package because it gets returned - # in unicode; set.update() doesn't work if some - # elements are unicode and other are strings. (I.e., - # u'foo' and 'foo' get treated as unique elements.) - packages.update([str(p) for p in result['packages']]) - unknown = set([str(p) for p in result['unknown']]) - + if not result: + # some sort of error, reported by call_helper() + return set(), packagelist + # json doesn't understand sets or tuples, so we get back a + # lists of lists (packages) and a list of unicode strings + # (unknown). turn those into a set of tuples and a set of + # strings, respectively. + unknown = set([str(u) for u in result['unknown']]) + packages = set([tuple(p) for p in result['packages']]) self.filter_unknown(unknown) - - return packages, unknown + return packages, unknown + else: + return set(), set() def call_helper(self, command, input=None): """ Make a call to bcfg2-yum-helper. The yum libs have @@ -388,16 +505,12 @@ class YumCollection(Collection): around that in long-running processes it to have a short-lived helper. No, seriously -- check out the yum-updatesd code. It's pure madness. """ - # it'd be nice if we could change this to be more verbose if - # -v was given to bcfg2-server, but Collection objects don't - # get the 'setup' variable, so we don't know how verbose - # bcfg2-server is. It'd also be nice if we could tell yum to - # log to syslog. So would a unicorn. cmd = [self.helper, "-c", self.cfgfile] - if self.debug_flag: + verbose = self.debug_flag or self.setup['verbose'] + if verbose: cmd.append("-v") cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd)) + self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose) try: helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) except OSError: @@ -415,9 +528,9 @@ class YumCollection(Collection): if rv: self.logger.error("Packages: error running bcfg2-yum-helper " "(returned %d): %s" % (rv, stderr)) - elif self.debug_flag: + else: self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr) + stderr, flag=verbose) try: return json.loads(stdout) except ValueError: @@ -500,15 +613,14 @@ class YumSource(Source): def save_state(self): if not self.use_yum: - cache = file(self.cachefile, 'wb') + cache = open(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, self.filemap, self.url_map), cache, 2) cache.close() - def load_state(self): if not self.use_yum: - data = file(self.cachefile) + data = open(self.cachefile) (self.packages, self.deps, self.provides, self.filemap, self.url_map) = cPickle.load(data) @@ -520,7 +632,7 @@ class YumSource(Source): usettings = [{'version':self.version, 'component':comp, 'arch':arch} for comp in self.components] - else: # rawurl given + else: # rawurl given usettings = [{'version':self.version, 'component':None, 'arch':arch}] @@ -546,6 +658,11 @@ class YumSource(Source): except ValueError: self.logger.error("Packages: Bad url string %s" % rmdurl) return [] + except URLError: + err = sys.exc_info()[1] + self.logger.error("Packages: Failed to fetch url %s. %s" % + (rmdurl, err)) + return [] except HTTPError: err = sys.exc_info()[1] self.logger.error("Packages: Failed to fetch url %s. code=%s" % diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index d789a6d39..d3095300a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -11,14 +11,16 @@ from Bcfg2.Bcfg2Py3k import ConfigParser, urlopen from Bcfg2.Server.Plugins.Packages import Collection from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +yum_config_default = "/etc/yum.repos.d/bcfg2.repo" +apt_config_default = "/etc/apt/sources.d/bcfg2" + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, - Bcfg2.Server.Plugin.GoalValidator): + Bcfg2.Server.Plugin.ClientRunHooks): name = 'Packages' conflicts = ['Pkgmgr'] - experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core, datastore): @@ -26,11 +28,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) self.sentinels = set() - self.cachepath = os.path.join(self.data, 'cache') - self.keypath = os.path.join(self.data, 'keys') + self.cachepath = \ + self.core.setup.cfp.get("packages", "cache", + default=os.path.join(self.data, 'cache')) + self.keypath = \ + self.core.setup.cfp.get("packages", "keycache", + default=os.path.join(self.data, 'keys')) if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) @@ -40,11 +46,16 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.core.setup) def toggle_debug(self): - Bcfg2.Server.Plugin.Plugin.toggle_debug(self) + rv = Bcfg2.Server.Plugin.Plugin.toggle_debug(self) self.sources.toggle_debug() + return rv @property def disableResolver(self): + if self.disableMetaData: + # disabling metadata without disabling the resolver Breaks + # Things + return True try: return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): @@ -87,16 +98,18 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if entry.tag == 'Package': collection = self._get_collection(metadata) entry.set('version', self.core.setup.cfp.get("packages", - "version", - default="auto")) + "version", + default="auto")) entry.set('type', collection.ptype) elif entry.tag == 'Path': - if (entry.get("name") == self.core.setup.cfp.get("packages", - "yum_config", - default="") or - entry.get("name") == self.core.setup.cfp.get("packages", - "apt_config", - default="")): + if (entry.get("name") == \ + self.core.setup.cfp.get("packages", + "yum_config", + default=yum_config_default) or + entry.get("name") == \ + self.core.setup.cfp.get("packages", + "apt_config", + default=apt_config_default)): self.create_config(entry, metadata) def HandlesEntry(self, entry, metadata): @@ -110,12 +123,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == self.core.setup.cfp.get("packages", - "yum_config", - default="") or - entry.get("name") == self.core.setup.cfp.get("packages", - "apt_config", - default="")): + if (entry.get("name") == \ + self.core.setup.cfp.get("packages", + "yum_config", + default=yum_config_default) or + entry.get("name") == \ + self.core.setup.cfp.get("packages", + "apt_config", + default=apt_config_default)): return True return False @@ -151,26 +166,24 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # essential pkgs are those marked as such by the distribution essential = collection.get_essential() to_remove = [] + groups = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): - initial.add(pkg.get("name")) + initial.update(collection.packages_from_entry(pkg)) elif pkg.get("group"): - try: - if pkg.get("type"): - gpkgs = collection.get_group(pkg.get("group"), - ptype=pkg.get("type")) - else: - gpkgs = collection.get_group(pkg.get("group")) - base.update(gpkgs) - except TypeError: - raise - self.logger.error("Could not resolve group %s" % - pkg.get("group")) + groups.append((pkg.get("group"), + pkg.get("type"))) to_remove.append(pkg) else: self.logger.error("Packages: Malformed Package: %s" % - lxml.etree.tostring(pkg)) + lxml.etree.tostring(pkg, + xml_declaration=False).decode('UTF-8')) + + gpkgs = collection.get_groups(groups) + for group, pkgs in gpkgs.items(): + base.update(pkgs) + base.update(initial | essential) for el in to_remove: el.getparent().remove(el) @@ -179,16 +192,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) - newpkgs = list(packages.difference(initial)) + newpkgs = collection.get_new_packages(initial, packages) self.debug_log("Packages: %d initial, %d complete, %d new" % (len(initial), len(packages), len(newpkgs))) newpkgs.sort() - for pkg in newpkgs: - lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, - version=self.core.setup.cfp.get("packages", - "version", - default="auto"), - type=collection.ptype, origin='Packages') + collection.packages_to_entry(newpkgs, independent) def Refresh(self): '''Packages.Refresh() => True|False\nReload configuration @@ -271,10 +279,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection = self._get_collection(metadata) return dict(sources=collection.get_additional_data()) - def validate_goals(self, metadata, _): - """ we abuse the GoalValidator plugin since validate_goals() - is the very last thing called during a client config run. so - we use this to clear the collection cache for this client, - which must persist only the duration of a client run """ + def end_client_run(self, metadata): + """ clear the collection cache for this client, which must + persist only the duration of a client run""" if metadata.hostname in Collection.clients: del Collection.clients[metadata.hostname] + + def end_statistics(self, metadata): + self.end_client_run(metadata) |