From c4154ca4d2b7b2a7bcc266d3f473ff1d247cac9f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 19 Oct 2011 10:14:43 -0400 Subject: Various Packages plugin fixes: * Added bcfg2-yum-helper to offload Yum API calls to a short-lived process. As absurd as this is, it appears to be the standard way to get around Yum's atrocious memory handling (cf. yum-updatesd). * Added SourceInitError for non-fatal errors instantiating new sources. * Fixed bug that caused all cached data to be removed on server startup (reported by mikemccllstr on IRC). * Fixed a number of overly aggressive caching issues. * Fixed the way sources are loaded at server startup. * Fixed handling of Yum package groups. --- src/lib/Server/Plugins/Packages/Collection.py | 91 ++-- src/lib/Server/Plugins/Packages/PackagesConfig.py | 7 +- src/lib/Server/Plugins/Packages/PackagesSources.py | 31 +- src/lib/Server/Plugins/Packages/Source.py | 4 + src/lib/Server/Plugins/Packages/Yum.py | 463 +++++++-------------- src/lib/Server/Plugins/Packages/__init__.py | 2 +- 6 files changed, 231 insertions(+), 367 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Packages/Collection.py b/src/lib/Server/Plugins/Packages/Collection.py index cde290d5e..bb72640fd 100644 --- a/src/lib/Server/Plugins/Packages/Collection.py +++ b/src/lib/Server/Plugins/Packages/Collection.py @@ -8,6 +8,16 @@ except ImportError: logger = logging.getLogger("Packages") +# we have to cache Collection objects so that calling Packages.Refresh +# or .Reload can tell the collection objects to clean up their cache, +# but we don't actually use the cache to return a Collection object +# when one is requested, because that prevents new machines from +# working, since a Collection object gets created by +# get_additional_data(), which is called for all clients at server +# startup. (It would also prevent machines that change groups from +# working properly; e.g., if you reinstall a machine with a new OS, +# then returning a cached Collection object would give the wrong +# sources to that client.) collections = dict() class Collection(object): @@ -35,8 +45,8 @@ class Collection(object): return md5(self.get_config()).hexdigest() def get_config(self): - self.logger.error("Packages: Cannot generate config for host with multiple " - "source types (%s)" % self.metadata.hostname) + self.logger.error("Packages: Cannot generate config for host with " + "multiple source types (%s)" % self.metadata.hostname) return "" def get_relevant_groups(self): @@ -285,51 +295,50 @@ def factory(metadata, sources, basepath): if not sources.loaded: # if sources.xml has not received a FAM event yet, defer; - # instantiate a dummy Collection object, but do not cache it - # in collections + # instantiate a dummy Collection object return Collection(metadata, [], basepath) sclasses = set() relevant = list() - + for source in sources: if source.applies(metadata): relevant.append(source) sclasses.update([source.__class__]) - # collections is a cache dict of Collection objects that is keyed - # off of the set of source urls that apply to each Collection - ckeydata = set() - for source in relevant: - ckeydata.update(source.urls) - ckey = tuple(sorted(list(ckeydata))) - if ckey not in collections: - if len(sclasses) > 1: - logger.warning("Packages: Multiple source types found for %s: %s" % - ",".join([s.__name__ for s in sclasses])) - cclass = Collection - elif len(sclasses) == 0: - logger.warning("Packages: No sources found for %s" % metadata.hostname) - cclass = Collection - else: - stype = sclasses.pop().__name__.replace("Source", "") - try: - module = \ - getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cclass = getattr(module, "%sCollection" % stype.title()) - except ImportError: - logger.error("Packages: Unknown source type %s" % stype) - except AttributeError: - logger.warning("Packages: No collection class found for %s sources" % - stype) - - logger.debug("Packages: Using %s for Collection of sources for %s" % - (cclass.__name__, metadata.hostname)) - - collection = cclass(metadata, relevant, basepath) - # reverse so that file order determines precedence - collection.reverse() - collections[ckey] = collection - return collections[ckey] + if len(sclasses) > 1: + logger.warning("Packages: Multiple source types found for %s: %s" % + ",".join([s.__name__ for s in sclasses])) + cclass = Collection + elif len(sclasses) == 0: + # you'd think this should be a warning, but it happens all the + # freaking time if you have a) machines in your clients.xml + # that do not have the proper groups set up yet (e.g., if you + # have multiple Bcfg2 servers and Packages-relevant groups set + # by probes); and b) templates that query all or multiple + # machines (e.g., with metadata.query.all_clients()) + logger.debug("Packages: No sources found for %s" % metadata.hostname) + cclass = Collection + else: + stype = sclasses.pop().__name__.replace("Source", "") + try: + module = \ + getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % + stype.title()).Server.Plugins.Packages, + stype.title()) + cclass = getattr(module, "%sCollection" % stype.title()) + except ImportError: + logger.error("Packages: Unknown source type %s" % stype) + except AttributeError: + logger.warning("Packages: No collection class found for %s sources" + % stype) + + logger.debug("Packages: Using %s for Collection of sources for %s" % + (cclass.__name__, metadata.hostname)) + + collection = cclass(metadata, relevant, basepath) + # reverse so that file order determines precedence + collection.reverse() + collections[metadata.hostname] = collection + return collection + diff --git a/src/lib/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Server/Plugins/Packages/PackagesConfig.py index 1bb250007..d3732bf96 100644 --- a/src/lib/Server/Plugins/Packages/PackagesConfig.py +++ b/src/lib/Server/Plugins/Packages/PackagesConfig.py @@ -25,4 +25,9 @@ class PackagesConfig(Bcfg2.Server.Plugin.FileBacked, for section in self.sections(): self.remove_section(section) self.read(self.name) - self.pkg_obj.Reload() + if self.pkg_obj.sources.loaded: + # only reload Packages plugin if sources have been loaded. + # otherwise, this is getting called on server startup, and + # we have to wait until all sources have been indexed + # before we can call Packages.Reload() + self.pkg_obj.Reload() diff --git a/src/lib/Server/Plugins/Packages/PackagesSources.py b/src/lib/Server/Plugins/Packages/PackagesSources.py index 6d0b1d732..4fbccab30 100644 --- a/src/lib/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Server/Plugins/Packages/PackagesSources.py @@ -3,10 +3,10 @@ import sys import lxml.etree import logging import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Packages.Source import SourceInitError logger = logging.getLogger("Packages") - class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, Bcfg2.Server.Plugin.StructFile): __identifier__ = None @@ -30,8 +30,19 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, # create cache directory if needed os.makedirs(self.cachepath) self.pkg_obj = packages + self.parsed = set() self.loaded = False + def HandleEvent(self, event=None): + Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event) + if event.filename != self.name: + self.parsed.add(os.path.basename(event.filename)) + + if sorted(list(self.parsed)) == sorted(self.extras): + logger.info("Reloading Packages plugin") + self.pkg_obj.Reload() + self.loaded = True + def Index(self): Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) self.entries = [] @@ -40,9 +51,6 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, if source is not None: self.entries.append(source) - self.pkg_obj.Reload() - self.loaded = True - def source_from_xml(self, xsource): """ create a *Source object from its XML representation in sources.xml """ @@ -60,7 +68,20 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, logger.error("Packages: Unknown source type %s" % stype) return None - return cls(self.cachepath, xsource, self.config) + try: + source = cls(self.cachepath, xsource, self.config) + except SourceInitError: + err = sys.exc_info()[1] + logger.error("Packages: %s" % err) + source = None + + return source def __getitem__(self, key): return self.entries[key] + + def __repr__(self): + return "PackagesSources: %s" % repr(self.entries) + + def __str__(self): + return "PackagesSources: %s" % str(self.entries) diff --git a/src/lib/Server/Plugins/Packages/Source.py b/src/lib/Server/Plugins/Packages/Source.py index 00018ef9a..c5ea49453 100644 --- a/src/lib/Server/Plugins/Packages/Source.py +++ b/src/lib/Server/Plugins/Packages/Source.py @@ -28,6 +28,10 @@ def fetch_url(url): return urlopen(url).read() +class SourceInitError(Exception): + pass + + class Source(object): reponame_re = re.compile(r'.*/(?:RPMS\.)?([^/]+)') basegroups = [] diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py index 4ab6aa269..12af46508 100644 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ b/src/lib/Server/Plugins/Packages/Yum.py @@ -9,10 +9,12 @@ import logging import threading import lxml.etree from UserDict import DictMixin +from subprocess import Popen, PIPE, STDOUT import Bcfg2.Server.Plugin from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import Source, fetch_url +from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ + fetch_url logger = logging.getLogger("Packages") @@ -33,6 +35,11 @@ except ImportError: logger.info("Packages: No yum libraries found; forcing use of internal dependency " "resolver") +try: + import json +except ImportError: + import simplejson as json + XP = '{http://linux.duke.edu/metadata/common}' RP = '{http://linux.duke.edu/metadata/rpm}' RPO = '{http://linux.duke.edu/metadata/repo}' @@ -41,6 +48,11 @@ FL = '{http://linux.duke.edu/metadata/filelists}' PULPSERVER = None PULPCONFIG = None +# dict of Cache objects that is keyed off of the set of source urls +# that apply to each Collection +CACHES = dict() + + def _setup_pulp(config): global PULPSERVER, PULPCONFIG if not has_pulp: @@ -85,6 +97,9 @@ class CacheItem(object): else: return False + def __str__(self): + return str(self.value) + class Cache(DictMixin): def __init__(self, expiration=None, tidy=None): @@ -119,7 +134,7 @@ class Cache(DictMixin): del self.cache[key] def __contains__(self, key): - self.expire(key) + self._expire(key) return key in self.cache def keys(self): @@ -162,7 +177,12 @@ class Cache(DictMixin): class YumCollection(Collection): + # options that are included in the [yum] section but that should + # not be included in the temporary yum.conf we write out + option_blacklist = ["use_yum_libraries", "helper"] + def __init__(self, metadata, sources, basepath): + global CACHES Collection.__init__(self, metadata, sources, basepath) self.keypath = os.path.join(self.basepath, "keys") @@ -177,7 +197,6 @@ class YumCollection(Collection): self.use_yum = False if self.use_yum: - self._yb = None self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) if not os.path.exists(self.cachefile): @@ -188,69 +207,52 @@ class YumCollection(Collection): os.mkdir(self.configdir) self.cfgfile = os.path.join(self.configdir, "%s-yum.conf" % self.cachekey) - if self.config.has_option("yum", "metadata_expire"): + self.write_config() + + # in order for use_yum to be true, there must be a [yum] + # section in the config file, so we don't have to check + # for NoSectionError here + try: cache_expire = self.config.getint("yum", "metadata_expire") - else: + except ConfigParser.NoOptionError: cache_expire = 21600 + + try: + self.helper = self.config.get("yum", "helper") + except ConfigParser.NoOptionError: + self.helper = "/usr/sbin/bcfg2-yum-helper" - self.pkgs_cache = Cache(expiration=cache_expire) - self.deps_cache = Cache(expiration=cache_expire) - self.vpkgs_cache = Cache(expiration=cache_expire) - self.group_cache = Cache(expiration=cache_expire) - self.pkgset_cache = Cache(expiration=cache_expire) + ckeydata = set() + for source in sources: + ckeydata.update(source.urls) + cachekey = tuple(sorted(list(ckeydata))) + if cachekey not in CACHES: + CACHES[cachekey] = Cache(expiration=cache_expire) + self.cache = CACHES[cachekey] if has_pulp: _setup_pulp(self.config) - @property - def yumbase(self): - """ if we try to access a Yum SQLitePackageSack object in a - different thread from the one it was created in, we get a - nasty error. but I can't find a way to detect when a new - thread is started (which happens for every new client - connection, I think), so this property creates a new YumBase - object if the old YumBase object was created in a different - thread than the current one. (I definitely don't want to - create a new YumBase object every time it's used, because that - involves writing a temp file, at least for now.) """ - if not self.use_yum: - self._yb = None - self._yb_thread = None - elif (self._yb is None or - self._yb_thread != threading.current_thread().ident): - self._yb = yum.YumBase() - self._yb_thread = threading.current_thread().ident - - if not os.path.exists(self.cfgfile): - # todo: detect yum version. Supposedly very new - # versions of yum have better support for - # reconfiguring on the fly using the RepoStorage API - yumconf = self.get_config(raw=True) - yumconf.add_section("main") + def write_config(self): + if not os.path.exists(self.cfgfile): + yumconf = self.get_config(raw=True) + yumconf.add_section("main") - mainopts = dict(cachedir=self.cachefile, - keepcache="0", - sslverify="0", - reposdir="/dev/null") - try: - for opt in self.config.options("yum"): - if opt != "use_yum_libraries": - mainopts[opt] = self.config.get("yum", opt) - except ConfigParser.NoSectionError: - pass - - for opt, val in list(mainopts.items()): - yumconf.set("main", opt, val) + mainopts = dict(cachedir=self.cachefile, + keepcache="0", + sslverify="0", + reposdir="/dev/null") + try: + for opt in self.config.options("yum"): + if opt not in self.option_blacklist: + mainopts[opt] = self.config.get("yum", opt) + except ConfigParser.NoSectionError: + pass - yumconf.write(open(self.cfgfile, 'w')) + for opt, val in list(mainopts.items()): + yumconf.set("main", opt, val) - # it'd be nice if we could change this to be more verbose - # if -v was given, but Collection objects don't get setup. - # It'd also be nice if we could tell yum to log to syslog, - # but so would a unicorn. - self._yb.preconf.debuglevel = 1 - self._yb.preconf.fn = self.cfgfile - return self._yb + yumconf.write(open(self.cfgfile, 'w')) def get_config(self, raw=False): config = ConfigParser.SafeConfigParser() @@ -393,107 +395,39 @@ class YumCollection(Collection): def is_package(self, package): if not self.use_yum: return Collection.is_package(self, package) - - if isinstance(package, tuple): + elif isinstance(package, tuple): if package[1] is None and package[2] == (None, None, None): package = package[0] else: return None - - try: - return self.pkgs_cache[package] - except KeyError: - pass - - self.pkgs_cache[package] = bool(self.get_package_object(package, - silent=True)) - return self.pkgs_cache[package] + else: + # this should really never get called; it's just provided + # for API completeness + return self.call_helper("is_package", package) def is_virtual_package(self, package): - if self.use_yum: - try: - return bool(self.vpkgs_cache[package]) - except KeyError: - return bool(self.get_provides(package, silent=True)) - else: + if not self.use_yum: return Collection.is_virtual_package(self, package) - - def get_package_object(self, package, silent=False): - """ package objects cannot be cached since they are sqlite - objects, so they can't be reused between threads. """ - try: - matches = self.yumbase.pkgSack.returnNewestByName(name=package) - except yum.Errors.PackageSackError: - if not silent: - self.logger.warning("Packages: Package '%s' not found" % - self.get_package_name(package)) - matches = [] - except yum.Errors.RepoError: - err = sys.exc_info()[1] - self.logger.error("Packages: Temporary failure loading metadata " - "for '%s': %s" % - (self.get_package_name(package), err)) - matches = [] - - pkgs = self._filter_arch(matches) - if pkgs: - return pkgs[0] else: - return None + # this should really never get called; it's just provided + # for API completeness + return self.call_helper("is_virtual_package", package) def get_deps(self, package): if not self.use_yum: return Collection.get_deps(self, package) - - try: - return self.deps_cache[package] - except KeyError: - pass - - pkg = self.get_package_object(package) - deps = [] - if pkg: - deps = set(pkg.requires) - # filter out things the package itself provides - deps.difference_update([dep for dep in deps - if pkg.checkPrco('provides', dep)]) else: - self.logger.error("Packages: No package available: %s" % - self.get_package_name(package)) - self.deps_cache[package] = deps - return self.deps_cache[package] + # this should really never get called; it's just provided + # for API completeness + return self.call_helper("get_deps", package) def get_provides(self, required, all=False, silent=False): if not self.use_yum: return Collection.get_provides(self, package) - - if not isinstance(required, tuple): - required = (required, None, (None, None, None)) - - try: - return self.vpkgs_cache[required] - except KeyError: - pass - - try: - prov = \ - self.yumbase.whatProvides(*required).returnNewestByNameArch() - except yum.Errors.NoMoreMirrorsRepoError: - err = sys.exc_info()[1] - self.logger.error("Packages: Temporary failure loading metadata " - "for '%s': %s" % - (self.get_package_name(required), - err)) - self.vpkgs_cache[required] = None - return [] - - if prov and not all: - prov = self._filter_provides(required, prov) - elif not prov and not silent: - self.logger.error("Packages: No package provides %s" % - self.get_package_name(required)) - self.vpkgs_cache[required] = prov - return self.vpkgs_cache[required] + else: + # this should really never get called; it's just provided + # for API completeness + return self.call_helper("get_provides", package) def get_group(self, group): if not self.use_yum: @@ -504,69 +438,16 @@ class YumCollection(Collection): if group.startswith("@"): group = group[1:] + cachekey = "group:%s" % group try: - return self.group_cache[group] + return self.cache[cachekey] except KeyError: pass - try: - if self.yumbase.comps.has_group(group): - pkgs = self.yumbase.comps.return_group(group).packages - else: - self.logger.warning("Packages: '%s' is not a valid group" % - group) - pkgs = [] - except yum.Errors.GroupsError: - err = sys.exc_info()[1] - self.logger.warning("Packages: %s" % err) - pkgs = [] - - self.group_cache[group] = pkgs - return self.group_cache[group] - - def _filter_provides(self, package, providers): - providers = [pkg for pkg in self._filter_arch(providers)] - if len(providers) > 1: - # go through each provider and make sure it's the newest - # package of its name available. If we have multiple - # providers, avoid installing old packages. - # - # For instance: on Fedora 14, - # perl-Sub-WrapPackages-2.0-2.fc14 erroneously provided - # perl(lib), which should not have been provided; - # perl(lib) is provided by the "perl" package. The bogus - # provide was removed in perl-Sub-WrapPackages-2.0-4.fc14, - # but if we just queried to resolve the "perl(lib)" - # dependency, we'd get both packages. By performing this - # check, we learn that there's a newer - # perl-Sub-WrapPackages available, so it can't be the best - # provider of perl(lib). - rv = [] - for pkg in providers: - if self.get_package_object(pkg.name) == pkg: - rv.append(pkg) - else: - rv = providers - return [p.name for p in rv] - - def _filter_arch(self, packages): - groups = set(list(self.get_relevant_groups()) + ["noarch"]) - matching = [pkg for pkg in packages if pkg.arch in groups] - if matching: - return matching - else: - # no packages match architecture; we'll assume that the - # user knows what s/he is doing and this is a multiarch - # box. - return packages - - def get_package_name(self, package): - """ get the name of a package or virtual package from the - internal representation used by this Collection class """ - if self.use_yum and isinstance(package, tuple): - return yum.misc.prco_tuple_to_string(package) - else: - return str(package) + pkgs = self.call_helper("get_group", group) + if pkgs: + self.cache[cachekey] = pkgs + return pkgs def complete(self, packagelist): if not self.use_yum: @@ -574,130 +455,76 @@ class YumCollection(Collection): cachekey = cPickle.dumps(sorted(packagelist)) try: - packages = self.pkgset_cache[cachekey] + packages, unknown = self.cache[cachekey] except KeyError: packages = set() - - pkgs = set(packagelist).difference(packages) - requires = set() - satisfied = set() - unknown = set() - final_pass = False - - while requires or pkgs: - # infinite loop protection - start_reqs = len(requires) - - while pkgs: - package = pkgs.pop() - if package in packages: - continue - - if not self.is_package(package): - # try this package out as a requirement - requires.add((package, None, (None, None, None))) - continue - - packages.add(package) - reqs = set(self.get_deps(package)).difference(satisfied) - if reqs: - requires.update(reqs) - - reqs_satisfied = set() - for req in requires: - if req in satisfied: - reqs_satisfied.add(req) - continue - - if req[1] is None and self.is_package(req[0]): - if req[0] not in packages: - pkgs.add(req[0]) - reqs_satisfied.add(req) - continue - - self.logger.debug("Packages: Handling requirement '%s'" % - self.get_package_name(req)) - providers = list(set(self.get_provides(req))) - if len(providers) > 1: - # hopefully one of the providing packages is already - # included - best = [p for p in providers if p in packages] - if best: - providers = best - else: - # pick a provider whose name matches the requirement - best = [p for p in providers if p == req[0]] - if len(best) == 1: - providers = best - elif not final_pass: - # found no "best" package, so defer - providers = None - # else: found no "best" package, but it's the - # final pass, so include them all - - if providers: - self.logger.debug("Packages: Requirement '%s' satisfied " - "by %s" % - (self.get_package_name(req), - ",".join([self.get_package_name(p) - for p in providers]))) - newpkgs = set(providers).difference(packages) - if newpkgs: - for package in newpkgs: - if self.is_package(package): - pkgs.add(package) - else: - unknown.add(package) - reqs_satisfied.add(req) - elif providers is not None: - # nothing provided this requirement at all - unknown.add(req) - reqs_satisfied.add(req) - # else, defer - requires.difference_update(reqs_satisfied) - - # infinite loop protection - if len(requires) == start_reqs and len(pkgs) == 0: - final_pass = True - - if final_pass and requires: - unknown.update(requires) - requires = set() - - self.filter_unknown(unknown) - unknown = [self.get_package_name(p) for p in unknown] - - # we do not cache unknown packages, since those are likely to - # be fixed - self.pkgset_cache[cachekey] = packages - + unknown = set(packagelist) + + if unknown: + result = \ + self.call_helper("complete", + dict(packages=list(unknown), + groups=list(self.get_relevant_groups()))) + if result and "packages" in result and "unknown" in result: + # we stringify every package because it gets returned + # in unicode; set.update() doesn't work if some + # elements are unicode and other are strings. (I.e., + # u'foo' and 'foo' get treated as unique elements.) + packages.update([str(p) for p in result['packages']]) + unknown = set([str(p) for p in result['unknown']]) + + self.filter_unknown(unknown) + self.cache[cachekey] = (packages, unknown) + return packages, unknown + def call_helper(self, command, input=None): + """ Make a call to bcfg2-yum-helper. The yum libs have + horrific memory leaks, so apparently the right way to get + around that in long-running processes it to have a short-lived + helper. No, seriously -- check out the yum-updatesd code. + It's pure madness. """ + # it'd be nice if we could change this to be more verbose if + # -v was given to bcfg2-server, but Collection objects don't + # get the 'setup' variable, so we don't know how verbose + # bcfg2-server is. It'd also be nice if we could tell yum to + # log to syslog. So would a unicorn. + cmd = [self.helper, "-c", self.cfgfile, command] + self.logger.debug("Packages: running %s" % " ".join(cmd)) + helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + if input: + idata = json.dumps(input) + (stdout, stderr) = helper.communicate(idata) + else: + (stdout, stderr) = helper.communicate() + rv = helper.wait() + if rv: + self.logger.error("Packages: error running bcfg2-yum-helper " + "(returned %d): %s" % (rv, stderr)) + try: + return json.loads(stdout) + except ValueError: + err = sys.exc_info()[1] + self.logger.error("Packages: error reading bcfg2-yum-helper " + "output: %s" % err) + return None + def setup_data(self, force_update=False): if not self.use_yum: return Collection.setup_data(self, force_update) - for cfile in glob.glob(os.path.join(self.configdir, "*-yum.conf")): - os.unlink(cfile) - self._yb = None - - self.pkgs_cache.clear() - self.deps_cache.clear() - self.vpkgs_cache.clear() - self.group_cache.clear() - self.pkgset_cache.clear() + if force_update: + # we call this twice: one to clean up data from the old + # config, and once to clean up data from the new config + self.call_helper("clean") + + os.unlink(self.cfgfile) + self.write_config() + self.cache.clear() + if force_update: - for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", - "ExpireCache"]: - # for reasons that are entirely obvious, all of the - # yum API clean* methods return a tuple of 0 (zero, - # always zero) and a list containing a single message - # about how many files were deleted. so useful. - # thanks, yum. - self.logger.info("Packages: %s" % - getattr(self.yumbase, - "clean%s" % mdtype)()[1][0]) + self.call_helper("clean") class YumSource(Source): @@ -727,16 +554,14 @@ class YumSource(Source): msg = "Packages: Error %d fetching pulp repo %s: %s" % (err[0], self.pulp_id, err[1]) - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError + raise SourceInitError(msg) except socket.error: err = sys.exc_info()[1] - logger.error("Packages: Could not contact Pulp server: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError + raise SourceInitError("Could not contact Pulp server: %s" % err) except: err = sys.exc_info()[1] - logger.error("Packages: Unknown error querying Pulp server: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError + raise SourceInitError("Unknown error querying Pulp server: %s" % + err) self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'], self.repo['relative_path']) self.arches = [self.repo['arch']] diff --git a/src/lib/Server/Plugins/Packages/__init__.py b/src/lib/Server/Plugins/Packages/__init__.py index 3ee4f21dd..c54f31019 100644 --- a/src/lib/Server/Plugins/Packages/__init__.py +++ b/src/lib/Server/Plugins/Packages/__init__.py @@ -13,7 +13,7 @@ from Bcfg2.Server.Plugins.Packages import Collection from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig -logger = logging.getLogger('Packages') +logger = logging.getLogger('Packages') class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, -- cgit v1.2.3-1-g7c22