path: root/src/sbin/bcfg2-yum-helper

#!/usr/bin/env python
""" Helper script for the Packages plugin, used if yum library support
is enabled.  The yum libs have horrific memory leaks, so apparently
the right way to get around that in long-running processes it to have
a short-lived helper.  No, seriously -- check out the yum-updatesd
code.  It's pure madness. """

import os
import sys
import yum
import logging
import Bcfg2.Logger
from Bcfg2.Compat import wraps
from lockfile import FileLock, LockTimeout
from optparse import OptionParser
try:
    import json
    # py2.4 json library is structured differently
    json.loads  # pylint: disable=W0104
except (ImportError, AttributeError):
    import simplejson as json


def pkg_to_tuple(package):
    """ json doesn't distinguish between tuples and lists, but yum
    does, so we convert a package in list format to one in tuple
    format """
    if isinstance(package, list):
        return tuple(package)
    else:
        return package


def pkgtup_to_string(package):
    """ given a package tuple, return a human-readable string
    describing the package """
    if package[3] in ['auto', 'any']:
        return package[0]

    rv = [package[0], "-"]
    if package[2]:
        rv.extend([package[2], ':'])
    rv.extend([package[3], '-', package[4]])
    if package[1]:
        rv.extend(['.', package[1]])
    return ''.join(str(e) for e in rv)


class YumHelper(object):
    """ Yum helper base object """

    def __init__(self, cfgfile, verbose=1):
        self.cfgfile = cfgfile
        self.yumbase = yum.YumBase()
        # pylint: disable=E1121,W0212
        try:
            self.yumbase.preconf.debuglevel = verbose
            self.yumbase.preconf.fn = cfgfile
            self.yumbase._getConfig()
        except AttributeError:
            self.yumbase._getConfig(cfgfile, debuglevel=verbose)
        # pylint: enable=E1121,W0212
        self.logger = logging.getLogger(self.__class__.__name__)


class DepSolver(YumHelper):
    """ Yum dependency solver.  This is used for operations that only
    read from the yum cache, and thus operates in cacheonly mode. """

    def __init__(self, cfgfile, verbose=1):
        YumHelper.__init__(self, cfgfile, verbose=verbose)
        # internally, yum uses an integer, not a boolean, for conf.cache
        self.yumbase.conf.cache = 1
        self._groups = None

    def get_groups(self):
        """ getter for the groups property """
        if self._groups is not None:
            return self._groups
        else:
            return ["noarch"]

    def set_groups(self, groups):
        """ setter for the groups property """
        self._groups = set(groups).union(["noarch"])

    groups = property(get_groups, set_groups)

    def get_package_object(self, pkgtup, silent=False):
        """ given a package tuple, get a yum package object """
        try:
            matches = yum.packageSack.packagesNewestByName(
                self.yumbase.pkgSack.searchPkgTuple(pkgtup))
        except yum.Errors.PackageSackError:
            if not silent:
                self.logger.warning("Package '%s' not found" %
                                    self.get_package_name(pkgtup))
            matches = []
        except yum.Errors.RepoError:
            err = sys.exc_info()[1]
            self.logger.error("Temporary failure loading metadata for %s: %s" %
                              (self.get_package_name(pkgtup), err))
            matches = []

        pkgs = self._filter_arch(matches)
        if pkgs:
            return pkgs[0]
        else:
            return None

    def get_group(self, group, ptype="default"):
        """ Resolve a package group name into a list of packages """
        if group.startswith("@"):
            group = group[1:]

        try:
            if self.yumbase.comps.has_group(group):
                group = self.yumbase.comps.return_group(group)
            else:
                self.logger.error("%s is not a valid group" % group)
                return []
        except yum.Errors.GroupsError:
            err = sys.exc_info()[1]
            self.logger.warning(err)
            return []

        if ptype == "default":
            return [p
                    for p, d in list(group.default_packages.items())
                    if d]
        elif ptype == "mandatory":
            return [p
                    for p, m in list(group.mandatory_packages.items())
                    if m]
        elif ptype == "optional" or ptype == "all":
            return group.packages
        else:
            self.logger.warning("Unknown group package type '%s'" % ptype)
            return []

    def _filter_arch(self, packages):
        """ filter packages in the given list that do not have an
        architecture in the list of groups for this client """
        matching = []
        for pkg in packages:
            if pkg.arch in self.groups:
                matching.append(pkg)
            else:
                self.logger.debug("%s has non-matching architecture (%s)" %
                                  (pkg, pkg.arch))
        if matching:
            return matching
        else:
            # no packages match architecture; we'll assume that the
            # user knows what s/he is doing and this is a multiarch
            # box.
            return packages

    def get_package_name(self, package):
        """ get the name of a package or virtual package from the
        internal representation used by this Collection class """
        if isinstance(package, tuple):
            if len(package) == 3:
                return yum.misc.prco_tuple_to_string(package)
            else:
                return pkgtup_to_string(package)
        else:
            return str(package)

    def complete(self, packagelist):
        """ resolve dependencies and generate a complete package list
        from the given list of initial packages """
        packages = set()
        unknown = set()
        for pkg in packagelist:
            if isinstance(pkg, tuple):
                pkgtup = pkg
            else:
                pkgtup = (pkg, None, None, None, None)
            pkgobj = self.get_package_object(pkgtup)
            if not pkgobj:
                self.logger.debug("Unknown package %s" %
                                  self.get_package_name(pkg))
                unknown.add(pkg)
            else:
                if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup):
                    self.logger.debug("%s added to transaction multiple times"
                                      % pkgobj)
                else:
                    self.logger.debug("Adding %s to transaction" % pkgobj)
                    self.yumbase.tsInfo.addInstall(pkgobj)
        self.yumbase.resolveDeps()

        for txmbr in self.yumbase.tsInfo:
            packages.add(txmbr.pkgtup)
        return list(packages), list(unknown)


def acquire_lock(func):
    """ decorator for CacheManager methods that gets and release a
    lock while the method runs """
    @wraps(func)
    def inner(self, *args, **kwargs):
        """ Get and release a lock while running the function this
        wraps. """
        self.logger.debug("Acquiring lock at %s" % self.lockfile)
        while not self.lock.i_am_locking():
            try:
                self.lock.acquire(timeout=60)  # wait up to 60 seconds
            except LockTimeout:
                self.lock.break_lock()
                self.lock.acquire()
        try:
            func(self, *args, **kwargs)
        finally:
            self.lock.release()
            self.logger.debug("Released lock at %s" % self.lockfile)

    return inner


class CacheManager(YumHelper):
    """ Yum cache manager.  Unlike :class:`DepSolver`, this can write
    to the yum cache, and so is used for operations that muck with the
    cache.  (Technically, :func:`CacheManager.clean_cache` could be in
    either DepSolver or CacheManager, but for consistency I've put it
    here.) """

    def __init__(self, cfgfile, verbose=1):
        YumHelper.__init__(self, cfgfile, verbose=verbose)
        self.lockfile = \
            os.path.join(os.path.dirname(self.yumbase.conf.config_file_path),
                         "lock")
        self.lock = FileLock(self.lockfile)

    @acquire_lock
    def clean_cache(self):
        """ clean the yum cache """
        for mdtype in ["Headers", "Packages", "Sqlite", "Metadata",
                       "ExpireCache"]:
            # for reasons that are entirely obvious, all of the yum
            # API clean* methods return a tuple of 0 (zero, always
            # zero) and a list containing a single message about how
            # many files were deleted.  so useful.  thanks, yum.
            msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0]
            if not msg.startswith("0 "):
                self.logger.info(msg)

    @acquire_lock
    def populate_cache(self):
        """ populate the yum cache """
        for repo in self.yumbase.repos.findRepos('*'):
            repo.metadata_expire = 0
            repo.mdpolicy = "group:all"
        self.yumbase.doRepoSetup()
        self.yumbase.repos.doSetup()
        for repo in self.yumbase.repos.listEnabled():
            # this populates the cache as a side effect
            repo.repoXML  # pylint: disable=W0104
            try:
                repo.getGroups()
            except yum.Errors.RepoMDError:
                pass  # this repo has no groups
        self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1)
        self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1)
        self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1)
        # this does something with the groups cache as a side effect
        self.yumbase.comps  # pylint: disable=W0104


def main():
    parser = OptionParser()
    parser.add_option("-c", "--config", help="Config file")
    parser.add_option("-v", "--verbose", help="Verbosity level",
                      action="count")
    (options, args) = parser.parse_args()

    if options.verbose:
        level = logging.DEBUG
        clevel = logging.DEBUG
    else:
        level = logging.WARNING
        clevel = logging.INFO
    Bcfg2.Logger.setup_logging('bcfg2-yum-helper', to_syslog=True,
                               to_console=clevel, level=level)
    logger = logging.getLogger('bcfg2-yum-helper')

    try:
        cmd = args[0]
    except IndexError:
        logger.error("No command given")
        return 1

    if not os.path.exists(options.config):
        logger.error("Config file %s not found" % options.config)
        return 1

    # pylint: disable=W0702
    rv = 0
    if cmd == "clean":
        cachemgr = CacheManager(options.config, options.verbose)
        try:
            cachemgr.clean_cache()
            print(json.dumps(True))
        except:
            logger.error("Unexpected error cleaning cache: %s" %
                         sys.exc_info()[1], exc_info=1)
            print(json.dumps(False))
            rv = 2
    elif cmd == "makecache":
        cachemgr = CacheManager(options.config, options.verbose)
        try:
            # this code copied from yumcommands.py
            cachemgr.populate_cache()
            print(json.dumps(True))
        except yum.Errors.YumBaseError:
            logger.error("Unexpected error creating cache: %s" %
                         sys.exc_info()[1], exc_info=1)
            print(json.dumps(False))
    elif cmd == "complete":
        depsolver = DepSolver(options.config, options.verbose)
        try:
            data = json.loads(sys.stdin.read())
        except:
            logger.error("Unexpected error decoding JSON input: %s" %
                         sys.exc_info()[1])
            rv = 2
        try:
            depsolver.groups = data['groups']
            (packages, unknown) = depsolver.complete(
                [pkg_to_tuple(p) for p in data['packages']])
            print(json.dumps(dict(packages=list(packages),
                                  unknown=list(unknown))))
        except:
            logger.error("Unexpected error completing package set: %s" %
                         sys.exc_info()[1], exc_info=1)
            print(json.dumps(dict(packages=[], unknown=data['packages'])))
            rv = 2
    elif cmd == "get_groups":
        depsolver = DepSolver(options.config, options.verbose)
        try:
            data = json.loads(sys.stdin.read())
            rv = dict()
            for gdata in data:
                if "type" in gdata:
                    packages = depsolver.get_group(gdata['group'],
                                                   ptype=gdata['type'])
                else:
                    packages = depsolver.get_group(gdata['group'])
                rv[gdata['group']] = list(packages)
            print(json.dumps(rv))
        except:
            logger.error("Unexpected error getting groups: %s" %
                         sys.exc_info()[1], exc_info=1)
            print(json.dumps(dict()))
            rv = 2
    else:
        logger.error("Unknown command %s" % cmd)
        print(json.dumps(None))
        rv = 2
    return rv

if __name__ == '__main__':
    sys.exit(main())