path: root/src/lib/Bcfg2/Server/Plugins/Packages/Source.py

import os
import re
import sys
import Bcfg2.Server.Plugin
from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \
     HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \
     urlopen, file, cPickle

try:
    from hashlib import md5
except ImportError:
    from md5 import md5

def fetch_url(url):
    if '@' in url:
        mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url)
        if not mobj:
            raise ValueError
        user = mobj.group(2)
        passwd = mobj.group(3)
        url = mobj.group(1) + mobj.group(4)
        auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
        auth.add_password(None, url, user, passwd)
        install_opener(build_opener(auth))
    return urlopen(url).read()


class SourceInitError(Exception):
    pass


class Source(Bcfg2.Server.Plugin.Debuggable):
    mrepo_re = re.compile(r'/RPMS\.([^/]+)')
    pulprepo_re = re.compile(r'pulp/repos/([^/]+)')
    genericrepo_re = re.compile('https?://.*?/([^/]+)/?$')
    basegroups = []

    def __init__(self, basepath, xsource, setup):
        Bcfg2.Server.Plugin.Debuggable.__init__(self)
        self.basepath = basepath
        self.xsource = xsource
        self.setup = setup
        self.essentialpkgs = set()

        try:
            self.version = xsource.find('Version').text
        except AttributeError:
            pass

        for key, tag in [('components', 'Component'), ('arches', 'Arch'),
                         ('blacklist', 'Blacklist'),
                         ('whitelist', 'Whitelist')]:
            setattr(self, key, [item.text for item in xsource.findall(tag)])
        self.server_options = dict()
        self.client_options = dict()
        opts = xsource.findall("Options")
        for el in opts:
            repoopts = dict([(k, v)
                             for k, v in el.attrib.items()
                             if k != "clientonly" and k != "serveronly"])
            if el.get("clientonly", "false").lower() == "false":
                self.server_options.update(repoopts)
            if el.get("serveronly", "false").lower() == "false":
                self.client_options.update(repoopts)

        self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]

        self.essential = xsource.get('essential', 'true').lower() == 'true'
        self.recommended = xsource.get('recommended', 'false').lower() == 'true'

        self.rawurl = xsource.get('rawurl', '')
        if self.rawurl and not self.rawurl.endswith("/"):
            self.rawurl += "/"
        self.url = xsource.get('url', '')
        if self.url and not self.url.endswith("/"):
            self.url += "/"
        self.version = xsource.get('version', '')

        # build the set of conditions to see if this source applies to
        # a given set of metadata
        self.conditions = []
        self.groups = [] # provided for some limited backwards compat
        for el in xsource.iterancestors():
            if el.tag == "Group":
                if el.get("negate", "false").lower() == "true":
                    self.conditions.append(lambda m, el=el:
                                           el.get("name") not in m.groups)
                else:
                    self.groups.append(el.get("name"))
                    self.conditions.append(lambda m, el=el:
                                           el.get("name") in m.groups)
            elif el.tag == "Client":
                if el.get("negate", "false").lower() == "true":
                    self.conditions.append(lambda m, el=el:
                                           el.get("name") != m.hostname)
                else:
                    self.conditions.append(lambda m, el=el:
                                           el.get("name") == m.hostname)

        self.deps = dict()
        self.provides = dict()

        self.cachefile = os.path.join(self.basepath,
                                      "cache-%s" % self.cachekey)
        self.url_map = []

    @property
    def cachekey(self):
        return md5(cPickle.dumps([self.version, self.components, self.url,
                                  self.rawurl, self.arches])).hexdigest()

    def get_relevant_groups(self, metadata):
        return sorted(list(set([g for g in metadata.groups
                                if (g in self.basegroups or
                                    g in self.groups or
                                    g in self.arches)])))

    def load_state(self):
        pass

    def setup_data(self, force_update=False):
        should_read = True
        should_download = False
        if os.path.exists(self.cachefile):
            try:
                self.load_state()
                should_read = False
            except:
                self.logger.error("Packages: Cachefile %s load failed; "
                                  "falling back to file read" % self.cachefile)
        if should_read:
            try:
                self.read_files()
            except:
                self.logger.error("Packages: File read failed; "
                                  "falling back to file download")
                should_download = True

        if should_download or force_update:
            try:
                self.update()
                self.read_files()
            except:
                self.logger.error("Packages: Failed to load data for Source "
                                  "of %s. Some Packages will be missing." %
                                  self.urls)

    def get_repo_name(self, url_map):
        # try to find a sensible name for a repo
        if 'components' in url_map and url_map['components']:
            # use the first component as the name
            rname = url_map['components'][0]
        else:
            name = None
            for repo_re in (self.mrepo_re,
                            self.pulprepo_re,
                            self.genericrepo_re):
                match = repo_re.search(url_map['url'])
                if match:
                    name = match.group(1)
                    break
            if name and self.groups:
                rname = "%s-%s" % (self.groups[0], name)
            elif self.groups:
                rname = self.groups[0]
            else:
                # a global source with no reasonable name.  just use
                # the full url and let the regex below make it even
                # uglier.
                rname = url_map['url']
        # see yum/__init__.py in the yum source, lines 441-449, for
        # the source of this regex.  yum doesn't like anything but
        # string.ascii_letters, string.digits, and [-_.:].  There
        # doesn't seem to be a reason for this, because yum.
        return re.sub(r'[^A-Za-z0-9-_.:]', '-', rname)

    def __str__(self):
        if self.rawurl:
            return "%s at %s" % (self.__class__.__name__, self.rawurl)
        elif self.url:
            return "%s at %s" % (self.__class__.__name__, self.url)
        else:
            return self.__class__.__name__

    def __repr__(self):
        return str(self)

    def get_urls(self):
        return []
    urls = property(get_urls)

    def get_files(self):
        return [self.escape_url(url) for url in self.urls]
    files = property(get_files)

    def get_vpkgs(self, metadata):
        agroups = ['global'] + [a for a in self.arches
                                if a in metadata.groups]
        vdict = dict()
        for agrp in agroups:
            if agrp not in self.provides:
                self.logger.warning("%s provides no packages for %s" %
                                    (self, agrp))
                continue
            for key, value in list(self.provides[agrp].items()):
                if key not in vdict:
                    vdict[key] = set(value)
                else:
                    vdict[key].update(value)
        return vdict

    def is_virtual_package(self, metadata, package):
        """ called to determine if a package is a virtual package.
        this is only invoked if the package is not listed in the dict
        returned by get_vpkgs """
        return False

    def escape_url(self, url):
        return os.path.join(self.basepath, url.replace('/', '@'))

    def file_init(self):
        pass

    def read_files(self):
        pass

    def filter_unknown(self, unknown):
        pass

    def update(self):
        for url in self.urls:
            self.logger.info("Packages: Updating %s" % url)
            fname = self.escape_url(url)
            try:
                data = fetch_url(url)
                file(fname, 'w').write(data)
            except ValueError:
                self.logger.error("Packages: Bad url string %s" % url)
                raise
            except HTTPError:
                err = sys.exc_info()[1]
                self.logger.error("Packages: Failed to fetch url %s. HTTP "
                                  "response code=%s" % (url, err.code))
                raise

    def applies(self, metadata):
        # check base groups
        if not self.magic_groups_match(metadata):
            return False

        # check Group/Client tags from sources.xml
        for condition in self.conditions:
            if not condition(metadata):
                return False

        return True

    def get_arches(self, metadata):
        return ['global'] + [a for a in self.arches if a in metadata.groups]

    def get_deps(self, metadata, pkgname):
        for arch in self.get_arches(metadata):
            if pkgname in self.deps[arch]:
                return self.deps[arch][pkgname]
        return []

    def get_provides(self, metadata, required):
        for arch in self.get_arches(metadata):
            if required in self.provides[arch]:
                return self.provides[arch][required]
        return []

    def is_package(self, metadata, _):
        return False

    def get_package(self, metadata, package):
        return package

    def get_group(self, metadata, group, ptype=None):
        return []

    def magic_groups_match(self, metadata):
        """ check to see if this source applies to the given host
        metadata by checking 'magic' (base) groups only, or if magic
        groups are off """
        # we always check that arch matches
        found_arch = False
        for arch in self.arches:
            if arch in metadata.groups:
                found_arch = True
                break
        if not found_arch:
            return False

        if not self.setup.cfp.getboolean("packages", "magic_groups",
                                         default=True):
            return True
        else:
            for group in self.basegroups:
                if group in metadata.groups:
                    return True
            return False