Allow to run directly from a git checkout (#1037)

Signed-off-by: Sol Jerome <sol.jerome@gmail.com>

1 files changed, 0 insertions, 688 deletions

diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py
deleted file mode 100644
index e13b28251..000000000
--- a/src/lib/Server/Plugins/Packages/Yum.py
+++ /dev/null
@@ -1,688 +0,0 @@
-import os
-import sys
-import time
-import copy
-import glob
-import socket
-import random
-import logging
-import threading
-import lxml.etree
-from UserDict import DictMixin
-from subprocess import Popen, PIPE, STDOUT
-import Bcfg2.Server.Plugin
-from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file
-from Bcfg2.Server.Plugins.Packages.Collection import Collection
-from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \
-     fetch_url
-
-logger = logging.getLogger(__name__)
-
-try:
-    from pulp.client.consumer.config import ConsumerConfig
-    from pulp.client.api.repository import RepositoryAPI
-    from pulp.client.api.consumer import ConsumerAPI
-    from pulp.client.api import server
-    has_pulp = True
-except ImportError:
-    has_pulp = False
-
-try:
-    import yum
-    has_yum = True
-except ImportError:
-    has_yum = False
-    logger.info("Packages: No yum libraries found; forcing use of internal "
-                "dependency resolver")
-
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-XP = '{http://linux.duke.edu/metadata/common}'
-RP = '{http://linux.duke.edu/metadata/rpm}'
-RPO = '{http://linux.duke.edu/metadata/repo}'
-FL = '{http://linux.duke.edu/metadata/filelists}'
-
-PULPSERVER = None
-PULPCONFIG = None
-
-
-def _setup_pulp(config):
-    global PULPSERVER, PULPCONFIG
-    if not has_pulp:
-        logger.error("Packages: Cannot create Pulp collection: Pulp libraries not "
-                     "found")
-        raise Bcfg2.Server.Plugin.PluginInitError
-
-    if PULPSERVER is None:
-        try:
-            username = config.get("pulp", "username")
-            password = config.get("pulp", "password")
-        except ConfigParser.NoSectionError:
-            logger.error("Packages: No [pulp] section found in Packages/packages.conf")
-            raise Bcfg2.Server.Plugin.PluginInitError
-        except ConfigParser.NoOptionError:
-            err = sys.exc_info()[1]
-            logger.error("Packages: Required option not found in "
-                         "Packages/packages.conf: %s" % err)
-            raise Bcfg2.Server.Plugin.PluginInitError
-        
-        PULPCONFIG = ConsumerConfig()
-        serveropts = PULPCONFIG.server
-        
-        PULPSERVER = server.PulpServer(serveropts['host'],
-                                       int(serveropts['port']),
-                                       serveropts['scheme'],
-                                       serveropts['path'])
-        PULPSERVER.set_basic_auth_credentials(username, password)
-        server.set_active_server(PULPSERVER)
-    return PULPSERVER
-
-
-class YumCollection(Collection):
-    # options that are included in the [yum] section but that should
-    # not be included in the temporary yum.conf we write out
-    option_blacklist = ["use_yum_libraries", "helper"]
-    
-    def __init__(self, metadata, sources, basepath, debug=False):
-        Collection.__init__(self, metadata, sources, basepath, debug=debug)
-        self.keypath = os.path.join(self.basepath, "keys")
-
-        if len(sources):
-            config = sources[0].config
-            self.use_yum = has_yum and config.getboolean("yum",
-                                                         "use_yum_libraries",
-                                                         default=False)
-        else:
-            self.use_yum = False
-
-        if self.use_yum:
-            self.cachefile = os.path.join(self.cachepath,
-                                         "cache-%s" % self.cachekey)
-            if not os.path.exists(self.cachefile):
-                os.mkdir(self.cachefile)
-                
-            self.configdir = os.path.join(self.basepath, "yum")
-            if not os.path.exists(self.configdir):
-                os.mkdir(self.configdir)
-            self.cfgfile = os.path.join(self.configdir,
-                                        "%s-yum.conf" % self.cachekey)
-            self.write_config()
-
-            self.helper = self.config.get("yum", "helper",
-                                          default="/usr/sbin/bcfg2-yum-helper")
-        if has_pulp:
-            _setup_pulp(self.config)
-
-    def write_config(self):
-        if not os.path.exists(self.cfgfile):
-            yumconf = self.get_config(raw=True)
-            yumconf.add_section("main")
-            
-            mainopts = dict(cachedir=self.cachefile,
-                            keepcache="0",
-                            sslverify="0",
-                            debuglevel="0",
-                            reposdir="/dev/null")
-            try:
-                for opt in self.config.options("yum"):
-                    if opt not in self.option_blacklist:
-                        mainopts[opt] = self.config.get("yum", opt)
-            except ConfigParser.NoSectionError:
-                pass
-
-            for opt, val in list(mainopts.items()):
-                yumconf.set("main", opt, val)
-
-            yumconf.write(open(self.cfgfile, 'w'))
-
-    def get_config(self, raw=False):
-        config = ConfigParser.SafeConfigParser()
-        for source in self.sources:
-            # get_urls() loads url_map as a side-effect
-            source.get_urls()
-            for url_map in source.url_map:
-                if url_map['arch'] in self.metadata.groups:
-                    basereponame = source.get_repo_name(url_map)
-                    reponame = basereponame
-
-                    added = False
-                    while not added:
-                        try:
-                            config.add_section(reponame)
-                            added = True
-                        except ConfigParser.DuplicateSectionError:
-                            match = re.match("-(\d)", reponame)
-                            if match:
-                                rid = int(match.group(1)) + 1
-                            else:
-                                rid = 1
-                            reponame = "%s-%d" % (basereponame, rid)
-                        
-                    config.set(reponame, "name", reponame)
-                    config.set(reponame, "baseurl", url_map['url'])
-                    config.set(reponame, "enabled", "1")
-                    if len(source.gpgkeys):
-                        config.set(reponame, "gpgcheck", "1")
-                        config.set(reponame, "gpgkey",
-                                   " ".join(source.gpgkeys))
-                    else:
-                        config.set(reponame, "gpgcheck", "0")
-
-                    if len(source.blacklist):
-                        config.set(reponame, "exclude",
-                                   " ".join(source.blacklist))
-                    if len(source.whitelist):
-                        config.set(reponame, "includepkgs",
-                                   " ".join(source.whitelist))
-
-        if raw:
-            return config
-        else:
-            # configparser only writes to file, so we have to use a
-            # StringIO object to get the data out as a string
-            buf = StringIO()
-            config.write(buf)
-            return "# This config was generated automatically by the Bcfg2 " \
-                   "Packages plugin\n\n" + buf.getvalue()
-
-    def build_extra_structures(self, independent):
-        """ build list of gpg keys to be added to the specification by
-        validate_structures() """
-        needkeys = set()
-        for source in self.sources:
-            for key in source.gpgkeys:
-                needkeys.add(key)
-
-        if len(needkeys):
-            if has_yum:
-                # this must be be has_yum, not use_yum, because
-                # regardless of whether the user wants to use the yum
-                # resolver we want to include gpg key data
-                keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey",
-                                            type=self.ptype, origin='Packages')
-            else:
-                self.logger.warning("GPGKeys were specified for yum sources in "
-                                    "sources.xml, but no yum libraries were "
-                                    "found")
-                self.logger.warning("GPG key version/release data cannot be "
-                                    "determined automatically")
-                self.logger.warning("Install yum libraries, or manage GPG keys "
-                                    "manually")
-                keypkg = None
-
-            for key in needkeys:
-                # figure out the path of the key on the client
-                keydir = self.config.get("global", "gpg_keypath",
-                                         default="/etc/pki/rpm-gpg")
-                remotekey = os.path.join(keydir, os.path.basename(key))
-                localkey = os.path.join(self.keypath, os.path.basename(key))
-                kdata = open(localkey).read()
-                
-                # copy the key to the client
-                keypath = lxml.etree.Element("BoundPath", name=remotekey,
-                                             encoding='ascii',
-                                             owner='root', group='root',
-                                             type='file', perms='0644',
-                                             important='true')
-                keypath.text = kdata
-
-                # hook to add version/release info if possible
-                self._add_gpg_instances(keypkg, kdata, localkey, remotekey)
-                independent.append(keypath)
-            if keypkg is not None:
-                independent.append(keypkg)
-
-        # see if there are any pulp sources to handle
-        has_pulp_sources = False
-        for source in self.sources:
-            if source.pulp_id:
-                has_pulp_sources = True
-                break
-
-        if has_pulp_sources:
-            consumerapi = ConsumerAPI()
-            consumer = self._get_pulp_consumer(consumerapi=consumerapi)
-            if consumer is None:
-                consumer = consumerapi.create(self.metadata.hostname,
-                                              self.metadata.hostname)
-                lxml.etree.SubElement(independent, "BoundAction",
-                                      name="pulp-update", timing="pre",
-                                      when="always", status="check",
-                                      command="pulp-consumer consumer update")
-
-            for source in self.sources:
-                # each pulp source can only have one arch, so we don't
-                # have to check the arch in url_map
-                if (source.pulp_id and
-                    source.pulp_id not in consumer['repoids']):
-                    consumerapi.bind(self.metadata.hostname, source.pulp_id)
-
-            crt = lxml.etree.SubElement(independent, "BoundPath",
-                                        name="/etc/pki/consumer/cert.pem",
-                                        type="file", owner="root",
-                                        group="root", perms="0644")
-            crt.text = consumerapi.certificate(self.metadata.hostname)
-
-    def _get_pulp_consumer(self, consumerapi=None):
-        if consumerapi is None:
-            consumerapi = ConsumerAPI()
-        consumer = None
-        try:
-            consumer = consumerapi.consumer(self.metadata.hostname)
-        except server.ServerRequestError:
-            # consumer does not exist
-            pass
-        except socket.error:
-            err = sys.exc_info()[1]
-            self.logger.error("Packages: Could not contact Pulp server: %s" %
-                              err)
-        except:
-            err = sys.exc_info()[1]
-            self.logger.error("Packages: Unknown error querying Pulp server: %s"
-                              % err)
-        return consumer
-
-    def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey):
-        """ add gpg keys to the specification to ensure they get
-        installed """
-        # this must be be has_yum, not use_yum, because regardless of
-        # whether the user wants to use the yum resolver we want to
-        # include gpg key data
-        if not has_yum:
-            return
-        
-        try:
-            kinfo = yum.misc.getgpgkeyinfo(keydata)
-            version = yum.misc.keyIdToRPMVer(kinfo['keyid'])
-            release = yum.misc.keyIdToRPMVer(kinfo['timestamp'])
-            
-            lxml.etree.SubElement(keyentry, 'Instance',
-                                  version=version,
-                                  release=release,
-                                  simplefile=remotekey)
-        except ValueError:
-            err = sys.exc_info()[1]
-            self.logger.error("Packages: Could not read GPG key %s: %s" %
-                              (localkey, err))
-
-    def is_package(self, package):
-        if not self.use_yum:
-            return Collection.is_package(self, package)
-        elif isinstance(package, tuple):
-            if package[1] is None and package[2] == (None, None, None):
-                package = package[0]
-            else:
-                return None
-        else:
-            # this should really never get called; it's just provided
-            # for API completeness
-            return self.call_helper("is_package", package)
-
-    def is_virtual_package(self, package):
-        if not self.use_yum:
-            return Collection.is_virtual_package(self, package)
-        else:
-            # this should really never get called; it's just provided
-            # for API completeness
-            return self.call_helper("is_virtual_package", package)
-
-    def get_deps(self, package):
-        if not self.use_yum:
-            return Collection.get_deps(self, package)
-        else:
-            # this should really never get called; it's just provided
-            # for API completeness
-            return self.call_helper("get_deps", package)
-
-    def get_provides(self, required, all=False, silent=False):
-        if not self.use_yum:
-            return Collection.get_provides(self, package)
-        else:
-            # this should really never get called; it's just provided
-            # for API completeness
-            return self.call_helper("get_provides", package)
-
-    def get_group(self, group, ptype="default"):
-        if not self.use_yum:
-            self.logger.warning("Packages: Package groups are not supported by "
-                                "Bcfg2's internal Yum dependency generator")
-            return []
-
-        if group.startswith("@"):
-            group = group[1:]
-
-        pkgs = self.call_helper("get_group", dict(group=group, type=ptype))
-        return pkgs
-
-    def complete(self, packagelist):
-        if not self.use_yum:
-            return Collection.complete(self, packagelist)
-
-        packages = set()
-        unknown = set(packagelist)
-
-        if unknown:
-            result = \
-                self.call_helper("complete",
-                                 dict(packages=list(unknown),
-                                      groups=list(self.get_relevant_groups())))
-            if result and "packages" in result and "unknown" in result:
-                # we stringify every package because it gets returned
-                # in unicode; set.update() doesn't work if some
-                # elements are unicode and other are strings.  (I.e.,
-                # u'foo' and 'foo' get treated as unique elements.)
-                packages.update([str(p) for p in result['packages']])
-                unknown = set([str(p) for p in result['unknown']])
-
-            self.filter_unknown(unknown)
-        
-        return packages, unknown
-
-    def call_helper(self, command, input=None):
-        """ Make a call to bcfg2-yum-helper.  The yum libs have
-        horrific memory leaks, so apparently the right way to get
-        around that in long-running processes it to have a short-lived
-        helper.  No, seriously -- check out the yum-updatesd code.
-        It's pure madness. """
-        # it'd be nice if we could change this to be more verbose if
-        # -v was given to bcfg2-server, but Collection objects don't
-        # get the 'setup' variable, so we don't know how verbose
-        # bcfg2-server is.  It'd also be nice if we could tell yum to
-        # log to syslog.  So would a unicorn.
-        cmd = [self.helper, "-c", self.cfgfile]
-        if self.debug_flag:
-            cmd.append("-v")
-        cmd.append(command)
-        self.debug_log("Packages: running %s" % " ".join(cmd))
-        try:
-            helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-        except OSError:
-            err = sys.exc_info()[1]
-            self.logger.error("Packages: Failed to execute %s: %s" %
-                              (" ".join(cmd), err))
-            return None
-        
-        if input:
-            idata = json.dumps(input)
-            (stdout, stderr) = helper.communicate(idata)
-        else:
-            (stdout, stderr) = helper.communicate()
-        rv = helper.wait()
-        if rv:
-            self.logger.error("Packages: error running bcfg2-yum-helper "
-                              "(returned %d): %s" % (rv, stderr))
-        elif self.debug_flag:
-            self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" %
-                           stderr)
-        try:
-            return json.loads(stdout)
-        except ValueError:
-            err = sys.exc_info()[1]
-            self.logger.error("Packages: error reading bcfg2-yum-helper "
-                              "output: %s" % err)
-            return None
-
-    def setup_data(self, force_update=False):
-        if not self.use_yum:
-            return Collection.setup_data(self, force_update)
-
-        if force_update:
-            # we call this twice: one to clean up data from the old
-            # config, and once to clean up data from the new config
-            self.call_helper("clean")
-
-        os.unlink(self.cfgfile)
-        self.write_config()
-        
-        if force_update:
-            self.call_helper("clean")
-
-
-class YumSource(Source):
-    basegroups = ['yum', 'redhat', 'centos', 'fedora']
-    ptype = 'yum'
-
-    def __init__(self, basepath, xsource, config):
-        Source.__init__(self, basepath, xsource, config)
-        self.pulp_id = None
-        if has_pulp and xsource.get("pulp_id"):
-            self.pulp_id = xsource.get("pulp_id")
-            
-            _setup_pulp(self.config)
-            repoapi = RepositoryAPI()
-            try:
-                self.repo = repoapi.repository(self.pulp_id)
-                self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key)
-                                for key in repoapi.listkeys(self.pulp_id)]
-            except server.ServerRequestError:
-                err = sys.exc_info()[1]
-                if err[0] == 401:
-                    msg = "Packages: Error authenticating to Pulp: %s" % err[1]
-                elif err[0] == 404:
-                    msg = "Packages: Pulp repo id %s not found: %s" % \
-                          (self.pulp_id, err[1])
-                else:
-                    msg = "Packages: Error %d fetching pulp repo %s: %s" % \
-                          (err[0], self.pulp_id, err[1])
-                raise SourceInitError(msg)
-            except socket.error:
-                err = sys.exc_info()[1]
-                raise SourceInitError("Could not contact Pulp server: %s" % err)
-            except:
-                err = sys.exc_info()[1]
-                raise SourceInitError("Unknown error querying Pulp server: %s" %
-                                      err)
-            self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'],
-                                     self.repo['relative_path'])
-            self.arches = [self.repo['arch']]
-        
-        if not self.rawurl:
-            self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/"
-        else:
-            self.baseurl = self.rawurl
-        self.packages = dict()
-        self.deps = dict([('global', dict())])
-        self.provides = dict([('global', dict())])
-        self.filemap = dict([(x, dict())
-                             for x in ['global'] + self.arches])
-        self.needed_paths = set()
-        self.file_to_arch = dict()
-
-        self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries",
-                                                     default=False)
-
-    def save_state(self):
-        if not self.use_yum:
-            cache = file(self.cachefile, 'wb')
-            cPickle.dump((self.packages, self.deps, self.provides,
-                          self.filemap, self.url_map), cache, 2)
-            cache.close()
-            
-
-    def load_state(self):
-        if not self.use_yum:
-            data = file(self.cachefile)
-            (self.packages, self.deps, self.provides,
-             self.filemap, self.url_map) = cPickle.load(data)
-
-    def get_urls(self):
-        surls = list()
-        self.url_map = []
-        for arch in self.arches:
-            if self.url:
-                usettings = [{'version':self.version, 'component':comp,
-                              'arch':arch}
-                             for comp in self.components]
-            else: # rawurl given 
-                usettings = [{'version':self.version, 'component':None,
-                              'arch':arch}]
-
-            for setting in usettings:
-                setting['url'] = self.baseurl % setting
-                self.url_map.append(copy.deepcopy(setting))
-            surls.append((arch, [setting['url'] for setting in usettings]))
-        urls = []
-        for (sarch, surl_list) in surls:
-            for surl in surl_list:
-                urls.extend(self._get_urls_from_repodata(surl, sarch))
-        return urls
-    urls = property(get_urls)
-
-    def _get_urls_from_repodata(self, url, arch):
-        if self.use_yum:
-            return [url]
-        
-        rmdurl = '%srepodata/repomd.xml' % url
-        try:
-            repomd = fetch_url(rmdurl)
-            xdata = lxml.etree.XML(repomd)
-        except ValueError:
-            self.logger.error("Packages: Bad url string %s" % rmdurl)
-            return []
-        except HTTPError:
-            err = sys.exc_info()[1]
-            self.logger.error("Packages: Failed to fetch url %s. code=%s" %
-                              (rmdurl, err.code))
-            return []
-        except lxml.etree.XMLSyntaxError:
-            err = sys.exc_info()[1]
-            self.logger.error("Packages: Failed to process metadata at %s: %s" %
-                              (rmdurl, err))
-            return []
-
-        urls = []
-        for elt in xdata.findall(RPO + 'data'):
-            if elt.get('type') in ['filelists', 'primary']:
-                floc = elt.find(RPO + 'location')
-                fullurl = url + floc.get('href')
-                urls.append(fullurl)
-                self.file_to_arch[self.escape_url(fullurl)] = arch
-        return urls
-
-    def read_files(self):
-        # we have to read primary.xml first, and filelists.xml afterwards;
-        primaries = list()
-        filelists = list()
-        for fname in self.files:
-            if fname.endswith('primary.xml.gz'):
-                primaries.append(fname)
-            elif fname.endswith('filelists.xml.gz'):
-                filelists.append(fname)
-
-        for fname in primaries:
-            farch = self.file_to_arch[fname]
-            fdata = lxml.etree.parse(fname).getroot()
-            self.parse_primary(fdata, farch)
-        for fname in filelists:
-            farch = self.file_to_arch[fname]
-            fdata = lxml.etree.parse(fname).getroot()
-            self.parse_filelist(fdata, farch)
-
-        # merge data
-        sdata = list(self.packages.values())
-        try:
-            self.packages['global'] = copy.deepcopy(sdata.pop())
-        except IndexError:
-            logger.error("Packages: No packages in repo")
-        while sdata:
-            self.packages['global'] = \
-                self.packages['global'].intersection(sdata.pop())
-
-        for key in self.packages:
-            if key == 'global':
-                continue
-            self.packages[key] = \
-                self.packages[key].difference(self.packages['global'])
-        self.save_state()
-
-    def parse_filelist(self, data, arch):
-        if arch not in self.filemap:
-            self.filemap[arch] = dict()
-        for pkg in data.findall(FL + 'package'):
-            for fentry in pkg.findall(FL + 'file'):
-                if fentry.text in self.needed_paths:
-                    if fentry.text in self.filemap[arch]:
-                        self.filemap[arch][fentry.text].add(pkg.get('name'))
-                    else:
-                        self.filemap[arch][fentry.text] = \
-                            set([pkg.get('name')])
-
-    def parse_primary(self, data, arch):
-        if arch not in self.packages:
-            self.packages[arch] = set()
-        if arch not in self.deps:
-            self.deps[arch] = dict()
-        if arch not in self.provides:
-            self.provides[arch] = dict()
-        for pkg in data.getchildren():
-            if not pkg.tag.endswith('package'):
-                continue
-            pkgname = pkg.find(XP + 'name').text
-            self.packages[arch].add(pkgname)
-
-            pdata = pkg.find(XP + 'format')
-            self.deps[arch][pkgname] = set()
-            pre = pdata.find(RP + 'requires')
-            if pre is not None:
-                for entry in pre.getchildren():
-                    self.deps[arch][pkgname].add(entry.get('name'))
-                    if entry.get('name').startswith('/'):
-                        self.needed_paths.add(entry.get('name'))
-            pro = pdata.find(RP + 'provides')
-            if pro != None:
-                for entry in pro.getchildren():
-                    prov = entry.get('name')
-                    if prov not in self.provides[arch]:
-                        self.provides[arch][prov] = list()
-                    self.provides[arch][prov].append(pkgname)
-
-    def is_package(self, metadata, item):
-        arch = [a for a in self.arches if a in metadata.groups]
-        if not arch:
-            return False
-        return ((item in self.packages['global'] or
-                 item in self.packages[arch[0]]) and
-                item not in self.blacklist and
-                (len(self.whitelist) == 0 or item in self.whitelist))
-
-    def get_vpkgs(self, metadata):
-        if self.use_yum:
-            return dict()
-        
-        rv = Source.get_vpkgs(self, metadata)
-        for arch, fmdata in list(self.filemap.items()):
-            if arch not in metadata.groups and arch != 'global':
-                continue
-            for filename, pkgs in list(fmdata.items()):
-                rv[filename] = pkgs
-        return rv
-
-    def filter_unknown(self, unknown):
-        if self.use_yum:
-            filtered = set()
-            for unk in unknown:
-                try:
-                    if unk.startswith('rpmlib'):
-                        filtered.update(unk)
-                except AttributeError:
-                    try:
-                        if unk[0].startswith('rpmlib'):
-                            filtered.update(unk)
-                    except (IndexError, AttributeError):
-                        pass
-        else:
-            filtered = set([u for u in unknown if u.startswith('rpmlib')])
-        unknown.difference_update(filtered)
-
-    def setup_data(self, force_update=False):
-        if not self.use_yum:
-            Source.setup_data(self, force_update=force_update)
-
-    def get_repo_name(self, url_map):
-        if self.pulp_id:
-            return self.pulp_id
-        else:
-            return Source.get_repo_name(self, url_map)