From dab1d03d81c538966d03fb9318a4588a9e803b44 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 24 Mar 2012 11:20:07 -0500 Subject: Allow to run directly from a git checkout (#1037) Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Packages/Yum.py | 688 --------------------------------- 1 file changed, 688 deletions(-) delete mode 100644 src/lib/Server/Plugins/Packages/Yum.py (limited to 'src/lib/Server/Plugins/Packages/Yum.py') diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py deleted file mode 100644 index e13b28251..000000000 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ /dev/null @@ -1,688 +0,0 @@ -import os -import sys -import time -import copy -import glob -import socket -import random -import logging -import threading -import lxml.etree -from UserDict import DictMixin -from subprocess import Popen, PIPE, STDOUT -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file -from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ - fetch_url - -logger = logging.getLogger(__name__) - -try: - from pulp.client.consumer.config import ConsumerConfig - from pulp.client.api.repository import RepositoryAPI - from pulp.client.api.consumer import ConsumerAPI - from pulp.client.api import server - has_pulp = True -except ImportError: - has_pulp = False - -try: - import yum - has_yum = True -except ImportError: - has_yum = False - logger.info("Packages: No yum libraries found; forcing use of internal " - "dependency resolver") - -try: - import json -except ImportError: - import simplejson as json - -XP = '{http://linux.duke.edu/metadata/common}' -RP = '{http://linux.duke.edu/metadata/rpm}' -RPO = '{http://linux.duke.edu/metadata/repo}' -FL = '{http://linux.duke.edu/metadata/filelists}' - -PULPSERVER = None -PULPCONFIG = None - - -def _setup_pulp(config): - global PULPSERVER, PULPCONFIG - if not has_pulp: - logger.error("Packages: Cannot create Pulp collection: Pulp libraries not " - "found") - raise Bcfg2.Server.Plugin.PluginInitError - - if PULPSERVER is None: - try: - username = config.get("pulp", "username") - password = config.get("pulp", "password") - except ConfigParser.NoSectionError: - logger.error("Packages: No [pulp] section found in Packages/packages.conf") - raise Bcfg2.Server.Plugin.PluginInitError - except ConfigParser.NoOptionError: - err = sys.exc_info()[1] - logger.error("Packages: Required option not found in " - "Packages/packages.conf: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError - - PULPCONFIG = ConsumerConfig() - serveropts = PULPCONFIG.server - - PULPSERVER = server.PulpServer(serveropts['host'], - int(serveropts['port']), - serveropts['scheme'], - serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) - server.set_active_server(PULPSERVER) - return PULPSERVER - - -class YumCollection(Collection): - # options that are included in the [yum] section but that should - # not be included in the temporary yum.conf we write out - option_blacklist = ["use_yum_libraries", "helper"] - - def __init__(self, metadata, sources, basepath, debug=False): - Collection.__init__(self, metadata, sources, basepath, debug=debug) - self.keypath = os.path.join(self.basepath, "keys") - - if len(sources): - config = sources[0].config - self.use_yum = has_yum and config.getboolean("yum", - "use_yum_libraries", - default=False) - else: - self.use_yum = False - - if self.use_yum: - self.cachefile = os.path.join(self.cachepath, - "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) - - self.configdir = os.path.join(self.basepath, "yum") - if not os.path.exists(self.configdir): - os.mkdir(self.configdir) - self.cfgfile = os.path.join(self.configdir, - "%s-yum.conf" % self.cachekey) - self.write_config() - - self.helper = self.config.get("yum", "helper", - default="/usr/sbin/bcfg2-yum-helper") - if has_pulp: - _setup_pulp(self.config) - - def write_config(self): - if not os.path.exists(self.cfgfile): - yumconf = self.get_config(raw=True) - yumconf.add_section("main") - - mainopts = dict(cachedir=self.cachefile, - keepcache="0", - sslverify="0", - debuglevel="0", - reposdir="/dev/null") - try: - for opt in self.config.options("yum"): - if opt not in self.option_blacklist: - mainopts[opt] = self.config.get("yum", opt) - except ConfigParser.NoSectionError: - pass - - for opt, val in list(mainopts.items()): - yumconf.set("main", opt, val) - - yumconf.write(open(self.cfgfile, 'w')) - - def get_config(self, raw=False): - config = ConfigParser.SafeConfigParser() - for source in self.sources: - # get_urls() loads url_map as a side-effect - source.get_urls() - for url_map in source.url_map: - if url_map['arch'] in self.metadata.groups: - basereponame = source.get_repo_name(url_map) - reponame = basereponame - - added = False - while not added: - try: - config.add_section(reponame) - added = True - except ConfigParser.DuplicateSectionError: - match = re.match("-(\d)", reponame) - if match: - rid = int(match.group(1)) + 1 - else: - rid = 1 - reponame = "%s-%d" % (basereponame, rid) - - config.set(reponame, "name", reponame) - config.set(reponame, "baseurl", url_map['url']) - config.set(reponame, "enabled", "1") - if len(source.gpgkeys): - config.set(reponame, "gpgcheck", "1") - config.set(reponame, "gpgkey", - " ".join(source.gpgkeys)) - else: - config.set(reponame, "gpgcheck", "0") - - if len(source.blacklist): - config.set(reponame, "exclude", - " ".join(source.blacklist)) - if len(source.whitelist): - config.set(reponame, "includepkgs", - " ".join(source.whitelist)) - - if raw: - return config - else: - # configparser only writes to file, so we have to use a - # StringIO object to get the data out as a string - buf = StringIO() - config.write(buf) - return "# This config was generated automatically by the Bcfg2 " \ - "Packages plugin\n\n" + buf.getvalue() - - def build_extra_structures(self, independent): - """ build list of gpg keys to be added to the specification by - validate_structures() """ - needkeys = set() - for source in self.sources: - for key in source.gpgkeys: - needkeys.add(key) - - if len(needkeys): - if has_yum: - # this must be be has_yum, not use_yum, because - # regardless of whether the user wants to use the yum - # resolver we want to include gpg key data - keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey", - type=self.ptype, origin='Packages') - else: - self.logger.warning("GPGKeys were specified for yum sources in " - "sources.xml, but no yum libraries were " - "found") - self.logger.warning("GPG key version/release data cannot be " - "determined automatically") - self.logger.warning("Install yum libraries, or manage GPG keys " - "manually") - keypkg = None - - for key in needkeys: - # figure out the path of the key on the client - keydir = self.config.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") - remotekey = os.path.join(keydir, os.path.basename(key)) - localkey = os.path.join(self.keypath, os.path.basename(key)) - kdata = open(localkey).read() - - # copy the key to the client - keypath = lxml.etree.Element("BoundPath", name=remotekey, - encoding='ascii', - owner='root', group='root', - type='file', perms='0644', - important='true') - keypath.text = kdata - - # hook to add version/release info if possible - self._add_gpg_instances(keypkg, kdata, localkey, remotekey) - independent.append(keypath) - if keypkg is not None: - independent.append(keypkg) - - # see if there are any pulp sources to handle - has_pulp_sources = False - for source in self.sources: - if source.pulp_id: - has_pulp_sources = True - break - - if has_pulp_sources: - consumerapi = ConsumerAPI() - consumer = self._get_pulp_consumer(consumerapi=consumerapi) - if consumer is None: - consumer = consumerapi.create(self.metadata.hostname, - self.metadata.hostname) - lxml.etree.SubElement(independent, "BoundAction", - name="pulp-update", timing="pre", - when="always", status="check", - command="pulp-consumer consumer update") - - for source in self.sources: - # each pulp source can only have one arch, so we don't - # have to check the arch in url_map - if (source.pulp_id and - source.pulp_id not in consumer['repoids']): - consumerapi.bind(self.metadata.hostname, source.pulp_id) - - crt = lxml.etree.SubElement(independent, "BoundPath", - name="/etc/pki/consumer/cert.pem", - type="file", owner="root", - group="root", perms="0644") - crt.text = consumerapi.certificate(self.metadata.hostname) - - def _get_pulp_consumer(self, consumerapi=None): - if consumerapi is None: - consumerapi = ConsumerAPI() - consumer = None - try: - consumer = consumerapi.consumer(self.metadata.hostname) - except server.ServerRequestError: - # consumer does not exist - pass - except socket.error: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not contact Pulp server: %s" % - err) - except: - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown error querying Pulp server: %s" - % err) - return consumer - - def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey): - """ add gpg keys to the specification to ensure they get - installed """ - # this must be be has_yum, not use_yum, because regardless of - # whether the user wants to use the yum resolver we want to - # include gpg key data - if not has_yum: - return - - try: - kinfo = yum.misc.getgpgkeyinfo(keydata) - version = yum.misc.keyIdToRPMVer(kinfo['keyid']) - release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) - - lxml.etree.SubElement(keyentry, 'Instance', - version=version, - release=release, - simplefile=remotekey) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not read GPG key %s: %s" % - (localkey, err)) - - def is_package(self, package): - if not self.use_yum: - return Collection.is_package(self, package) - elif isinstance(package, tuple): - if package[1] is None and package[2] == (None, None, None): - package = package[0] - else: - return None - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("is_package", package) - - def is_virtual_package(self, package): - if not self.use_yum: - return Collection.is_virtual_package(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("is_virtual_package", package) - - def get_deps(self, package): - if not self.use_yum: - return Collection.get_deps(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("get_deps", package) - - def get_provides(self, required, all=False, silent=False): - if not self.use_yum: - return Collection.get_provides(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("get_provides", package) - - def get_group(self, group, ptype="default"): - if not self.use_yum: - self.logger.warning("Packages: Package groups are not supported by " - "Bcfg2's internal Yum dependency generator") - return [] - - if group.startswith("@"): - group = group[1:] - - pkgs = self.call_helper("get_group", dict(group=group, type=ptype)) - return pkgs - - def complete(self, packagelist): - if not self.use_yum: - return Collection.complete(self, packagelist) - - packages = set() - unknown = set(packagelist) - - if unknown: - result = \ - self.call_helper("complete", - dict(packages=list(unknown), - groups=list(self.get_relevant_groups()))) - if result and "packages" in result and "unknown" in result: - # we stringify every package because it gets returned - # in unicode; set.update() doesn't work if some - # elements are unicode and other are strings. (I.e., - # u'foo' and 'foo' get treated as unique elements.) - packages.update([str(p) for p in result['packages']]) - unknown = set([str(p) for p in result['unknown']]) - - self.filter_unknown(unknown) - - return packages, unknown - - def call_helper(self, command, input=None): - """ Make a call to bcfg2-yum-helper. The yum libs have - horrific memory leaks, so apparently the right way to get - around that in long-running processes it to have a short-lived - helper. No, seriously -- check out the yum-updatesd code. - It's pure madness. """ - # it'd be nice if we could change this to be more verbose if - # -v was given to bcfg2-server, but Collection objects don't - # get the 'setup' variable, so we don't know how verbose - # bcfg2-server is. It'd also be nice if we could tell yum to - # log to syslog. So would a unicorn. - cmd = [self.helper, "-c", self.cfgfile] - if self.debug_flag: - cmd.append("-v") - cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd)) - try: - helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to execute %s: %s" % - (" ".join(cmd), err)) - return None - - if input: - idata = json.dumps(input) - (stdout, stderr) = helper.communicate(idata) - else: - (stdout, stderr) = helper.communicate() - rv = helper.wait() - if rv: - self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, stderr)) - elif self.debug_flag: - self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr) - try: - return json.loads(stdout) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: error reading bcfg2-yum-helper " - "output: %s" % err) - return None - - def setup_data(self, force_update=False): - if not self.use_yum: - return Collection.setup_data(self, force_update) - - if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config - self.call_helper("clean") - - os.unlink(self.cfgfile) - self.write_config() - - if force_update: - self.call_helper("clean") - - -class YumSource(Source): - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - ptype = 'yum' - - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) - self.pulp_id = None - if has_pulp and xsource.get("pulp_id"): - self.pulp_id = xsource.get("pulp_id") - - _setup_pulp(self.config) - repoapi = RepositoryAPI() - try: - self.repo = repoapi.repository(self.pulp_id) - self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key) - for key in repoapi.listkeys(self.pulp_id)] - except server.ServerRequestError: - err = sys.exc_info()[1] - if err[0] == 401: - msg = "Packages: Error authenticating to Pulp: %s" % err[1] - elif err[0] == 404: - msg = "Packages: Pulp repo id %s not found: %s" % \ - (self.pulp_id, err[1]) - else: - msg = "Packages: Error %d fetching pulp repo %s: %s" % \ - (err[0], self.pulp_id, err[1]) - raise SourceInitError(msg) - except socket.error: - err = sys.exc_info()[1] - raise SourceInitError("Could not contact Pulp server: %s" % err) - except: - err = sys.exc_info()[1] - raise SourceInitError("Unknown error querying Pulp server: %s" % - err) - self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'], - self.repo['relative_path']) - self.arches = [self.repo['arch']] - - if not self.rawurl: - self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" - else: - self.baseurl = self.rawurl - self.packages = dict() - self.deps = dict([('global', dict())]) - self.provides = dict([('global', dict())]) - self.filemap = dict([(x, dict()) - for x in ['global'] + self.arches]) - self.needed_paths = set() - self.file_to_arch = dict() - - self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries", - default=False) - - def save_state(self): - if not self.use_yum: - cache = file(self.cachefile, 'wb') - cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) - cache.close() - - - def load_state(self): - if not self.use_yum: - data = file(self.cachefile) - (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) - - def get_urls(self): - surls = list() - self.url_map = [] - for arch in self.arches: - if self.url: - usettings = [{'version':self.version, 'component':comp, - 'arch':arch} - for comp in self.components] - else: # rawurl given - usettings = [{'version':self.version, 'component':None, - 'arch':arch}] - - for setting in usettings: - setting['url'] = self.baseurl % setting - self.url_map.append(copy.deepcopy(setting)) - surls.append((arch, [setting['url'] for setting in usettings])) - urls = [] - for (sarch, surl_list) in surls: - for surl in surl_list: - urls.extend(self._get_urls_from_repodata(surl, sarch)) - return urls - urls = property(get_urls) - - def _get_urls_from_repodata(self, url, arch): - if self.use_yum: - return [url] - - rmdurl = '%srepodata/repomd.xml' % url - try: - repomd = fetch_url(rmdurl) - xdata = lxml.etree.XML(repomd) - except ValueError: - self.logger.error("Packages: Bad url string %s" % rmdurl) - return [] - except HTTPError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to fetch url %s. code=%s" % - (rmdurl, err.code)) - return [] - except lxml.etree.XMLSyntaxError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to process metadata at %s: %s" % - (rmdurl, err)) - return [] - - urls = [] - for elt in xdata.findall(RPO + 'data'): - if elt.get('type') in ['filelists', 'primary']: - floc = elt.find(RPO + 'location') - fullurl = url + floc.get('href') - urls.append(fullurl) - self.file_to_arch[self.escape_url(fullurl)] = arch - return urls - - def read_files(self): - # we have to read primary.xml first, and filelists.xml afterwards; - primaries = list() - filelists = list() - for fname in self.files: - if fname.endswith('primary.xml.gz'): - primaries.append(fname) - elif fname.endswith('filelists.xml.gz'): - filelists.append(fname) - - for fname in primaries: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_primary(fdata, farch) - for fname in filelists: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_filelist(fdata, farch) - - # merge data - sdata = list(self.packages.values()) - try: - self.packages['global'] = copy.deepcopy(sdata.pop()) - except IndexError: - logger.error("Packages: No packages in repo") - while sdata: - self.packages['global'] = \ - self.packages['global'].intersection(sdata.pop()) - - for key in self.packages: - if key == 'global': - continue - self.packages[key] = \ - self.packages[key].difference(self.packages['global']) - self.save_state() - - def parse_filelist(self, data, arch): - if arch not in self.filemap: - self.filemap[arch] = dict() - for pkg in data.findall(FL + 'package'): - for fentry in pkg.findall(FL + 'file'): - if fentry.text in self.needed_paths: - if fentry.text in self.filemap[arch]: - self.filemap[arch][fentry.text].add(pkg.get('name')) - else: - self.filemap[arch][fentry.text] = \ - set([pkg.get('name')]) - - def parse_primary(self, data, arch): - if arch not in self.packages: - self.packages[arch] = set() - if arch not in self.deps: - self.deps[arch] = dict() - if arch not in self.provides: - self.provides[arch] = dict() - for pkg in data.getchildren(): - if not pkg.tag.endswith('package'): - continue - pkgname = pkg.find(XP + 'name').text - self.packages[arch].add(pkgname) - - pdata = pkg.find(XP + 'format') - self.deps[arch][pkgname] = set() - pre = pdata.find(RP + 'requires') - if pre is not None: - for entry in pre.getchildren(): - self.deps[arch][pkgname].add(entry.get('name')) - if entry.get('name').startswith('/'): - self.needed_paths.add(entry.get('name')) - pro = pdata.find(RP + 'provides') - if pro != None: - for entry in pro.getchildren(): - prov = entry.get('name') - if prov not in self.provides[arch]: - self.provides[arch][prov] = list() - self.provides[arch][prov].append(pkgname) - - def is_package(self, metadata, item): - arch = [a for a in self.arches if a in metadata.groups] - if not arch: - return False - return ((item in self.packages['global'] or - item in self.packages[arch[0]]) and - item not in self.blacklist and - (len(self.whitelist) == 0 or item in self.whitelist)) - - def get_vpkgs(self, metadata): - if self.use_yum: - return dict() - - rv = Source.get_vpkgs(self, metadata) - for arch, fmdata in list(self.filemap.items()): - if arch not in metadata.groups and arch != 'global': - continue - for filename, pkgs in list(fmdata.items()): - rv[filename] = pkgs - return rv - - def filter_unknown(self, unknown): - if self.use_yum: - filtered = set() - for unk in unknown: - try: - if unk.startswith('rpmlib'): - filtered.update(unk) - except AttributeError: - try: - if unk[0].startswith('rpmlib'): - filtered.update(unk) - except (IndexError, AttributeError): - pass - else: - filtered = set([u for u in unknown if u.startswith('rpmlib')]) - unknown.difference_update(filtered) - - def setup_data(self, force_update=False): - if not self.use_yum: - Source.setup_data(self, force_update=force_update) - - def get_repo_name(self, url_map): - if self.pulp_id: - return self.pulp_id - else: - return Source.get_repo_name(self, url_map) -- cgit v1.2.3-1-g7c22