import os import re import sys import copy import gzip import glob import base64 import logging import tarfile import lxml.etree # Compatibility imports from Bcfg2.Bcfg2Py3k import cPickle from Bcfg2.Bcfg2Py3k import HTTPBasicAuthHandler from Bcfg2.Bcfg2Py3k import HTTPPasswordMgrWithDefaultRealm from Bcfg2.Bcfg2Py3k import HTTPError from Bcfg2.Bcfg2Py3k import install_opener from Bcfg2.Bcfg2Py3k import build_opener from Bcfg2.Bcfg2Py3k import urlopen from Bcfg2.Bcfg2Py3k import ConfigParser # py3k compatibility if sys.hexversion >= 0x03000000: from io import FileIO as BUILTIN_FILE_TYPE else: BUILTIN_FILE_TYPE = file try: import yum.misc has_yum = True except ImportError: has_yum = False try: import pulp.client.server import pulp.client.config import pulp.client.api.repository import pulp.client.api.consumer has_pulp = True except ImportError: has_pulp = False try: from hashlib import md5 except ImportError: from md5 import md5 import Bcfg2.Logger import Bcfg2.Server.Plugin # build sources.list? # caching for yum class NoData(Exception): pass class SomeData(Exception): pass logger = logging.getLogger('Packages') def source_from_xml(xsource, cachepath): """ create a *Source object from its XML representation in sources.xml """ stype = xsource.get("type") if stype is None: logger.error("No type specified for source, skipping") return None try: cls = globals()["%sSource" % stype.upper()] except KeyError: logger.error("Unknown source type %s") return None return cls(cachepath, xsource) def _fetch_url(url): if '@' in url: mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) if not mobj: raise ValueError user = mobj.group(2) passwd = mobj.group(3) url = mobj.group(1) + mobj.group(4) auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) auth.add_password(None, url, user, passwd) install_opener(build_opener(auth)) return urlopen(url).read() class Source(object): basegroups = [] def __init__(self, basepath, xsource): self.basepath = basepath self.xsource = xsource try: self.version = xsource.find('Version').text except AttributeError: pass for key, tag in [('components', 'Component'), ('arches', 'Arch'), ('blacklist', 'Blacklist'), ('whitelist', 'Whitelist')]: self.__dict__[key] = [item.text for item in xsource.findall(tag)] self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] self.recommended = xsource.get('recommended', 'false').lower() == 'true' self.id = xsource.get('id') self.rawurl = xsource.get('rawurl', '') if self.rawurl and not self.rawurl.endswith("/"): self.rawurl += "/" self.url = xsource.get('url', '') if self.url and not self.url.endswith("/"): self.url += "/" self.version = xsource.get('version', '') # build the set of conditions to see if this source applies to # a given set of metadata self.conditions = [] self.groups = [] # provided for some limited backwards compat for el in xsource.iterancestors(): if el.tag == "Group": if el.get("negate", "false").lower() == "true": self.conditions.append(lambda m, el=el: el.get("name") not in m.groups) else: self.groups.append(el.get("name")) self.conditions.append(lambda m, el=el: el.get("name") in m.groups) elif el.tag == "Client": if el.get("negate", "false").lower() == "true": self.conditions.append(lambda m, el=el: el.get("name") != m.hostname) else: self.conditions.append(lambda m, el=el: el.get("name") == m.hostname) self.deps = dict() self.provides = dict() self.cachefile = \ os.path.join(self.basepath, "cache-%s" % md5(cPickle.dumps([self.version, self.components, self.url, self.rawurl, self.arches])).hexdigest()) self.url_map = [] def load_state(self): pass def setup_data(self, force_update=False): should_read = True should_download = False if os.path.exists(self.cachefile): try: self.load_state() should_read = False except: logger.error("Cachefile %s load failed; " "falling back to file read" % self.cachefile) if should_read: try: self.read_files() except: logger.error("Packages: File read failed; " "falling back to file download") should_download = True if should_download or force_update: try: self.update() self.read_files() except: logger.error("Failed to update source", exc_info=1) def get_urls(self): return [] urls = property(get_urls) def get_files(self): return [self.escape_url(url) for url in self.urls] files = property(get_files) def get_vpkgs(self, meta): agroups = ['global'] + [a for a in self.arches if a in meta.groups] vdict = dict() for agrp in agroups: for key, value in list(self.provides[agrp].items()): if key not in vdict: vdict[key] = set(value) else: vdict[key].update(value) return vdict def escape_url(self, url): return os.path.join(self.basepath, url.replace('/', '@')) def file_init(self): pass def read_files(self): pass def update(self): for url in self.urls: logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: data = _fetch_url(url) except ValueError: logger.error("Packages: Bad url string %s" % url) continue except HTTPError: err = sys.exc_info()[1] logger.error("Packages: Failed to fetch url %s. code=%s" % (url, err.code)) continue BUILTIN_FILE_TYPE(fname, 'w').write(data) def applies(self, metadata): # check base groups if len([g for g in self.basegroups if g in metadata.groups]) == 0: return False # check Group/Client tags from sources.xml for condition in self.conditions: if not condition(metadata): return False return True def get_arches(self, metadata): return ['global'] + [a for a in self.arches if a in metadata.groups] def get_deps(self, metadata, pkgname): for arch in self.get_arches(metadata): if pkgname in self.deps[arch]: return self.deps[arch][pkgname] raise NoData def get_provides(self, metadata, required): for arch in self.get_arches(metadata): if required in self.provides[arch]: return self.provides[arch][required] raise NoData def is_package(self, metadata, _): return False class YUMSource(Source): xp = '{http://linux.duke.edu/metadata/common}' rp = '{http://linux.duke.edu/metadata/rpm}' rpo = '{http://linux.duke.edu/metadata/repo}' fl = '{http://linux.duke.edu/metadata/filelists}' basegroups = ['yum', 'redhat', 'centos', 'fedora'] ptype = 'yum' def __init__(self, basepath, xsource): Source.__init__(self, basepath, xsource) if not self.rawurl: self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" else: self.baseurl = self.rawurl self.packages = dict() self.deps = dict([('global', dict())]) self.provides = dict([('global', dict())]) self.filemap = dict([(x, dict()) for x in ['global'] + self.arches]) self.needed_paths = set() self.file_to_arch = dict() def save_state(self): cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, self.filemap, self.url_map), cache, 2) cache.close() def load_state(self): data = BUILTIN_FILE_TYPE(self.cachefile) (self.packages, self.deps, self.provides, self.filemap, self.url_map) = cPickle.load(data) def get_urls(self): surls = list() self.url_map = [] for arch in self.arches: if self.url: usettings = [{'version':self.version, 'component':comp, 'arch':arch} for comp in self.components] else: # rawurl given usettings = [{'version':self.version, 'component':None, 'arch':arch}] for setting in usettings: setting['url'] = self.baseurl % setting self.url_map.append(copy.deepcopy(setting)) surls.append((arch, [setting['url'] for setting in usettings])) urls = [] for (sarch, surl_list) in surls: for surl in surl_list: urls.extend(self._get_urls_from_repodata(surl, sarch)) return urls urls = property(get_urls) def _get_urls_from_repodata(self, url, arch): rmdurl = '%srepodata/repomd.xml' % url try: repomd = _fetch_url(rmdurl) xdata = lxml.etree.XML(repomd) except ValueError: logger.error("Packages: Bad url string %s" % rmdurl) return [] except HTTPError: err = sys.exc_info()[1] logger.error("Packages: Failed to fetch url %s. code=%s" % (rmdurl, err.code)) return [] except lxml.etree.XMLSyntaxError: err = sys.exc_info()[1] logger.error("Packages: Failed to process metadata at %s: %s" % (rmdurl, err)) return [] urls = [] for elt in xdata.findall(self.rpo + 'data'): if elt.get('type') in ['filelists', 'primary']: floc = elt.find(self.rpo + 'location') fullurl = url + floc.get('href') urls.append(fullurl) self.file_to_arch[self.escape_url(fullurl)] = arch return urls def read_files(self): # we have to read primary.xml first, and filelists.xml afterwards; primaries = list() filelists = list() for fname in self.files: if fname.endswith('primary.xml.gz'): primaries.append(fname) elif fname.endswith('filelists.xml.gz'): filelists.append(fname) for fname in primaries: farch = self.file_to_arch[fname] fdata = lxml.etree.parse(fname).getroot() self.parse_primary(fdata, farch) for fname in filelists: farch = self.file_to_arch[fname] fdata = lxml.etree.parse(fname).getroot() self.parse_filelist(fdata, farch) # merge data sdata = list(self.packages.values()) try: self.packages['global'] = copy.deepcopy(sdata.pop()) except IndexError: logger.error("No packages in repo") while sdata: self.packages['global'] = \ self.packages['global'].intersection(sdata.pop()) for key in self.packages: if key == 'global': continue self.packages[key] = \ self.packages[key].difference(self.packages['global']) self.save_state() def parse_filelist(self, data, arch): if arch not in self.filemap: self.filemap[arch] = dict() for pkg in data.findall(self.fl + 'package'): for fentry in pkg.findall(self.fl + 'file'): if fentry.text in self.needed_paths: if fentry.text in self.filemap[arch]: self.filemap[arch][fentry.text].add(pkg.get('name')) else: self.filemap[arch][fentry.text] = set([pkg.get('name')]) def parse_primary(self, data, arch): if arch not in self.packages: self.packages[arch] = set() if arch not in self.deps: self.deps[arch] = dict() if arch not in self.provides: self.provides[arch] = dict() for pkg in data.getchildren(): if not pkg.tag.endswith('package'): continue pkgname = pkg.find(self.xp + 'name').text self.packages[arch].add(pkgname) pdata = pkg.find(self.xp + 'format') pre = pdata.find(self.rp + 'requires') self.deps[arch][pkgname] = set() for entry in pre.getchildren(): self.deps[arch][pkgname].add(entry.get('name')) if entry.get('name').startswith('/'): self.needed_paths.add(entry.get('name')) pro = pdata.find(self.rp + 'provides') if pro != None: for entry in pro.getchildren(): prov = entry.get('name') if prov not in self.provides[arch]: self.provides[arch][prov] = list() self.provides[arch][prov].append(pkgname) def is_package(self, metadata, item): arch = [a for a in self.arches if a in metadata.groups] if not arch: return False return ((item in self.packages['global'] or item in self.packages[arch[0]]) and item not in self.blacklist and (len(self.whitelist) == 0 or item in self.whitelist)) def get_vpkgs(self, metadata): rv = Source.get_vpkgs(self, metadata) for arch, fmdata in list(self.filemap.items()): if arch not in metadata.groups and arch != 'global': continue for filename, pkgs in list(fmdata.items()): rv[filename] = pkgs return rv def filter_unknown(self, unknown): filtered = set([u for u in unknown if u.startswith('rpmlib')]) unknown.difference_update(filtered) class PulpSource(Source): basegroups = ['yum', 'redhat', 'centos', 'fedora'] ptype = 'yum' def __init__(self, basepath, xsource): Source.__init__(self, basepath, xsource) if not has_pulp: logger.error("Cannot create pulp source: pulp libraries not found") raise Bcfg2.Server.Plugin.PluginInitError self._config = pulp.client.config.Config() self._repoapi = pulp.client.api.repository.RepositoryAPI() self._repo = self._repoapi.repository(self.id) if self._repo is None: logger.error("Repo id %s not found") else: self.baseurl = "%s/%s" % (self._config.cds.baseurl, self._repo['relative_path']) self.gpgkeys = ["%s/%s" % (self._config.cds.keyurl, key) for key in self._repoapi.listkeys(self.id)] self.url_map = [{'version': self.version, 'component': None, 'arch': self.arches[0], 'url': self.baseurl}] def save_state(self): cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, self._config, self.filemap, self.url_map, self._repoapi, self._repo), cache, 2) cache.close() def load_state(self): cache = BUILTIN_FILE_TYPE(self.cachefile) (self.packages, self.deps, self.provides, self._config, self.filemap, self.url_map, self._repoapi, self._repo) = cPickle.load(cache) cache.close() def read_files(self): """ ignore the yum files; we can get this information directly from pulp """ for pkg in self._repoapi.packages(self.id): try: self.packages[pkg['arch']].append(pkg['name']) except KeyError: self.packages[pkg['arch']] = [pkg['name']] self.save_state() class APTSource(Source): basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] ptype = 'deb' def __init__(self, basepath, xsource): Source.__init__(self, basepath, xsource) self.pkgnames = set() self.url_map = [{'rawurl': self.rawurl, 'url': self.url, 'version': self.version, 'components': self.components, 'arches': self.arches}] def save_state(self): cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides), cache, 2) cache.close() def load_state(self): data = BUILTIN_FILE_TYPE(self.cachefile) self.pkgnames, self.deps, self.provides = cPickle.load(data) def filter_unknown(self, unknown): filtered = set([u for u in unknown if u.startswith('choice')]) unknown.difference_update(filtered) def get_urls(self): if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: rv.append("%sdists/%s/%s/binary-%s/Packages.gz" % (self.url, self.version, part, arch)) return rv else: return ["%sPackages.gz" % self.rawurl] urls = property(get_urls) def read_files(self): bdeps = dict() bprov = dict() if self.recommended: depfnames = ['Depends', 'Pre-Depends', 'Recommends'] else: depfnames = ['Depends', 'Pre-Depends'] for fname in self.files: if not self.rawurl: barch = [x for x in fname.split('@') if x.startswith('binary-')][0][7:] else: # RawURL entries assume that they only have one # element and that it is the architecture of the source. barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() bprov[barch] = dict() try: reader = gzip.GzipFile(fname) except: print("Failed to read file %s" % fname) raise for line in reader.readlines(): words = str(line.strip()).split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] elif words[0] in depfnames: vindex = 0 for dep in words[1].split(','): if '|' in dep: cdeps = [re.sub('\s+', '', re.sub('\(.*\)', '', cdep)) for cdep in dep.split('|')] dyn_dname = "choice-%s-%s-%s" % (pkgname, barch, vindex) vindex += 1 bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: raw_dep = re.sub('\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': for pkg in words[1].split(','): dname = pkg.rstrip().strip() if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) self.deps['global'] = dict() self.provides['global'] = dict() for barch in bdeps: self.deps[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() for barch in bdeps: if pkgname not in bdeps[barch]: bdeps[barch][pkgname] = [] pset.add(tuple(bdeps[barch][pkgname])) if len(pset) == 1: self.deps['global'][pkgname] = pset.pop() else: for barch in bdeps: self.deps[barch][pkgname] = bdeps[barch][pkgname] provided = set() for bprovided in list(bprov.values()): provided.update(set(bprovided)) for prov in provided: prset = set() for barch in bprov: if prov not in bprov[barch]: continue prset.add(tuple(bprov[barch].get(prov, ()))) if len(prset) == 1: self.provides['global'][prov] = prset.pop() else: for barch in bprov: self.provides[barch][prov] = bprov[barch].get(prov, ()) self.save_state() def is_package(self, _, pkg): return (pkg in self.pkgnames and pkg not in self.blacklist and (len(self.whitelist) == 0 or pkg in self.whitelist)) class PACSource(Source): basegroups = ['arch', 'parabola'] ptype = 'pacman' def __init__(self, basepath, xsource): Source.__init__(self, basepath, xsource) self.pkgnames = set() self.url_map = [{'rawurl': self.rawurl, 'url': self.url, 'version': self.version, 'components': self.components, 'arches': self.arches}] def save_state(self): cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides), cache, 2) cache.close() def load_state(self): data = BUILTIN_FILE_TYPE(self.cachefile) self.pkgnames, self.deps, self.provides = cPickle.load(data) def filter_unknown(self, unknown): filtered = set([u for u in unknown if u.startswith('choice')]) unknown.difference_update(filtered) def get_urls(self): if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: rv.append("%s%s/os/%s/%s.db.tar.gz" % (self.url, part, arch, part)) return rv else: raise Exception("PACSource : RAWUrl not supported (yet)") urls = property(get_urls) def read_files(self): bdeps = dict() bprov = dict() if self.recommended: depfnames = ['Depends', 'Pre-Depends', 'Recommends'] else: depfnames = ['Depends', 'Pre-Depends'] for fname in self.files: if not self.rawurl: barch = [x for x in fname.split('@') if x in self.arches][0] else: # RawURL entries assume that they only have one # element and that it is the architecture of the source. barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() bprov[barch] = dict() try: print("try to read : " + fname) tar = tarfile.open(fname, "r") reader = gzip.GzipFile(fname) except: print("Failed to read file %s" % fname) raise for tarinfo in tar: if tarinfo.isdir(): self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) print("added : " + tarinfo.name.rsplit("-", 2)[0]) tar.close() self.deps['global'] = dict() self.provides['global'] = dict() for barch in bdeps: self.deps[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() for barch in bdeps: if pkgname not in bdeps[barch]: bdeps[barch][pkgname] = [] pset.add(tuple(bdeps[barch][pkgname])) if len(pset) == 1: self.deps['global'][pkgname] = pset.pop() else: for barch in bdeps: self.deps[barch][pkgname] = bdeps[barch][pkgname] provided = set() for bprovided in list(bprov.values()): provided.update(set(bprovided)) for prov in provided: prset = set() for barch in bprov: if prov not in bprov[barch]: continue prset.add(tuple(bprov[barch].get(prov, ()))) if len(prset) == 1: self.provides['global'][prov] = prset.pop() else: for barch in bprov: self.provides[barch][prov] = bprov[barch].get(prov, ()) self.save_state() def is_package(self, _, pkg): return (pkg in self.pkgnames and pkg not in self.blacklist and (len(self.whitelist) == 0 or pkg in self.whitelist)) class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, Bcfg2.Server.Plugin.StructFile): def __init__(self, filename, cachepath, fam, packages): try: Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) except OSError: err = sys.exc_info()[1] logger.error("Packages: Error processing sources: %s" % err) raise Bcfg2.Server.Plugin.PluginInitError Bcfg2.Server.Plugin.StructFile.__init__(self, filename) self.cachepath = cachepath if not os.path.exists(self.cachepath): # create cache directory if needed os.makedirs(self.cachepath) self.extras = [] self.fam = fam self.pkg_obj = packages def Index(self): try: self.xdata = lxml.etree.XML(self.data, base_url=self.name) except lxml.etree.XMLSyntaxError: err = sys.exc_info()[1] logger.error("Packages: Error processing sources: %s" % err) raise Bcfg2.Server.Plugin.PluginInitError included = [ent.get('href') for ent in self.xdata.findall('./{http://www.w3.org/2001/XInclude}include')] if included: for name in included: if name not in self.extras: self.add_monitor(name) try: self.xdata.getroottree().xinclude() except lxml.etree.XIncludeError: err = sys.exc_info()[1] logger.error("Packages: Error processing sources: %s" % err) if self.__identifier__ is not None: self.label = self.xdata.attrib[self.__identifier__] self.entries = [] for xsource in self.xdata.findall('.//Source'): source = source_from_xml(xsource, self.cachepath) if source is not None: self.entries.append(source) self.pkg_obj.Reload() def add_monitor(self, fname): """Add a fam monitor for an included file""" self.fam.AddMonitor(os.path.join(os.path.dirname(self.name), fname), self) self.extras.append(fname) class PackagesConfig(Bcfg2.Server.Plugin.FileBacked, ConfigParser.SafeConfigParser): def __init__(self, filename, fam): Bcfg2.Server.Plugin.FileBacked.__init__(self, filename) ConfigParser.SafeConfigParser.__init__(self) # packages.conf isn't strictly necessary, so only set a # monitor if it exists. if it gets added, that will require a # server restart if os.path.exists(filename): fam.AddMonitor(filename, self) def Index(self): """ Build local data structures """ for section in self.sections(): self.remove_section(section) self.read(self.name) class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector): name = 'Packages' conflicts = ['Pkgmgr'] experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) self.sentinels = set() self.virt_pkgs = dict() self.ptypes = dict() self.cachepath = os.path.join(self.data, 'cache') self.keypath = os.path.join(self.data, 'keys') if not os.path.exists(self.keypath): # create key directory if needed os.makedirs(self.keypath) # set up config files self.config = PackagesConfig(os.path.join(self.data, "packages.conf"), core.fam) self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self) @property def disableResolver(self): try: return self.config.get("global", "resolver").lower() == "disabled" except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False @property def disableMetaData(self): try: return self.config.get("global", "metadata").lower() == "disabled" except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False def create_apt_conf(self, entry, metadata): """ create apt config for the specified host """ raise NotImplementedError def create_yum_conf(self, entry, metadata): """ create yum config for the specified host """ yum_attrib = {'encoding': 'ascii', 'owner': 'root', 'group': 'root', 'type': 'file', 'perms': '0644'} stanzas = [] reponame_re = re.compile(r'.*/(?:RPMS\.)?([^/]+)') for source in self.get_matching_sources(metadata): for url_map in source.url_map: if url_map['arch'] in metadata.groups: # try to find a sensible name for the repo name = None if source.id: reponame = source.id else: match = reponame_re.search(url_map['url']) if url_map['component']: name = url_map['component'] elif match: name = match.group(1) else: # couldn't figure out the name from the # source ID, URL or URL map (which # probably means its a screwy URL), so we # just generate a random one name = base64.b64encode(os.urandom(16))[:-2] reponame = "%s-%s" % (source.groups[0], name) stanza = ["[%s]" % reponame, "name=%s" % reponame, "baseurl=%s" % url_map['url'], "enabled=1"] if len(source.gpgkeys): stanza.append("gpgcheck=1") stanza.append("gpgkey=%s" % " ".join(source.gpgkeys)) else: stanza.append("gpgcheck=0") stanzas.append("\n".join(stanza)) entry.text = "%s\n" % "\n\n".join(stanzas) for (key, value) in list(yum_attrib.items()): entry.attrib.__setitem__(key, value) def get_relevant_groups(self, meta): mgrps = [] for source in self.get_matching_sources(meta): mgrps.extend(list(set([g for g in meta.groups if (g in source.basegroups or g in source.groups or g in source.arches)]))) mgrps.sort() return tuple(mgrps) def _setup_pulp(self): try: rouser = self.config.get("pulp", "rouser") ropass = self.config.get("pulp", "ropass") except ConfigParser.NoSectionError: logger.error("No [pulp] section found in Packages/packages.conf") raise Bcfg2.Server.Plugin.PluginInitError except ConfigParser.NoOptionError: err = sys.exc_info()[1] logger.error("Required option not found in " "Packages/packages.conf: %s" % err) raise Bcfg2.Server.Plugin.PluginInitError pulpconfig = pulp.client.config.Config() serveropts = pulpconfig.server self._server = pulp.client.server.PulpServer(serveropts['host'], int(serveropts['port']), serveropts['scheme'], serveropts['path']) self._server.set_basic_auth_credentials(rouser, ropass) pulp.client.server.set_active_server(self._server) def build_vpkgs_entry(self, meta): # build single entry for all matching sources vpkgs = dict() for source in self.get_matching_sources(meta): s_vpkgs = source.get_vpkgs(meta) for name, prov_set in list(s_vpkgs.items()): if name not in vpkgs: vpkgs[name] = set(prov_set) else: vpkgs[name].update(prov_set) return vpkgs def get_matching_sources(self, meta): return [s for s in self.sources if s.applies(meta)] def get_ptype(self, metadata): """ return the package type relevant to this client """ if metadata.hostname not in self.ptypes: for source in self.sources: for grp in metadata.groups: if grp in source.basegroups: self.ptypes[metadata.hostname] = source.ptype break try: return self.ptypes[metadata.hostname] except KeyError: return None def HandleEntry(self, entry, metadata): if entry.tag == 'Package': entry.set('version', 'auto') entry.set('type', self.get_ptype(metadata)) elif entry.tag == 'Path': if (self.config.has_option("global", "yum_config") and entry.get("name") == self.config.get("global", "yum_config")): self.create_yum_conf(entry, metadata) elif (self.config.has_option("global", "apt_config") and entry.get("name") == self.config.get("global", "apt_config")): self.create_apt_conf(entry, metadata) def HandlesEntry(self, entry, metadata): if entry.tag == 'Package': for grp in metadata.groups: if grp in self.sentinels: return True elif entry.tag == 'Path': # managed entries for yum/apt configs if ((self.config.has_option("global", "yum_config") and entry.get("name") == self.config.get("global", "yum_config")) or (self.config.has_option("global", "apt_config") and entry.get("name") == self.config.get("global", "apt_config"))): return True return False def complete(self, meta, input_requirements, debug=False): '''Build the transitive closure of all package dependencies Arguments: meta - client metadata instance packages - set of package names debug - print out debug information for the decision making process returns => (set(packages), set(unsatisfied requirements), package type) ''' sources = self.get_matching_sources(meta) # reverse list so that priorities correspond to file order sources.reverse() if len(sources) == 0: self.logger.error("Packages: No matching sources for client %s; " "improper group memberships?" % meta.hostname) return set(), set(), 'failed' ptype = self.get_ptype(meta) if ptype is None: return set(), set(), 'failed' # setup vpkg cache pgrps = self.get_relevant_groups(meta) if pgrps not in self.virt_pkgs: self.virt_pkgs[pgrps] = self.build_vpkgs_entry(meta) vpkg_cache = self.virt_pkgs[pgrps] # unclassified is set of unsatisfied requirements (may be pkg for vpkg) unclassified = set(input_requirements) vpkgs = set() both = set() pkgs = set(input_requirements) packages = set() examined = set() unknown = set() final_pass = False really_done = False # do while unclassified or vpkgs or both or pkgs while unclassified or pkgs or both or final_pass: if really_done: break if len(unclassified) + len(pkgs) + len(both) == 0: # one more pass then exit really_done = True while unclassified: current = unclassified.pop() examined.add(current) is_pkg = False for source in sources: if source.is_package(meta, current): is_pkg = True break is_vpkg = current in vpkg_cache if is_pkg and is_vpkg: both.add(current) elif is_pkg and not is_vpkg: pkgs.add(current) elif is_vpkg and not is_pkg: vpkgs.add(current) elif not is_vpkg and not is_pkg: unknown.add(current) while pkgs: # direct packages; current can be added, and all deps # should be resolved current = pkgs.pop() if debug: self.logger.debug("Packages: handling package requirement " "%s" % current) deps = () for source in sources: if source.is_package(meta, current): try: deps = source.get_deps(meta, current) break except: continue packages.add(current) newdeps = set(deps).difference(examined) if debug and newdeps: self.logger.debug("Packages: Package %s added " "requirements %s" % (current, newdeps)) unclassified.update(newdeps) satisfied_vpkgs = set() for current in vpkgs: # virtual dependencies, satisfied if one of N in the # config, or can be forced if only one provider if len(vpkg_cache[current]) == 1: if debug: self.logger.debug("Packages: requirement %s satisfied " "by %s" % (current, vpkg_cache[current])) unclassified.update(vpkg_cache[current].difference(examined)) satisfied_vpkgs.add(current) elif [item for item in vpkg_cache[current] if item in packages]: if debug: self.logger.debug("Packages: requirement %s satisfied " "by %s" % (current, [item for item in vpkg_cache[current] if item in packages])) satisfied_vpkgs.add(current) vpkgs.difference_update(satisfied_vpkgs) satisfied_both = set() for current in both: # packages that are both have virtual providers as # well as a package with that name. allow use of virt # through explicit specification, then fall back to # forcing current on last pass if [item for item in vpkg_cache[current] if item in packages]: if debug: self.logger.debug("Packages: requirement %s satisfied " "by %s" % (current, [item for item in vpkg_cache[current] if item in packages])) satisfied_both.add(current) elif current in input_requirements or final_pass: pkgs.add(current) satisfied_both.add(current) both.difference_update(satisfied_both) if len(unclassified) + len(pkgs) == 0: final_pass = True else: final_pass = False for source in sources: source.filter_unknown(unknown) return packages, unknown, ptype def validate_structures(self, metadata, structures): '''Ensure client configurations include all needed prerequisites Arguments: metadata - client metadata instance structures - a list of structure-stage entry combinations ''' indep = lxml.etree.Element('Independent') self._build_packages(metadata, indep, structures) self._build_gpgkeys(metadata, indep) self._build_pulp_entries(metadata, indep) structures.append(indep) def _build_pulp_entries(self, metadata, independent): """ build list of Pulp actions that need to be included in the specification by validate_structures() """ if not has_pulp: return # if there are no Pulp sources for this host, we don't need to # worry about registering it build_actions = False for source in self.get_matching_sources(metadata): if isinstance(source, PulpSource): build_actions = True break if not build_actions: self.logger.debug("No Pulp sources apply to %s, skipping Pulp " "registration" % metadata.hostname) return consumerapi = pulp.client.api.consumer.ConsumerAPI() try: consumer = consumerapi.consumer(metadata.hostname) except pulp.client.server.ServerRequestError: try: reguser = self.config.get("pulp", "reguser") regpass = self.config.get("pulp", "regpass") reg_cmd = ("pulp-client -u '%s' -p '%s' consumer create " "--id='%s'" % (reguser, regpass, metadata.hostname)) lxml.etree.SubElement(independent, "BoundAction", name="pulp-register", timing="pre", when="always", status="check", command=reg_cmd) except ConfigParser.NoOptionError: err = sys.exc_info()[1] self.logger.error("Required option not found in " "Packages/packages.conf: %s. Pulp consumers " "will not be registered" % err) return for source in self.get_matching_sources(metadata): # each pulp source can only have one arch, so we don't # have to check the arch in url_map if source.id not in consumer['repoids']: bind_cmd = "pulp-client consumer bind --repoid=%s" % source.id lxml.etree.SubElement(independent, "BoundAction", name="pulp-bind-%s" % source.id, timing="pre", when="always", status="check", command=bind_cmd) def _build_packages(self, metadata, independent, structures): """ build list of packages that need to be included in the specification by validate_structures() """ if self.disableResolver: # Config requests no resolver return initial = set([pkg.get('name') for struct in structures for pkg in struct.findall('Package') + \ struct.findall('BoundPackage')]) packages, unknown, ptype = self.complete(metadata, initial, debug=self.debug_flag) if unknown: self.logger.info("Got unknown entries") self.logger.info(list(unknown)) newpkgs = list(packages.difference(initial)) newpkgs.sort() for pkg in newpkgs: lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, type=ptype, version='auto', origin='Packages') def _build_gpgkeys(self, metadata, independent): """ build list of gpg keys to be added to the specification by validate_structures() """ needkeys = set() for source in self.get_matching_sources(metadata): for key in source.gpgkeys: needkeys.add(key) if len(needkeys): keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey", type=self.get_ptype(metadata), origin='Packages') for key in needkeys: # figure out the path of the key on the client try: keydir = self.config.get("global", "gpg_keypath") except ConfigParser.NoOptionError: keydir = "/etc/pki/rpm-gpg" except ConfigParser.NoSectionError: keydir = "/etc/pki/rpm-gpg" remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() # copy the key to the client keypath = lxml.etree.Element("BoundPath", name=remotekey, encoding='ascii', owner='root', group='root', type='file', perms='0644', important='true') keypath.text = kdata independent.append(keypath) if has_yum: # add the key to the specification to ensure it # gets installed try: kinfo = yum.misc.getgpgkeyinfo(kdata) version = yum.misc.keyIdToRPMVer(kinfo['keyid']) release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) lxml.etree.SubElement(keypkg, 'Instance', version=version, release=release, simplefile=remotekey) except ValueError: err = sys.exc_info()[1] self.logger.error("Could not read GPG key %s: %s" % (localkey, err)) else: self.logger.info("Yum libraries not found; GPG keys will " "not be handled automatically") independent.append(keypkg) def Refresh(self): '''Packages.Refresh() => True|False\nReload configuration specification and download sources\n''' self._load_config(force_update=True) return True def Reload(self): '''Packages.Refresh() => True|False\nReload configuration specification and sources\n''' self._load_config() return True def _load_config(self, force_update=False): ''' Load the configuration data and setup sources Keyword args: force_update Force downloading repo data ''' self._load_sources(force_update) self._load_gpg_keys(force_update) def _load_sources(self, force_update): """ Load sources from the config """ self.virt_pkgs = dict() self.sentinels = set() cachefiles = [] for source in self.sources: cachefiles.append(source.cachefile) if not self.disableMetaData: source.setup_data(force_update) self.sentinels.update(source.basegroups) for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): if cfile not in cachefiles: os.unlink(cfile) def _load_gpg_keys(self, force_update): """ Load gpg keys from the config """ keyfiles = [] for source in self.sources: for key in source.gpgkeys: localfile = os.path.join(self.keypath, os.path.basename(key)) if localfile not in keyfiles: keyfiles.append(localfile) if force_update or not os.path.exists(localfile): logger.debug("Downloading and parsing %s" % key) response = urlopen(key) open(localfile, 'w').write(response.read()) for kfile in glob.glob(os.path.join(self.keypath, "*")): if kfile not in keyfiles: os.unlink(kfile) def get_additional_data(self, meta): sdata = [] [sdata.extend(copy.deepcopy(src.url_map)) for src in self.get_matching_sources(meta)] return dict(sources=sdata)