diff options
Diffstat (limited to 'src/lib/Server/Plugins/Packages.py')
| -rw-r--r-- | src/lib/Server/Plugins/Packages.py | 1320 |
1 files changed, 0 insertions, 1320 deletions
diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py deleted file mode 100644 index 155b78581..000000000 --- a/src/lib/Server/Plugins/Packages.py +++ /dev/null @@ -1,1320 +0,0 @@ -import os -import re -import sys -import copy -import gzip -import glob -import base64 -import logging -import tarfile -import lxml.etree - -# Compatibility imports -from Bcfg2.Bcfg2Py3k import cPickle -from Bcfg2.Bcfg2Py3k import HTTPBasicAuthHandler -from Bcfg2.Bcfg2Py3k import HTTPPasswordMgrWithDefaultRealm -from Bcfg2.Bcfg2Py3k import HTTPError -from Bcfg2.Bcfg2Py3k import install_opener -from Bcfg2.Bcfg2Py3k import build_opener -from Bcfg2.Bcfg2Py3k import urlopen -from Bcfg2.Bcfg2Py3k import ConfigParser - -# py3k compatibility -if sys.hexversion >= 0x03000000: - from io import FileIO as BUILTIN_FILE_TYPE -else: - BUILTIN_FILE_TYPE = file - -try: - import yum.misc - has_yum = True -except ImportError: - has_yum = False - -try: - import pulp.client.server - import pulp.client.config - import pulp.client.api.repository - import pulp.client.api.consumer - has_pulp = True -except ImportError: - has_pulp = False - -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - -import Bcfg2.Logger -import Bcfg2.Server.Plugin - -# build sources.list? -# caching for yum - -class NoData(Exception): - pass - - -class SomeData(Exception): - pass - -logger = logging.getLogger('Packages') - - -def source_from_xml(xsource, cachepath): - """ create a *Source object from its XML representation in - sources.xml """ - stype = xsource.get("type") - if stype is None: - logger.error("No type specified for source, skipping") - return None - - try: - cls = globals()["%sSource" % stype.upper()] - except KeyError: - logger.error("Unknown source type %s") - return None - - return cls(cachepath, xsource) - - -def _fetch_url(url): - if '@' in url: - mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) - if not mobj: - raise ValueError - user = mobj.group(2) - passwd = mobj.group(3) - url = mobj.group(1) + mobj.group(4) - auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) - auth.add_password(None, url, user, passwd) - install_opener(build_opener(auth)) - return urlopen(url).read() - - -class Source(object): - basegroups = [] - - def __init__(self, basepath, xsource): - self.basepath = basepath - self.xsource = xsource - - try: - self.version = xsource.find('Version').text - except AttributeError: - pass - - for key, tag in [('components', 'Component'), ('arches', 'Arch'), - ('blacklist', 'Blacklist'), - ('whitelist', 'Whitelist')]: - self.__dict__[key] = [item.text for item in xsource.findall(tag)] - - self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] - - self.recommended = xsource.get('recommended', 'false').lower() == 'true' - self.id = xsource.get('id') - - self.rawurl = xsource.get('rawurl', '') - if self.rawurl and not self.rawurl.endswith("/"): - self.rawurl += "/" - self.url = xsource.get('url', '') - if self.url and not self.url.endswith("/"): - self.url += "/" - self.version = xsource.get('version', '') - - # build the set of conditions to see if this source applies to - # a given set of metadata - self.conditions = [] - self.groups = [] # provided for some limited backwards compat - for el in xsource.iterancestors(): - if el.tag == "Group": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") not in m.groups) - else: - self.groups.append(el.get("name")) - self.conditions.append(lambda m, el=el: - el.get("name") in m.groups) - elif el.tag == "Client": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") != m.hostname) - else: - self.conditions.append(lambda m, el=el: - el.get("name") == m.hostname) - - self.deps = dict() - self.provides = dict() - - self.cachefile = \ - os.path.join(self.basepath, - "cache-%s" % - md5(cPickle.dumps([self.version, self.components, - self.url, self.rawurl, - self.arches])).hexdigest()) - self.url_map = [] - - def load_state(self): - pass - - def setup_data(self, force_update=False): - should_read = True - should_download = False - if os.path.exists(self.cachefile): - try: - self.load_state() - should_read = False - except: - logger.error("Cachefile %s load failed; " - "falling back to file read" % self.cachefile) - if should_read: - try: - self.read_files() - except: - logger.error("Packages: File read failed; " - "falling back to file download") - should_download = True - - if should_download or force_update: - try: - self.update() - self.read_files() - except: - logger.error("Failed to update source", exc_info=1) - - def get_urls(self): - return [] - urls = property(get_urls) - - def get_files(self): - return [self.escape_url(url) for url in self.urls] - files = property(get_files) - - def get_vpkgs(self, meta): - agroups = ['global'] + [a for a in self.arches if a in meta.groups] - vdict = dict() - for agrp in agroups: - for key, value in list(self.provides[agrp].items()): - if key not in vdict: - vdict[key] = set(value) - else: - vdict[key].update(value) - return vdict - - def escape_url(self, url): - return os.path.join(self.basepath, url.replace('/', '@')) - - def file_init(self): - pass - - def read_files(self): - pass - - def update(self): - for url in self.urls: - logger.info("Packages: Updating %s" % url) - fname = self.escape_url(url) - try: - data = _fetch_url(url) - except ValueError: - logger.error("Packages: Bad url string %s" % url) - continue - except HTTPError: - err = sys.exc_info()[1] - logger.error("Packages: Failed to fetch url %s. code=%s" % - (url, err.code)) - continue - BUILTIN_FILE_TYPE(fname, 'w').write(data) - - def applies(self, metadata): - # check base groups - if len([g for g in self.basegroups if g in metadata.groups]) == 0: - return False - - # check Group/Client tags from sources.xml - for condition in self.conditions: - if not condition(metadata): - return False - - return True - - def get_arches(self, metadata): - return ['global'] + [a for a in self.arches if a in metadata.groups] - - def get_deps(self, metadata, pkgname): - for arch in self.get_arches(metadata): - if pkgname in self.deps[arch]: - return self.deps[arch][pkgname] - raise NoData - - def get_provides(self, metadata, required): - for arch in self.get_arches(metadata): - if required in self.provides[arch]: - return self.provides[arch][required] - raise NoData - - def is_package(self, metadata, _): - return False - - -class YUMSource(Source): - xp = '{http://linux.duke.edu/metadata/common}' - rp = '{http://linux.duke.edu/metadata/rpm}' - rpo = '{http://linux.duke.edu/metadata/repo}' - fl = '{http://linux.duke.edu/metadata/filelists}' - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - ptype = 'yum' - - def __init__(self, basepath, xsource): - Source.__init__(self, basepath, xsource) - if not self.rawurl: - self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" - else: - self.baseurl = self.rawurl - self.packages = dict() - self.deps = dict([('global', dict())]) - self.provides = dict([('global', dict())]) - self.filemap = dict([(x, dict()) for x in ['global'] + self.arches]) - self.needed_paths = set() - self.file_to_arch = dict() - - def save_state(self): - cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') - cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) - cache.close() - - def load_state(self): - data = BUILTIN_FILE_TYPE(self.cachefile) - (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) - - def get_urls(self): - surls = list() - self.url_map = [] - for arch in self.arches: - if self.url: - usettings = [{'version':self.version, 'component':comp, - 'arch':arch} - for comp in self.components] - else: # rawurl given - usettings = [{'version':self.version, 'component':None, - 'arch':arch}] - - for setting in usettings: - setting['url'] = self.baseurl % setting - self.url_map.append(copy.deepcopy(setting)) - surls.append((arch, [setting['url'] for setting in usettings])) - urls = [] - for (sarch, surl_list) in surls: - for surl in surl_list: - urls.extend(self._get_urls_from_repodata(surl, sarch)) - return urls - urls = property(get_urls) - - def _get_urls_from_repodata(self, url, arch): - rmdurl = '%srepodata/repomd.xml' % url - try: - repomd = _fetch_url(rmdurl) - xdata = lxml.etree.XML(repomd) - except ValueError: - logger.error("Packages: Bad url string %s" % rmdurl) - return [] - except HTTPError: - err = sys.exc_info()[1] - logger.error("Packages: Failed to fetch url %s. code=%s" % - (rmdurl, err.code)) - return [] - except lxml.etree.XMLSyntaxError: - err = sys.exc_info()[1] - logger.error("Packages: Failed to process metadata at %s: %s" % - (rmdurl, err)) - return [] - - urls = [] - for elt in xdata.findall(self.rpo + 'data'): - if elt.get('type') in ['filelists', 'primary']: - floc = elt.find(self.rpo + 'location') - fullurl = url + floc.get('href') - urls.append(fullurl) - self.file_to_arch[self.escape_url(fullurl)] = arch - return urls - - def read_files(self): - # we have to read primary.xml first, and filelists.xml afterwards; - primaries = list() - filelists = list() - for fname in self.files: - if fname.endswith('primary.xml.gz'): - primaries.append(fname) - elif fname.endswith('filelists.xml.gz'): - filelists.append(fname) - - for fname in primaries: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_primary(fdata, farch) - for fname in filelists: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_filelist(fdata, farch) - - # merge data - sdata = list(self.packages.values()) - try: - self.packages['global'] = copy.deepcopy(sdata.pop()) - except IndexError: - logger.error("No packages in repo") - while sdata: - self.packages['global'] = \ - self.packages['global'].intersection(sdata.pop()) - - for key in self.packages: - if key == 'global': - continue - self.packages[key] = \ - self.packages[key].difference(self.packages['global']) - self.save_state() - - def parse_filelist(self, data, arch): - if arch not in self.filemap: - self.filemap[arch] = dict() - for pkg in data.findall(self.fl + 'package'): - for fentry in pkg.findall(self.fl + 'file'): - if fentry.text in self.needed_paths: - if fentry.text in self.filemap[arch]: - self.filemap[arch][fentry.text].add(pkg.get('name')) - else: - self.filemap[arch][fentry.text] = set([pkg.get('name')]) - - def parse_primary(self, data, arch): - if arch not in self.packages: - self.packages[arch] = set() - if arch not in self.deps: - self.deps[arch] = dict() - if arch not in self.provides: - self.provides[arch] = dict() - for pkg in data.getchildren(): - if not pkg.tag.endswith('package'): - continue - pkgname = pkg.find(self.xp + 'name').text - self.packages[arch].add(pkgname) - - pdata = pkg.find(self.xp + 'format') - pre = pdata.find(self.rp + 'requires') - self.deps[arch][pkgname] = set() - for entry in pre.getchildren(): - self.deps[arch][pkgname].add(entry.get('name')) - if entry.get('name').startswith('/'): - self.needed_paths.add(entry.get('name')) - pro = pdata.find(self.rp + 'provides') - if pro != None: - for entry in pro.getchildren(): - prov = entry.get('name') - if prov not in self.provides[arch]: - self.provides[arch][prov] = list() - self.provides[arch][prov].append(pkgname) - - def is_package(self, metadata, item): - arch = [a for a in self.arches if a in metadata.groups] - if not arch: - return False - return ((item in self.packages['global'] or - item in self.packages[arch[0]]) and - item not in self.blacklist and - (len(self.whitelist) == 0 or item in self.whitelist)) - - def get_vpkgs(self, metadata): - rv = Source.get_vpkgs(self, metadata) - for arch, fmdata in list(self.filemap.items()): - if arch not in metadata.groups and arch != 'global': - continue - for filename, pkgs in list(fmdata.items()): - rv[filename] = pkgs - return rv - - def filter_unknown(self, unknown): - filtered = set([u for u in unknown if u.startswith('rpmlib')]) - unknown.difference_update(filtered) - - -class PulpSource(Source): - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - ptype = 'yum' - - def __init__(self, basepath, xsource): - Source.__init__(self, basepath, xsource) - if not has_pulp: - logger.error("Cannot create pulp source: pulp libraries not found") - raise Bcfg2.Server.Plugin.PluginInitError - - self._config = pulp.client.config.Config() - - self._repoapi = pulp.client.api.repository.RepositoryAPI() - self._repo = self._repoapi.repository(self.id) - if self._repo is None: - logger.error("Repo id %s not found") - else: - self.baseurl = "%s/%s" % (self._config.cds.baseurl, - self._repo['relative_path']) - - self.gpgkeys = ["%s/%s" % (self._config.cds.keyurl, key) - for key in self._repoapi.listkeys(self.id)] - - self.url_map = [{'version': self.version, 'component': None, - 'arch': self.arches[0], 'url': self.baseurl}] - - def save_state(self): - cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') - cPickle.dump((self.packages, self.deps, self.provides, self._config, - self.filemap, self.url_map, self._repoapi, self._repo), - cache, 2) - cache.close() - - def load_state(self): - cache = BUILTIN_FILE_TYPE(self.cachefile) - (self.packages, self.deps, self.provides, self._config, self.filemap, - self.url_map, self._repoapi, self._repo) = cPickle.load(cache) - cache.close() - - def read_files(self): - """ ignore the yum files; we can get this information directly - from pulp """ - for pkg in self._repoapi.packages(self.id): - try: - self.packages[pkg['arch']].append(pkg['name']) - except KeyError: - self.packages[pkg['arch']] = [pkg['name']] - self.save_state() - - -class APTSource(Source): - basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] - ptype = 'deb' - - def __init__(self, basepath, xsource): - Source.__init__(self, basepath, xsource) - self.pkgnames = set() - - self.url_map = [{'rawurl': self.rawurl, 'url': self.url, - 'version': self.version, - 'components': self.components, 'arches': self.arches}] - - def save_state(self): - cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') - cPickle.dump((self.pkgnames, self.deps, self.provides), - cache, 2) - cache.close() - - def load_state(self): - data = BUILTIN_FILE_TYPE(self.cachefile) - self.pkgnames, self.deps, self.provides = cPickle.load(data) - - def filter_unknown(self, unknown): - filtered = set([u for u in unknown if u.startswith('choice')]) - unknown.difference_update(filtered) - - def get_urls(self): - if not self.rawurl: - rv = [] - for part in self.components: - for arch in self.arches: - rv.append("%sdists/%s/%s/binary-%s/Packages.gz" % - (self.url, self.version, part, arch)) - return rv - else: - return ["%sPackages.gz" % self.rawurl] - urls = property(get_urls) - - def read_files(self): - bdeps = dict() - bprov = dict() - if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] - for fname in self.files: - if not self.rawurl: - barch = [x - for x in fname.split('@') - if x.startswith('binary-')][0][7:] - else: - # RawURL entries assume that they only have one <Arch></Arch> - # element and that it is the architecture of the source. - barch = self.arches[0] - if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() - try: - reader = gzip.GzipFile(fname) - except: - print("Failed to read file %s" % fname) - raise - for line in reader.readlines(): - words = str(line.strip()).split(':', 1) - if words[0] == 'Package': - pkgname = words[1].strip().rstrip() - self.pkgnames.add(pkgname) - bdeps[barch][pkgname] = [] - elif words[0] in depfnames: - vindex = 0 - for dep in words[1].split(','): - if '|' in dep: - cdeps = [re.sub('\s+', '', - re.sub('\(.*\)', '', cdep)) - for cdep in dep.split('|')] - dyn_dname = "choice-%s-%s-%s" % (pkgname, - barch, - vindex) - vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) - bprov[barch][dyn_dname] = set(cdeps) - else: - raw_dep = re.sub('\(.*\)', '', dep) - raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) - elif words[0] == 'Provides': - for pkg in words[1].split(','): - dname = pkg.rstrip().strip() - if dname not in bprov[barch]: - bprov[barch][dname] = set() - bprov[barch][dname].add(pkgname) - - self.deps['global'] = dict() - self.provides['global'] = dict() - for barch in bdeps: - self.deps[barch] = dict() - self.provides[barch] = dict() - for pkgname in self.pkgnames: - pset = set() - for barch in bdeps: - if pkgname not in bdeps[barch]: - bdeps[barch][pkgname] = [] - pset.add(tuple(bdeps[barch][pkgname])) - if len(pset) == 1: - self.deps['global'][pkgname] = pset.pop() - else: - for barch in bdeps: - self.deps[barch][pkgname] = bdeps[barch][pkgname] - provided = set() - for bprovided in list(bprov.values()): - provided.update(set(bprovided)) - for prov in provided: - prset = set() - for barch in bprov: - if prov not in bprov[barch]: - continue - prset.add(tuple(bprov[barch].get(prov, ()))) - if len(prset) == 1: - self.provides['global'][prov] = prset.pop() - else: - for barch in bprov: - self.provides[barch][prov] = bprov[barch].get(prov, ()) - self.save_state() - - def is_package(self, _, pkg): - return (pkg in self.pkgnames and - pkg not in self.blacklist and - (len(self.whitelist) == 0 or pkg in self.whitelist)) - - -class PACSource(Source): - basegroups = ['arch', 'parabola'] - ptype = 'pacman' - - def __init__(self, basepath, xsource): - Source.__init__(self, basepath, xsource) - self.pkgnames = set() - - self.url_map = [{'rawurl': self.rawurl, 'url': self.url, - 'version': self.version, - 'components': self.components, 'arches': self.arches}] - - def save_state(self): - cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') - cPickle.dump((self.pkgnames, self.deps, self.provides), - cache, 2) - cache.close() - - def load_state(self): - data = BUILTIN_FILE_TYPE(self.cachefile) - self.pkgnames, self.deps, self.provides = cPickle.load(data) - - def filter_unknown(self, unknown): - filtered = set([u for u in unknown if u.startswith('choice')]) - unknown.difference_update(filtered) - - def get_urls(self): - if not self.rawurl: - rv = [] - for part in self.components: - for arch in self.arches: - rv.append("%s%s/os/%s/%s.db.tar.gz" % - (self.url, part, arch, part)) - return rv - else: - raise Exception("PACSource : RAWUrl not supported (yet)") - urls = property(get_urls) - - def read_files(self): - bdeps = dict() - bprov = dict() - - if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] - - for fname in self.files: - if not self.rawurl: - barch = [x for x in fname.split('@') if x in self.arches][0] - else: - # RawURL entries assume that they only have one <Arch></Arch> - # element and that it is the architecture of the source. - barch = self.arches[0] - - if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() - try: - print("try to read : " + fname) - tar = tarfile.open(fname, "r") - reader = gzip.GzipFile(fname) - except: - print("Failed to read file %s" % fname) - raise - - for tarinfo in tar: - if tarinfo.isdir(): - self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - print("added : " + tarinfo.name.rsplit("-", 2)[0]) - tar.close() - - self.deps['global'] = dict() - self.provides['global'] = dict() - for barch in bdeps: - self.deps[barch] = dict() - self.provides[barch] = dict() - for pkgname in self.pkgnames: - pset = set() - for barch in bdeps: - if pkgname not in bdeps[barch]: - bdeps[barch][pkgname] = [] - pset.add(tuple(bdeps[barch][pkgname])) - if len(pset) == 1: - self.deps['global'][pkgname] = pset.pop() - else: - for barch in bdeps: - self.deps[barch][pkgname] = bdeps[barch][pkgname] - provided = set() - for bprovided in list(bprov.values()): - provided.update(set(bprovided)) - for prov in provided: - prset = set() - for barch in bprov: - if prov not in bprov[barch]: - continue - prset.add(tuple(bprov[barch].get(prov, ()))) - if len(prset) == 1: - self.provides['global'][prov] = prset.pop() - else: - for barch in bprov: - self.provides[barch][prov] = bprov[barch].get(prov, ()) - self.save_state() - - def is_package(self, _, pkg): - return (pkg in self.pkgnames and - pkg not in self.blacklist and - (len(self.whitelist) == 0 or pkg in self.whitelist)) - - -class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile): - def __init__(self, filename, cachepath, fam, packages): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - self.cachepath = cachepath - if not os.path.exists(self.cachepath): - # create cache directory if needed - os.makedirs(self.cachepath) - self.extras = [] - self.fam = fam - self.pkg_obj = packages - - def Index(self): - try: - self.xdata = lxml.etree.XML(self.data, base_url=self.name) - except lxml.etree.XMLSyntaxError: - err = sys.exc_info()[1] - logger.error("Packages: Error processing sources: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError - - included = [ent.get('href') - for ent in self.xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.add_monitor(name) - try: - self.xdata.getroottree().xinclude() - except lxml.etree.XIncludeError: - err = sys.exc_info()[1] - logger.error("Packages: Error processing sources: %s" % err) - - if self.__identifier__ is not None: - self.label = self.xdata.attrib[self.__identifier__] - - self.entries = [] - for xsource in self.xdata.findall('.//Source'): - source = source_from_xml(xsource, self.cachepath) - if source is not None: - self.entries.append(source) - - self.pkg_obj.Reload() - - def add_monitor(self, fname): - """Add a fam monitor for an included file""" - self.fam.AddMonitor(os.path.join(os.path.dirname(self.name), fname), - self) - self.extras.append(fname) - - -class PackagesConfig(Bcfg2.Server.Plugin.FileBacked, - ConfigParser.SafeConfigParser): - def __init__(self, filename, fam): - Bcfg2.Server.Plugin.FileBacked.__init__(self, filename) - ConfigParser.SafeConfigParser.__init__(self) - # packages.conf isn't strictly necessary, so only set a - # monitor if it exists. if it gets added, that will require a - # server restart - if os.path.exists(filename): - fam.AddMonitor(filename, self) - - def Index(self): - """ Build local data structures """ - for section in self.sections(): - self.remove_section(section) - self.read(self.name) - - -class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.StructureValidator, - Bcfg2.Server.Plugin.Generator, - Bcfg2.Server.Plugin.Connector): - name = 'Packages' - conflicts = ['Pkgmgr'] - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.StructureValidator.__init__(self) - Bcfg2.Server.Plugin.Generator.__init__(self) - Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.Probing.__init__(self) - - self.sentinels = set() - self.virt_pkgs = dict() - self.ptypes = dict() - self.cachepath = os.path.join(self.data, 'cache') - self.keypath = os.path.join(self.data, 'keys') - if not os.path.exists(self.keypath): - # create key directory if needed - os.makedirs(self.keypath) - - # set up config files - self.config = PackagesConfig(os.path.join(self.data, "packages.conf"), - core.fam) - self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), - self.cachepath, core.fam, self) - - @property - def disableResolver(self): - try: - return self.config.get("global", "resolver").lower() == "disabled" - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - - @property - def disableMetaData(self): - try: - return self.config.get("global", "metadata").lower() == "disabled" - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - - def create_apt_conf(self, entry, metadata): - """ create apt config for the specified host """ - raise NotImplementedError - - def create_yum_conf(self, entry, metadata): - """ create yum config for the specified host """ - yum_attrib = {'encoding': 'ascii', - 'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0644'} - - stanzas = [] - reponame_re = re.compile(r'.*/(?:RPMS\.)?([^/]+)') - for source in self.get_matching_sources(metadata): - for url_map in source.url_map: - if url_map['arch'] in metadata.groups: - # try to find a sensible name for the repo - name = None - if source.id: - reponame = source.id - else: - match = reponame_re.search(url_map['url']) - if url_map['component']: - name = url_map['component'] - elif match: - name = match.group(1) - else: - # couldn't figure out the name from the - # source ID, URL or URL map (which - # probably means its a screwy URL), so we - # just generate a random one - name = base64.b64encode(os.urandom(16))[:-2] - reponame = "%s-%s" % (source.groups[0], name) - - stanza = ["[%s]" % reponame, - "name=%s" % reponame, - "baseurl=%s" % url_map['url'], - "enabled=1"] - if len(source.gpgkeys): - stanza.append("gpgcheck=1") - stanza.append("gpgkey=%s" % - " ".join(source.gpgkeys)) - else: - stanza.append("gpgcheck=0") - stanzas.append("\n".join(stanza)) - - entry.text = "%s\n" % "\n\n".join(stanzas) - for (key, value) in list(yum_attrib.items()): - entry.attrib.__setitem__(key, value) - - def get_relevant_groups(self, meta): - mgrps = [] - for source in self.get_matching_sources(meta): - mgrps.extend(list(set([g for g in meta.groups - if (g in source.basegroups or - g in source.groups or - g in source.arches)]))) - mgrps.sort() - return tuple(mgrps) - - def _setup_pulp(self): - try: - rouser = self.config.get("pulp", "rouser") - ropass = self.config.get("pulp", "ropass") - except ConfigParser.NoSectionError: - logger.error("No [pulp] section found in Packages/packages.conf") - raise Bcfg2.Server.Plugin.PluginInitError - except ConfigParser.NoOptionError: - err = sys.exc_info()[1] - logger.error("Required option not found in " - "Packages/packages.conf: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError - - pulpconfig = pulp.client.config.Config() - serveropts = pulpconfig.server - - self._server = pulp.client.server.PulpServer(serveropts['host'], - int(serveropts['port']), - serveropts['scheme'], - serveropts['path']) - self._server.set_basic_auth_credentials(rouser, ropass) - pulp.client.server.set_active_server(self._server) - - def build_vpkgs_entry(self, meta): - # build single entry for all matching sources - vpkgs = dict() - for source in self.get_matching_sources(meta): - s_vpkgs = source.get_vpkgs(meta) - for name, prov_set in list(s_vpkgs.items()): - if name not in vpkgs: - vpkgs[name] = set(prov_set) - else: - vpkgs[name].update(prov_set) - return vpkgs - - def get_matching_sources(self, meta): - return [s for s in self.sources if s.applies(meta)] - - def get_ptype(self, metadata): - """ return the package type relevant to this client """ - if metadata.hostname not in self.ptypes: - for source in self.sources: - for grp in metadata.groups: - if grp in source.basegroups: - self.ptypes[metadata.hostname] = source.ptype - break - try: - return self.ptypes[metadata.hostname] - except KeyError: - return None - - def HandleEntry(self, entry, metadata): - if entry.tag == 'Package': - entry.set('version', 'auto') - entry.set('type', self.get_ptype(metadata)) - elif entry.tag == 'Path': - if (self.config.has_option("global", "yum_config") and - entry.get("name") == self.config.get("global", "yum_config")): - self.create_yum_conf(entry, metadata) - elif (self.config.has_option("global", "apt_config") and - entry.get("name") == self.config.get("global", "apt_config")): - self.create_apt_conf(entry, metadata) - - def HandlesEntry(self, entry, metadata): - if entry.tag == 'Package': - for grp in metadata.groups: - if grp in self.sentinels: - return True - elif entry.tag == 'Path': - # managed entries for yum/apt configs - if ((self.config.has_option("global", "yum_config") and - entry.get("name") == self.config.get("global", - "yum_config")) or - (self.config.has_option("global", "apt_config") and - entry.get("name") == self.config.get("global", "apt_config"))): - return True - return False - - def complete(self, meta, input_requirements, debug=False): - '''Build the transitive closure of all package dependencies - - Arguments: - meta - client metadata instance - packages - set of package names - debug - print out debug information for the decision making process - returns => (set(packages), set(unsatisfied requirements), package type) - ''' - sources = self.get_matching_sources(meta) - # reverse list so that priorities correspond to file order - sources.reverse() - if len(sources) == 0: - self.logger.error("Packages: No matching sources for client %s; " - "improper group memberships?" % meta.hostname) - return set(), set(), 'failed' - ptype = self.get_ptype(meta) - if ptype is None: - return set(), set(), 'failed' - - # setup vpkg cache - pgrps = self.get_relevant_groups(meta) - if pgrps not in self.virt_pkgs: - self.virt_pkgs[pgrps] = self.build_vpkgs_entry(meta) - vpkg_cache = self.virt_pkgs[pgrps] - - # unclassified is set of unsatisfied requirements (may be pkg for vpkg) - unclassified = set(input_requirements) - vpkgs = set() - both = set() - pkgs = set(input_requirements) - - packages = set() - examined = set() - unknown = set() - - final_pass = False - really_done = False - # do while unclassified or vpkgs or both or pkgs - while unclassified or pkgs or both or final_pass: - if really_done: - break - if len(unclassified) + len(pkgs) + len(both) == 0: - # one more pass then exit - really_done = True - - while unclassified: - current = unclassified.pop() - examined.add(current) - is_pkg = False - for source in sources: - if source.is_package(meta, current): - is_pkg = True - break - - is_vpkg = current in vpkg_cache - - if is_pkg and is_vpkg: - both.add(current) - elif is_pkg and not is_vpkg: - pkgs.add(current) - elif is_vpkg and not is_pkg: - vpkgs.add(current) - elif not is_vpkg and not is_pkg: - unknown.add(current) - - while pkgs: - # direct packages; current can be added, and all deps - # should be resolved - current = pkgs.pop() - if debug: - self.logger.debug("Packages: handling package requirement " - "%s" % current) - deps = () - for source in sources: - if source.is_package(meta, current): - try: - deps = source.get_deps(meta, current) - break - except: - continue - packages.add(current) - newdeps = set(deps).difference(examined) - if debug and newdeps: - self.logger.debug("Packages: Package %s added " - "requirements %s" % (current, newdeps)) - unclassified.update(newdeps) - - satisfied_vpkgs = set() - for current in vpkgs: - # virtual dependencies, satisfied if one of N in the - # config, or can be forced if only one provider - if len(vpkg_cache[current]) == 1: - if debug: - self.logger.debug("Packages: requirement %s satisfied " - "by %s" % (current, - vpkg_cache[current])) - unclassified.update(vpkg_cache[current].difference(examined)) - satisfied_vpkgs.add(current) - elif [item for item in vpkg_cache[current] if item in packages]: - if debug: - self.logger.debug("Packages: requirement %s satisfied " - "by %s" % - (current, - [item for item in vpkg_cache[current] - if item in packages])) - satisfied_vpkgs.add(current) - vpkgs.difference_update(satisfied_vpkgs) - - satisfied_both = set() - for current in both: - # packages that are both have virtual providers as - # well as a package with that name. allow use of virt - # through explicit specification, then fall back to - # forcing current on last pass - if [item for item in vpkg_cache[current] if item in packages]: - if debug: - self.logger.debug("Packages: requirement %s satisfied " - "by %s" % - (current, - [item for item in vpkg_cache[current] - if item in packages])) - satisfied_both.add(current) - elif current in input_requirements or final_pass: - pkgs.add(current) - satisfied_both.add(current) - both.difference_update(satisfied_both) - - if len(unclassified) + len(pkgs) == 0: - final_pass = True - else: - final_pass = False - - for source in sources: - source.filter_unknown(unknown) - - return packages, unknown, ptype - - def validate_structures(self, metadata, structures): - '''Ensure client configurations include all needed prerequisites - - Arguments: - metadata - client metadata instance - structures - a list of structure-stage entry combinations - ''' - indep = lxml.etree.Element('Independent') - self._build_packages(metadata, indep, structures) - self._build_gpgkeys(metadata, indep) - self._build_pulp_entries(metadata, indep) - structures.append(indep) - - def _build_pulp_entries(self, metadata, independent): - """ build list of Pulp actions that need to be included in the - specification by validate_structures() """ - if not has_pulp: - return - - # if there are no Pulp sources for this host, we don't need to - # worry about registering it - build_actions = False - for source in self.get_matching_sources(metadata): - if isinstance(source, PulpSource): - build_actions = True - break - - if not build_actions: - self.logger.debug("No Pulp sources apply to %s, skipping Pulp " - "registration" % metadata.hostname) - return - - consumerapi = pulp.client.api.consumer.ConsumerAPI() - try: - consumer = consumerapi.consumer(metadata.hostname) - except pulp.client.server.ServerRequestError: - try: - reguser = self.config.get("pulp", "reguser") - regpass = self.config.get("pulp", "regpass") - reg_cmd = ("pulp-client -u '%s' -p '%s' consumer create " - "--id='%s'" % (reguser, regpass, metadata.hostname)) - lxml.etree.SubElement(independent, "BoundAction", - name="pulp-register", timing="pre", - when="always", status="check", - command=reg_cmd) - except ConfigParser.NoOptionError: - err = sys.exc_info()[1] - self.logger.error("Required option not found in " - "Packages/packages.conf: %s. Pulp consumers " - "will not be registered" % err) - return - - for source in self.get_matching_sources(metadata): - # each pulp source can only have one arch, so we don't - # have to check the arch in url_map - if source.id not in consumer['repoids']: - bind_cmd = "pulp-client consumer bind --repoid=%s" % source.id - lxml.etree.SubElement(independent, "BoundAction", - name="pulp-bind-%s" % source.id, - timing="pre", when="always", - status="check", command=bind_cmd) - - def _build_packages(self, metadata, independent, structures): - """ build list of packages that need to be included in the - specification by validate_structures() """ - if self.disableResolver: - # Config requests no resolver - return - - initial = set([pkg.get('name') - for struct in structures - for pkg in struct.findall('Package') + \ - struct.findall('BoundPackage')]) - packages, unknown, ptype = self.complete(metadata, initial, - debug=self.debug_flag) - if unknown: - self.logger.info("Got unknown entries") - self.logger.info(list(unknown)) - newpkgs = list(packages.difference(initial)) - newpkgs.sort() - for pkg in newpkgs: - lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, - type=ptype, version='auto', origin='Packages') - - def _build_gpgkeys(self, metadata, independent): - """ build list of gpg keys to be added to the specification by - validate_structures() """ - needkeys = set() - for source in self.get_matching_sources(metadata): - for key in source.gpgkeys: - needkeys.add(key) - - if len(needkeys): - keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey", - type=self.get_ptype(metadata), - origin='Packages') - - for key in needkeys: - # figure out the path of the key on the client - try: - keydir = self.config.get("global", "gpg_keypath") - except ConfigParser.NoOptionError: - keydir = "/etc/pki/rpm-gpg" - except ConfigParser.NoSectionError: - keydir = "/etc/pki/rpm-gpg" - remotekey = os.path.join(keydir, os.path.basename(key)) - localkey = os.path.join(self.keypath, os.path.basename(key)) - kdata = open(localkey).read() - - # copy the key to the client - keypath = lxml.etree.Element("BoundPath", name=remotekey, - encoding='ascii', - owner='root', group='root', - type='file', perms='0644', - important='true') - keypath.text = kdata - independent.append(keypath) - - if has_yum: - # add the key to the specification to ensure it - # gets installed - try: - kinfo = yum.misc.getgpgkeyinfo(kdata) - version = yum.misc.keyIdToRPMVer(kinfo['keyid']) - release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) - - lxml.etree.SubElement(keypkg, 'Instance', - version=version, - release=release, - simplefile=remotekey) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Could not read GPG key %s: %s" % - (localkey, err)) - else: - self.logger.info("Yum libraries not found; GPG keys will " - "not be handled automatically") - independent.append(keypkg) - - def Refresh(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and download sources\n''' - self._load_config(force_update=True) - return True - - def Reload(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and sources\n''' - self._load_config() - return True - - def _load_config(self, force_update=False): - ''' - Load the configuration data and setup sources - - Keyword args: - force_update Force downloading repo data - ''' - self._load_sources(force_update) - self._load_gpg_keys(force_update) - - def _load_sources(self, force_update): - """ Load sources from the config """ - self.virt_pkgs = dict() - self.sentinels = set() - - cachefiles = [] - for source in self.sources: - cachefiles.append(source.cachefile) - if not self.disableMetaData: - source.setup_data(force_update) - self.sentinels.update(source.basegroups) - - for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): - if cfile not in cachefiles: - os.unlink(cfile) - - def _load_gpg_keys(self, force_update): - """ Load gpg keys from the config """ - keyfiles = [] - for source in self.sources: - for key in source.gpgkeys: - localfile = os.path.join(self.keypath, os.path.basename(key)) - if localfile not in keyfiles: - keyfiles.append(localfile) - if force_update or not os.path.exists(localfile): - logger.debug("Downloading and parsing %s" % key) - response = urlopen(key) - open(localfile, 'w').write(response.read()) - - for kfile in glob.glob(os.path.join(self.keypath, "*")): - if kfile not in keyfiles: - os.unlink(kfile) - - def get_additional_data(self, meta): - sdata = [] - [sdata.extend(copy.deepcopy(src.url_map)) - for src in self.get_matching_sources(meta)] - return dict(sources=sdata) |