From c6576d5e5fefdec91b031e1fd7ee0103c6c98cdc Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Thu, 19 Mar 2009 02:07:13 +0000 Subject: Packages: Implement full yum dep resolver and improve error handling git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5127 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Server/Plugins/Packages.py | 295 +++++++++++++++++++++++++++---------- 1 file changed, 217 insertions(+), 78 deletions(-) diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index 1246c35cf..70dfd78c9 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -7,7 +7,10 @@ import Bcfg2.Server.Plugin # pinning # multi apt-source from xml -def apt_source_from_xml(xsource): +class NoData(Exception): + pass + +def source_from_xml(xsource): ret = dict() for key, tag in [('groups', 'Group'), ('components', 'Component'), ('arches', 'Arch')]: @@ -16,19 +19,185 @@ def apt_source_from_xml(xsource): ret['url'] = xsource.find('URL').text return ret -class APTSource(object): +class Source(object): + basegroups = [] def __init__(self, basepath, url, version, arches, components, groups): self.basepath = basepath self.version = version self.components = components self.url = url + self.groups = groups + self.arches = arches + self.deps = dict() + self.provides = dict() + self.files = [self.mk_fname(url) for url in self.urls] + + def mk_fname(self, url): + return "%s/%s" % (self.basepath, url.replace('/', '@')) + + def file_init(self): + pass + + def read_files(self): + pass + + def update(self): + for url in self.urls: + print "updating", url + fname = self.mk_fname(url) + data = urllib.urlopen(url).read() + file(fname, 'w').write(data) + + def applies(self, metadata): + return len([g for g in self.basegroups if g in metadata.groups]) != 0 and \ + len([g for g in metadata.groups if g in self.groups]) \ + == len(self.groups) + + def get_arches(self, metadata): + return ['global'] + [a for a in self.arches if a in metadata.groups] + + def get_deps(self, metadata, pkgname): + for arch in self.get_arches(metadata): + if pkgname in self.deps[arch]: + return self.deps[arch][pkgname] + raise NoData + + def get_provides(self, metadata, required): + for arch in self.get_arches(metadata): + if required in self.provides[arch]: + return self.provides[arch][required] + raise NoData + + def is_package(self, metadata, _): + return False + + def complete(self, metadata, packages, unresolved): + # perhaps cache arch? + #arch = [a for a in self.arches if a in metadata.groups][0] + # return newpkg, unknown + newpkg = set(packages) + unknown = set() + work = set(unresolved) + seen = set() + while work: + item = work.pop() + seen.add(item) + if self.is_package(metadata, item): + newpkg.add(item) + try: + work.update(self.get_deps(metadata, item)) + except NoData: + continue + else: + # provides dep + try: + work.update(self.get_provides(metadata, item)) + except NoData: + unknown.add(item) + work = work.difference(seen) + return (newpkg, unknown) + +class YUMSource(Source): + xp = '{http://linux.duke.edu/metadata/common}' + rp = '{http://linux.duke.edu/metadata/rpm}' + fl = '{http://linux.duke.edu/metadata/filelists}' + basegroups = ['redhat', 'centos'] + ptype = 'yum' + + def __init__(self, basepath, url, version, arches, components, groups): + self.urls = ["%s/%s/%s/%s/repodata/%s.xml.gz" % \ + (url, version, part, arch, basename) for part in components \ + for arch in arches for basename in ['primary', 'filelists']] + Source.__init__(self, basepath, url, version, arches, components, groups) + self.packages = dict() + self.deps = dict([('global', dict())]) + self.provides = dict([('global', dict())]) + self.filemap = dict([(x, dict()) for x in ['global'] + self.arches]) + + def read_files(self): + for fname in self.files: + farch = fname.split('@')[-3] + fdata = lxml.etree.parse(fname).getroot() + if fname.endswith('primary.xml.gz'): + self.parse_primary(fdata, farch) + elif fname.endswith('filelists.xml.gz'): + self.parse_filelist(fdata, farch) + # merge data + sdata = self.packages.values() + self.packages['global'] = copy.deepcopy(sdata.pop()) + while sdata: + self.packages['global'].intersection(sdata.pop()) + + for key in self.packages: + if key == 'global': + continue + self.packages[key] = self.packages['global'].difference(self.packages[key]) + + def parse_filelist(self, data, arch): + for pkg in data.findall(self.fl + 'package'): + for fentry in pkg.findall(self.fl + 'file'): + self.filemap[arch][fentry.text] = pkg.get('name') + + def parse_primary(self, data, arch): + if arch not in self.packages: + self.packages[arch] = set() + if arch not in self.deps: + self.deps[arch] = dict() + if arch not in self.provides: + self.provides[arch] = dict() + for pkg in data.getchildren(): + if not pkg.tag.endswith('package'): + continue + pkgname = pkg.find(self.xp + 'name').text + self.packages[arch].add(pkgname) + + pdata = pkg.find(self.xp + 'format') + if pkgname in self.deps[arch]: + continue + pre = pdata.find(self.rp + 'requires') + self.deps[arch][pkgname] = set() + for entry in pre.getchildren(): + self.deps[arch][pkgname].add(entry.get('name')) + pro = pdata.find(self.rp + 'provides') + for entry in pro.getchildren(): + prov = entry.get('name') + if prov not in self.provides[arch]: + self.provides[arch][prov] = list() + self.provides[arch][prov].append(pkgname) + + def is_package(self, metadata, item): + arch = [a for a in self.arches if a in metadata.groups][0] + return item in self.packages['global'] or item in self.packages[arch] + + def get_provides(self, metadata, required): + ret = set() + arch = [a for a in self.arches if a in metadata.groups][0] + if required in self.provides['global']: + ret.update(Source.get_provides(self, metadata, required)) + elif required in self.provides[arch]: + ret.update(Source.get_provides(self, metadata, required)) + elif required in self.filemap['global']: + ret.update([self.filemap['global'][required]]) + elif required in self.filemap[arch]: + ret.update([self.filemap[arch][required]]) + else: + raise NoData + return ret + + def complete(self, metadata, packages, unknown): + p1, u1 = Source.complete(self, metadata, packages, unknown) + return (p1, set([u for u in u1 if not u.startswith('rpmlib')])) + +class APTSource(Source): + basegroups = ['debian', 'ubuntu', 'nexenta'] + ptype = 'deb' + + def __init__(self, basepath, url, version, arches, components, groups): self.urls = ["%s/dists/%s/%s/binary-%s/Packages.gz" % \ (url, version, part, arch) for part in components \ for arch in arches] - self.files = [self.mk_fname(url) for url in self.urls] - self.groups = groups - self.deps = dict() - self.provides = {} + Source.__init__(self, basepath, url, version, arches, components, groups) + self.pkgnames = set() def get_aptsrc(self): return ["deb %s %s %s" % (self.url, self.version, @@ -36,14 +205,11 @@ class APTSource(object): "deb-src %s %s %s" % (self.url, self.version, " ".join(self.components))] - def mk_fname(self, url): - return "%s/%s" % (self.basepath, url.replace('/', '_')) - def read_files(self): bdeps = dict() bprov = dict() for fname in self.files: - bin = [x for x in fname.split('_') if x.startswith('binary-')][0][7:] + bin = [x for x in fname.split('@') if x.startswith('binary-')][0][7:] if bin not in bdeps: bdeps[bin] = dict() bprov[bin] = dict() @@ -57,6 +223,7 @@ class APTSource(object): words = line.strip().split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() + self.pkgnames.add(pkgname) elif words[0] == 'Depends': bdeps[bin][pkgname] = [] for dep in words[1].split(','): @@ -73,14 +240,12 @@ class APTSource(object): bprov[bin][dname] = set() bprov[bin][dname].add(pkgname) - pkgnames = set() self.deps['global'] = dict() self.provides['global'] = dict() for bin in bdeps: - [pkgnames.add(pname) for pname in bdeps[bin]] self.deps[bin] = dict() self.provides[bin] = dict() - for pkgname in pkgnames: + for pkgname in self.pkgnames: pset = set() for bin in bdeps: if pkgname not in bdeps[bin]: @@ -107,43 +272,16 @@ class APTSource(object): for bin in bprov: self.provides[bin][prov] = bprov[bin].get(prov, ()) + def is_package(self, _, pkg): + return pkg in self.pkgnames - def update(self): - for url in self.urls: - print "updating", url - fname = self.mk_fname(url) - data = urllib.urlopen(url).read() - output = file(fname, 'w').write(data) - - def applies(self, metadata): - return len([g for g in metadata.groups if g in self.groups]) \ - == len(self.groups) - - def find_provides(self, metadata, pkgname): + def get_provides(self, metadata, pkgname): arches = [ar for ar in self.provides if ar in metadata.groups] for arch in ['global'] + arches: if pkgname in self.provides[arch]: # FIXME next round of provides HACK alert - return self.provides[arch][pkgname][0] - return False - - def get_deps(self, metadata, pkgname): - data = False - if pkgname in self.deps['global']: - data = self.deps['global'][pkgname] - for arch in [ar for ar in self.deps if ar in metadata.groups]: - if pkgname in self.deps[arch]: - data = self.deps[arch][pkgname] - if data: - ret = list() - for item in data: - prov = self.find_provides(metadata, item) - if prov: - ret.append(prov) - else: - ret.append(item) - return tuple(ret) - return False + return set([self.provides[arch][pkgname][0]]) + raise NoData class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, @@ -162,7 +300,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, xdata = lxml.etree.parse(self.data + '/config.xml').getroot() self.sources = [] for s in xdata.findall('APTSource'): - self.sources.append(APTSource(cachepath, **apt_source_from_xml(s))) + self.sources.append(APTSource(cachepath, **source_from_xml(s))) + for s in xdata.findall('YUMSource'): + self.sources.append(YUMSource(cachepath, **source_from_xml(s))) for source in self.sources: try: source.read_files() @@ -170,49 +310,48 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.logger.info("File read failed; updating sources", exc_info=1) source.update() source.read_files() - self.pkgmap = dict() - def find_deps(self, metadata, pkgname): - for source in self.sources: - if source.applies(metadata): - deps = source.get_deps(metadata, pkgname) - if deps: - return deps - return () + def get_matching_sources(self, meta): + return [s for s in self.sources if s.applies(meta)] def HandlesEntry(self, entry, metadata): - if [x for x in metadata.groups if x in ['debian', 'ubuntu']] and \ - entry.tag == 'Package': + if [x for x in metadata.groups if x in ['debian', 'ubuntu', 'redhat']] \ + and entry.tag == 'Package': return True return False def HandleEntry(self, entry, metadata): entry.set('version', 'auto') - entry.set('type', 'deb') + for source in self.sources: + if [x for x in metadata.groups if x in source.basegroups]: + entry.set('type', source.ptype) + + def complete(self, meta, packages): + sources = self.get_matching_sources(meta) + ptype = set([s.ptype for s in sources]) + if len(ptype) > 1: + return set(), set(), 'failed' + pkgs = set(packages) + unknown = set(packages) + oldp = set() + oldu = set() + while unknown and (pkgs != oldp or unknown != oldu): + # loop through sources until no progress is made + oldp = pkgs + oldu = unknown + for source in sources: + pkgs, unknown = source.complete(meta, pkgs, unknown) + return pkgs, unknown, ptype.pop() def validate_structures(self, meta, structures): - if [g for g in meta.groups if g in ['debian', 'ubuntu']]: - ptype = 'deb' - else: - return - pkgnames = set() - for struct in structures: - for pkg in struct.findall('Package'): - pkgnames.add(pkg.get('name')) - all = copy.copy(pkgnames) - work = copy.copy(pkgnames) - new = set() - while work: - next = work.pop() - for dep in self.find_deps(meta, next): - if dep in all: - continue - else: - new.add(dep) - all.add(dep) - work.add(dep) + initial = set([pkg.get('name') for struct in structures \ + for pkg in struct.findall('Package')]) news = lxml.etree.Element('Independent') - for pkg in new: + packages, unknown, ptype = self.complete(meta, initial) + if unknown: + self.logger.info("Got unknown entries") + self.logger.info(list(unknown)) + for pkg in packages.difference(initial): lxml.etree.SubElement(news, 'BoundPackage', name=pkg, type=ptype, version='auto') structures.append(news) -- cgit v1.2.3-1-g7c22