From 35cd270f7d7579e57ef68a848af615f4d0778ec1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 20 May 2013 08:40:25 -0400 Subject: Replaced XMLSrc with StructFile XMLSrc and StructFile were very different implementations of basically the same thing. StructFile has gotten lots and lots of improvements, while XMLSrc, with its wonky, unintuitive interface, has floundered. This replaces XMLSrc with StructFile (nearly) everywhere, and rebases everything that inherited from XMLSrc with StructFile. XMLSrc lives on somewhat in the Pkgmgr plugin's PNode and PkgSrc objects, where originally inherited from the XMLSrc and INode objects but are now the only implementation of that older interface. Pkgmgr was left as-is because a) it's a little-used plugin; b) it's in need of a rewrite anyway; and c) it has deep, deep hooks into the guts of XMLSrc and INode, and rewriting it with StructFile was deeply nontrivial. --- src/lib/Bcfg2/Server/Plugins/Pkgmgr.py | 223 +++++++++++++++++++++++++-------- 1 file changed, 172 insertions(+), 51 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Pkgmgr.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index a1dcb575f..293ec8e1a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -2,23 +2,20 @@ import os import re +import sys import glob import logging import lxml.etree -import Bcfg2.Server.Plugin import Bcfg2.Server.Lint +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError -try: - set -except NameError: - # deprecated since python 2.6 - from sets import Set as set logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr') class FuzzyDict(dict): - fuzzy = re.compile('(?P.*):(?P\S+(,\S+)*)') + fuzzy = re.compile(r'(?P.*):(?P\S+(,\S+)*)') def __getitem__(self, key): if isinstance(key, str): @@ -47,95 +44,217 @@ class FuzzyDict(dict): raise -class PNode(Bcfg2.Server.Plugin.INode): +class PNode(object): """PNode has a list of packages available at a particular group intersection. """ - splitters = {'rpm': re.compile('^(.*/)?(?P[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ - '(?P[\w\d\.]+-([\w\d\.]+))\.(?P\S+)\.rpm$'), - 'encap': re.compile('^(?P[\w-]+)-(?P[\w\d\.+-]+).encap.*$')} + splitters = dict( + rpm=re.compile( + r'^(.*/)?(?P[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + + r'(?P[\w\d\.]+-([\w\d\.]+))\.(?P\S+)\.rpm$'), + encap=re.compile( + r'^(?P[\w-]+)-(?P[\w\d\.+-]+).encap.*$')) + raw = dict( + Client="lambda m, e:'%(name)s' == m.hostname and predicate(m, e)", + Group="lambda m, e:'%(name)s' in m.groups and predicate(m, e)") + nraw = dict( + Client="lambda m, e:'%(name)s' != m.hostname and predicate(m, e)", + Group="lambda m, e:'%(name)s' not in m.groups and predicate(m, e)") + containers = ['Group', 'Client'] ignore = ['Package'] - def Match(self, metadata, data, entry=lxml.etree.Element("None")): - """Return a dictionary of package mappings.""" - if self.predicate(metadata, entry): - for key in self.contents: - try: - data[key].update(self.contents[key]) - except: - data[key] = FuzzyDict() - data[key].update(self.contents[key]) - for child in self.children: - child.Match(metadata, data) - def __init__(self, data, pdict, parent=None): # copy local attributes to all child nodes if no local attribute exists if 'Package' not in pdict: pdict['Package'] = set() for child in data.getchildren(): - attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name']) + attrs = set(data.attrib.keys()).difference( + child.attrib.keys() + ['name']) for attr in attrs: try: child.set(attr, data.get(attr)) except: - # don't fail on things like comments and other immutable elements + # don't fail on things like comments and other + # immutable elements pass - Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent) + self.data = data + self.contents = {} + if parent is None: + self.predicate = lambda m, e: True + else: + predicate = parent.predicate + if data.get('negate', 'false').lower() == 'true': + psrc = self.nraw + else: + psrc = self.raw + if data.tag in list(psrc.keys()): + self.predicate = eval(psrc[data.tag] % + {'name': data.get('name')}, + {'predicate': predicate}) + else: + raise PluginExecutionError("Unknown tag: %s" % data.tag) + self.children = [] + self._load_children(data, pdict) + if 'Package' not in self.contents: self.contents['Package'] = FuzzyDict() for pkg in data.findall('./Package'): - if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']: + if ('name' in pkg.attrib and + pkg.get('name') not in pdict['Package']): pdict['Package'].add(pkg.get('name')) - if pkg.get('name') != None: + if pkg.get('name') is not None: self.contents['Package'][pkg.get('name')] = {} if pkg.getchildren(): self.contents['Package'][pkg.get('name')]['__children__'] \ - = pkg.getchildren() + = pkg.getchildren() if 'simplefile' in pkg.attrib: - pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) + pkg.set('url', + "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) self.contents['Package'][pkg.get('name')].update(pkg.attrib) else: if 'file' in pkg.attrib: if 'multiarch' in pkg.attrib: archs = pkg.get('multiarch').split() srcs = pkg.get('srcs', pkg.get('multiarch')).split() - url = ' '.join(["%s/%s" % (pkg.get('uri'), - pkg.get('file') % {'src':srcs[idx], - 'arch':archs[idx]}) - for idx in range(len(archs))]) + url = ' '.join( + ["%s/%s" % (pkg.get('uri'), + pkg.get('file') % {'src': srcs[idx], + 'arch': archs[idx]}) + for idx in range(len(archs))]) pkg.set('url', url) else: pkg.set('url', '%s/%s' % (pkg.get('uri'), pkg.get('file'))) - if pkg.get('type') in self.splitters and pkg.get('file') != None: - mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) + if (pkg.get('type') in self.splitters and + pkg.get('file') is not None): + mdata = \ + self.splitters[pkg.get('type')].match(pkg.get('file')) if not mdata: - logger.error("Failed to match pkg %s" % pkg.get('file')) + logger.error("Failed to match pkg %s" % + pkg.get('file')) continue pkgname = mdata.group('name') self.contents['Package'][pkgname] = mdata.groupdict() self.contents['Package'][pkgname].update(pkg.attrib) if pkg.attrib.get('file'): - self.contents['Package'][pkgname]['url'] = pkg.get('url') - self.contents['Package'][pkgname]['type'] = pkg.get('type') + self.contents['Package'][pkgname]['url'] = \ + pkg.get('url') + self.contents['Package'][pkgname]['type'] = \ + pkg.get('type') if pkg.get('verify'): - self.contents['Package'][pkgname]['verify'] = pkg.get('verify') + self.contents['Package'][pkgname]['verify'] = \ + pkg.get('verify') if pkg.get('multiarch'): - self.contents['Package'][pkgname]['multiarch'] = pkg.get('multiarch') + self.contents['Package'][pkgname]['multiarch'] = \ + pkg.get('multiarch') if pkgname not in pdict['Package']: pdict['Package'].add(pkgname) if pkg.getchildren(): - self.contents['Package'][pkgname]['__children__'] = pkg.getchildren() + self.contents['Package'][pkgname]['__children__'] = \ + pkg.getchildren() else: - self.contents['Package'][pkg.get('name')].update(pkg.attrib) + self.contents['Package'][pkg.get('name')].update( + pkg.attrib) + + def _load_children(self, data, idict): + """ load children """ + for item in data.getchildren(): + if item.tag in self.ignore: + continue + elif item.tag in self.containers: + self.children.append(self.__class__(item, idict, self)) + else: + try: + self.contents[item.tag][item.get('name')] = \ + dict(item.attrib) + except KeyError: + self.contents[item.tag] = \ + {item.get('name'): dict(item.attrib)} + if item.text: + self.contents[item.tag][item.get('name')]['__text__'] = \ + item.text + if item.getchildren(): + self.contents[item.tag][item.get('name')]['__children__'] \ + = item.getchildren() + try: + idict[item.tag].append(item.get('name')) + except KeyError: + idict[item.tag] = [item.get('name')] + def Match(self, metadata, data, entry=lxml.etree.Element("None")): + """Return a dictionary of package mappings.""" + if self.predicate(metadata, entry): + for key in self.contents: + try: + data[key].update(self.contents[key]) + except: # pylint: disable=W0702 + data[key] = FuzzyDict() + data[key].update(self.contents[key]) + for child in self.children: + child.Match(metadata, data) -class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): - """PkgSrc files contain a PNode hierarchy that - returns matching package entries. - """ + +class PkgSrc(Bcfg2.Server.Plugin.XMLFileBacked): + """ XMLSrc files contain a + :class:`Bcfg2.Server.Plugin.helpers.INode` hierarchy that returns + matching entries. XMLSrc objects are deprecated and + :class:`Bcfg2.Server.Plugin.helpers.StructFile` should be + preferred where possible.""" __node__ = PNode __cacheobj__ = FuzzyDict + __priority_required__ = True + + def __init__(self, filename, should_monitor=False): + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, + should_monitor) + self.items = {} + self.cache = None + self.pnode = None + self.priority = -1 + + def HandleEvent(self, _=None): + """Read file upon update.""" + try: + data = open(self.name).read() + except IOError: + msg = "Failed to read file %s: %s" % (self.name, sys.exc_info()[1]) + logger.error(msg) + raise PluginExecutionError(msg) + self.items = {} + try: + xdata = lxml.etree.XML(data, parser=Bcfg2.Server.XMLParser) + except lxml.etree.XMLSyntaxError: + msg = "Failed to parse file %s: %s" % (self.name, + sys.exc_info()[1]) + logger.error(msg) + raise PluginExecutionError(msg) + self.pnode = self.__node__(xdata, self.items) + self.cache = None + try: + self.priority = int(xdata.get('priority')) + except (ValueError, TypeError): + if self.__priority_required__: + msg = "Got bogus priority %s for file %s" % \ + (xdata.get('priority'), self.name) + logger.error(msg) + raise PluginExecutionError(msg) + + del xdata, data + + def Cache(self, metadata): + """Build a package dict for a given host.""" + if self.cache is None or self.cache[0] != metadata: + cache = (metadata, self.__cacheobj__()) + if self.pnode is None: + logger.error("Cache method called early for %s; " + "forcing data load" % self.name) + self.HandleEvent() + return + self.pnode.Match(metadata, cache[1]) + self.cache = cache + + def __str__(self): + return str(self.items) class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): @@ -165,12 +284,14 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): mdata = FuzzyDict.fuzzy.match(pname) if mdata: arches = mdata.group('alist').split(',') - [entry.remove(inst) for inst in \ - entry.findall('Instance') \ - if inst.get('arch') not in arches] + for inst in entry.findall('Instance'): + if inst.get('arch') not in arches: + entry.remove(inst) def HandlesEntry(self, entry, metadata): - return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys()) + return ( + entry.tag == 'Package' and + entry.get('name').split(':')[0] in self.Entries['Package'].keys()) def HandleEntry(self, entry, metadata): self.BindEntry(entry, metadata) -- cgit v1.2.3-1-g7c22