diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Source.py')
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 122 |
1 files changed, 110 insertions, 12 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 1e1aaade1..b57d1b0cc 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -1,3 +1,49 @@ +""" ``Source`` objects represent a single <Source> tag in +``sources.xml``. Note that a single Source tag can itself describe +multiple repositories (if it uses the "url" attribute instead of +"rawurl"), and so can the ``Source`` object. This can be the source +(har har) of some confusion. See +:func:`Bcfg2.Server.Plugins.Packages.Collection._Collection.sourcelist` +for the proper way to get all repos from a ``Source`` object. + +Source objects are aggregated into +:class:`Bcfg2.Server.Plugins.Packages.Collection._Collection` +objects, which are actually called by +:class:`Bcfg2.Server.Plugins.Packages.Packages`. This way a more +advanced subclass can query repositories in aggregate rather than +individually, which may give faster or more accurate results. + +The base ``Source`` object must be subclassed to handle each +repository type. How you subclass ``Source`` will depend on how you +subclassed +:class:`Bcfg2.Server.Plugins.Packages.Collection._Collection`; see +:mod:`Bcfg2.Server.Plugins.Packages.Collection` for more details on +different methods for doing that. + +If you are using the stock (or a near-stock) +:class:`Bcfg2.Server.Plugins.Packages.Collection._Collection` object, +then you will need to implement the following methods and attributes +in your ``Source`` subclass: + +* :func:`Source.get_urls` +* :func:`Source.read_files` +* :attr:`Source.basegroups` +* :attr:`Source.ptype` + +Additionally, you may want to consider overriding the following +methods and attributes: + +* :func:`Source.is_virtual_package` +* :func:`Source.get_group` +* :attr:`Source.unknown_filter` +* :attr:`Source.load_state` +* :attr:`Source.save_state` + +If you are overriding the ``_Collection`` object in more depth, then +you have more leeway in what you might want to override or implement +in your ``Source`` subclass. +""" + import os import re import sys @@ -8,6 +54,13 @@ from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ def fetch_url(url): + """ Return the content of the given URL. + + :param url: The URL to fetch content from. + :type url: string + :raises: ValueError - Malformed URL + :raises: URLError - Failure fetching URL + :returns: string - the content of the page at the given URL """ if '@' in url: mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) if not mobj: @@ -22,6 +75,7 @@ def fetch_url(url): class SourceInitError(Exception): + """ Raised when a :class:`Source` object fails instantiation. """ pass @@ -30,6 +84,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): pulprepo_re = re.compile(r'pulp/repos/([^/]+)') genericrepo_re = re.compile('https?://.*?/([^/]+)/?$') basegroups = [] + unknown_filter = lambda p: p.startswith("choice") def __init__(self, basepath, xsource, setup): Bcfg2.Server.Plugin.Debuggable.__init__(self) @@ -37,16 +92,18 @@ class Source(Bcfg2.Server.Plugin.Debuggable): self.xsource = xsource self.setup = setup self.essentialpkgs = set() + self.pkgnames = set() try: self.version = xsource.find('Version').text except AttributeError: self.version = None - for key, tag in [('components', 'Component'), ('arches', 'Arch'), - ('blacklist', 'Blacklist'), - ('whitelist', 'Whitelist')]: - setattr(self, key, [item.text for item in xsource.findall(tag)]) + self.components = [item.text for item in xsource.findall('Component')] + self.arches = [item.text for item in xsource.findall('Arch')] + self.blacklist = [item.text for item in xsource.findall('Blacklist')] + self.whitelist = [item.text for item in xsource.findall('Whitelist')] + self.server_options = dict() self.client_options = dict() opts = xsource.findall("Options") @@ -112,7 +169,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): else: # rawurl given usettings = [dict(version=self.version, component=None, arch=arch)] - + for setting in usettings: if not self.rawurl: setting['baseurl'] = self.url @@ -133,7 +190,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable): g in self.arches)]))) def load_state(self): - pass + data = open(self.cachefile) + (self.pkgnames, self.deps, self.provides, + self.essentialpkgs) = cPickle.load(data) + + def save_state(self): + cache = open(self.cachefile, 'wb') + cPickle.dump((self.pkgnames, self.deps, self.provides, + self.essentialpkgs), cache, 2) + cache.close() def setup_data(self, force_update=False): should_read = True @@ -234,14 +299,45 @@ class Source(Bcfg2.Server.Plugin.Debuggable): def escape_url(self, url): return os.path.join(self.basepath, url.replace('/', '@')) - def file_init(self): - pass - def read_files(self): pass + def process_files(self, deps, prov): + self.deps['global'] = dict() + self.provides['global'] = dict() + for barch in deps: + self.deps[barch] = dict() + self.provides[barch] = dict() + for pkgname in self.pkgnames: + pset = set() + for barch in deps: + if pkgname not in deps[barch]: + deps[barch][pkgname] = [] + pset.add(tuple(deps[barch][pkgname])) + if len(pset) == 1: + self.deps['global'][pkgname] = pset.pop() + else: + for barch in deps: + self.deps[barch][pkgname] = deps[barch][pkgname] + provided = set() + for bprovided in list(prov.values()): + provided.update(set(bprovided)) + for prov in provided: + prset = set() + for barch in prov: + if prov not in prov[barch]: + continue + prset.add(tuple(prov[barch].get(prov, ()))) + if len(prset) == 1: + self.provides['global'][prov] = prset.pop() + else: + for barch in prov: + self.provides[barch][prov] = prov[barch].get(prov, ()) + self.save_state() + def filter_unknown(self, unknown): - pass + unknown.difference_update(set([u for u in unknown + if self.unknown_filter(u)])) def update(self): for url in self.urls: @@ -286,8 +382,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): return self.provides[arch][required] return [] - def is_package(self, metadata, _): - return False + def is_package(self, metadata, pkg): + return (pkg in self.pkgnames and + pkg not in self.blacklist and + (len(self.whitelist) == 0 or pkg in self.whitelist)) def get_package(self, metadata, package): return package |