summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py58
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Collection.py9
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Pac.py59
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py122
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Yum.py22
5 files changed, 130 insertions, 140 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index a85750651..1c48fb40b 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -2,7 +2,6 @@ import re
import gzip
from Bcfg2.Server.Plugins.Packages.Collection import _Collection
from Bcfg2.Server.Plugins.Packages.Source import Source
-from Bcfg2.Compat import cPickle
class AptCollection(_Collection):
@@ -25,25 +24,6 @@ class AptSource(Source):
basegroups = ['apt', 'debian', 'ubuntu', 'nexenta']
ptype = 'deb'
- def __init__(self, basepath, xsource, config):
- Source.__init__(self, basepath, xsource, config)
- self.pkgnames = set()
-
- def save_state(self):
- cache = open(self.cachefile, 'wb')
- cPickle.dump((self.pkgnames, self.deps, self.provides,
- self.essentialpkgs), cache, 2)
- cache.close()
-
- def load_state(self):
- data = open(self.cachefile)
- (self.pkgnames, self.deps, self.provides,
- self.essentialpkgs) = cPickle.load(data)
-
- def filter_unknown(self, unknown):
- filtered = set([u for u in unknown if u.startswith('choice')])
- unknown.difference_update(filtered)
-
def get_urls(self):
if not self.rawurl:
rv = []
@@ -110,40 +90,4 @@ class AptSource(Source):
if dname not in bprov[barch]:
bprov[barch][dname] = set()
bprov[barch][dname].add(pkgname)
-
- self.deps['global'] = dict()
- self.provides['global'] = dict()
- for barch in bdeps:
- self.deps[barch] = dict()
- self.provides[barch] = dict()
- for pkgname in self.pkgnames:
- pset = set()
- for barch in bdeps:
- if pkgname not in bdeps[barch]:
- bdeps[barch][pkgname] = []
- pset.add(tuple(bdeps[barch][pkgname]))
- if len(pset) == 1:
- self.deps['global'][pkgname] = pset.pop()
- else:
- for barch in bdeps:
- self.deps[barch][pkgname] = bdeps[barch][pkgname]
- provided = set()
- for bprovided in list(bprov.values()):
- provided.update(set(bprovided))
- for prov in provided:
- prset = set()
- for barch in bprov:
- if prov not in bprov[barch]:
- continue
- prset.add(tuple(bprov[barch].get(prov, ())))
- if len(prset) == 1:
- self.provides['global'][prov] = prset.pop()
- else:
- for barch in bprov:
- self.provides[barch][prov] = bprov[barch].get(prov, ())
- self.save_state()
-
- def is_package(self, _, pkg):
- return (pkg in self.pkgnames and
- pkg not in self.blacklist and
- (len(self.whitelist) == 0 or pkg in self.whitelist))
+ self.process_files(bdeps, bprov)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index 31c832893..033eb2fc8 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -1,4 +1,4 @@
-"""``_Collection`` objects represent the set of
+""" ``_Collection`` objects represent the set of
:class:`Bcfg2.Server.Plugins.Packages.Source.Source` objects that apply
to a given client, and can be used to query all software repositories
for a client in aggregate. In some cases this can give faster or more
@@ -35,7 +35,7 @@ In either case, you may want to override
:func:`_Collection.filter_unknown`, and
:func:`_Collection.build_extra_structures`.
-.. _pkg-objects::
+.. _pkg-objects:
Conversion Between Package Objects and XML Entries
--------------------------------------------------
@@ -59,6 +59,9 @@ functions that take a package as an argument (e.g.,
In the documentation below, the actual parameter return type (usually
.``string``) used in this base implementation is noted, as well as this
fact.
+
+The Collection Module
+---------------------
"""
import sys
@@ -447,7 +450,7 @@ class _Collection(list, Bcfg2.Server.Plugin.Debuggable):
package(s) described by it in a format appropriate for passing
to :func:`Bcfg2.Server.Plugins.Packages.Packages.complete`.
By default, that's just the name; only the
- :module:`Bcfg2.Server.Plugins.Packages.Yum` backend supports
+ :mod:`Bcfg2.Server.Plugins.Packages.Yum` backend supports
versions or other extended data. See :ref:`pkg-objects` for
more details.
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
index 9a508daac..2418e6f2b 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
@@ -1,6 +1,4 @@
-import gzip
import tarfile
-from Bcfg2.Compat import cPickle
from Bcfg2.Server.Plugins.Packages.Collection import _Collection
from Bcfg2.Server.Plugins.Packages.Source import Source
@@ -13,24 +11,6 @@ class PacSource(Source):
basegroups = ['arch', 'parabola']
ptype = 'pacman'
- def __init__(self, basepath, xsource, config):
- Source.__init__(self, basepath, xsource, config)
- self.pkgnames = set()
-
- def save_state(self):
- cache = open(self.cachefile, 'wb')
- cPickle.dump((self.pkgnames, self.deps, self.provides),
- cache, 2)
- cache.close()
-
- def load_state(self):
- data = open(self.cachefile)
- self.pkgnames, self.deps, self.provides = cPickle.load(data)
-
- def filter_unknown(self, unknown):
- filtered = set([u for u in unknown if u.startswith('choice')])
- unknown.difference_update(filtered)
-
def get_urls(self):
if not self.rawurl:
rv = []
@@ -65,7 +45,6 @@ class PacSource(Source):
try:
self.debug_log("Packages: try to read %s" % fname)
tar = tarfile.open(fname, "r")
- reader = gzip.GzipFile(fname)
except:
self.logger.error("Packages: Failed to read file %s" % fname)
raise
@@ -76,40 +55,4 @@ class PacSource(Source):
self.debug_log("Packages: added %s" %
tarinfo.name.rsplit("-", 2)[0])
tar.close()
-
- self.deps['global'] = dict()
- self.provides['global'] = dict()
- for barch in bdeps:
- self.deps[barch] = dict()
- self.provides[barch] = dict()
- for pkgname in self.pkgnames:
- pset = set()
- for barch in bdeps:
- if pkgname not in bdeps[barch]:
- bdeps[barch][pkgname] = []
- pset.add(tuple(bdeps[barch][pkgname]))
- if len(pset) == 1:
- self.deps['global'][pkgname] = pset.pop()
- else:
- for barch in bdeps:
- self.deps[barch][pkgname] = bdeps[barch][pkgname]
- provided = set()
- for bprovided in list(bprov.values()):
- provided.update(set(bprovided))
- for prov in provided:
- prset = set()
- for barch in bprov:
- if prov not in bprov[barch]:
- continue
- prset.add(tuple(bprov[barch].get(prov, ())))
- if len(prset) == 1:
- self.provides['global'][prov] = prset.pop()
- else:
- for barch in bprov:
- self.provides[barch][prov] = bprov[barch].get(prov, ())
- self.save_state()
-
- def is_package(self, _, pkg):
- return (pkg in self.pkgnames and
- pkg not in self.blacklist and
- (len(self.whitelist) == 0 or pkg in self.whitelist))
+ self.process_files(bdeps, bprov)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 1e1aaade1..b57d1b0cc 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -1,3 +1,49 @@
+""" ``Source`` objects represent a single <Source> tag in
+``sources.xml``. Note that a single Source tag can itself describe
+multiple repositories (if it uses the "url" attribute instead of
+"rawurl"), and so can the ``Source`` object. This can be the source
+(har har) of some confusion. See
+:func:`Bcfg2.Server.Plugins.Packages.Collection._Collection.sourcelist`
+for the proper way to get all repos from a ``Source`` object.
+
+Source objects are aggregated into
+:class:`Bcfg2.Server.Plugins.Packages.Collection._Collection`
+objects, which are actually called by
+:class:`Bcfg2.Server.Plugins.Packages.Packages`. This way a more
+advanced subclass can query repositories in aggregate rather than
+individually, which may give faster or more accurate results.
+
+The base ``Source`` object must be subclassed to handle each
+repository type. How you subclass ``Source`` will depend on how you
+subclassed
+:class:`Bcfg2.Server.Plugins.Packages.Collection._Collection`; see
+:mod:`Bcfg2.Server.Plugins.Packages.Collection` for more details on
+different methods for doing that.
+
+If you are using the stock (or a near-stock)
+:class:`Bcfg2.Server.Plugins.Packages.Collection._Collection` object,
+then you will need to implement the following methods and attributes
+in your ``Source`` subclass:
+
+* :func:`Source.get_urls`
+* :func:`Source.read_files`
+* :attr:`Source.basegroups`
+* :attr:`Source.ptype`
+
+Additionally, you may want to consider overriding the following
+methods and attributes:
+
+* :func:`Source.is_virtual_package`
+* :func:`Source.get_group`
+* :attr:`Source.unknown_filter`
+* :attr:`Source.load_state`
+* :attr:`Source.save_state`
+
+If you are overriding the ``_Collection`` object in more depth, then
+you have more leeway in what you might want to override or implement
+in your ``Source`` subclass.
+"""
+
import os
import re
import sys
@@ -8,6 +54,13 @@ from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \
def fetch_url(url):
+ """ Return the content of the given URL.
+
+ :param url: The URL to fetch content from.
+ :type url: string
+ :raises: ValueError - Malformed URL
+ :raises: URLError - Failure fetching URL
+ :returns: string - the content of the page at the given URL """
if '@' in url:
mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url)
if not mobj:
@@ -22,6 +75,7 @@ def fetch_url(url):
class SourceInitError(Exception):
+ """ Raised when a :class:`Source` object fails instantiation. """
pass
@@ -30,6 +84,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
pulprepo_re = re.compile(r'pulp/repos/([^/]+)')
genericrepo_re = re.compile('https?://.*?/([^/]+)/?$')
basegroups = []
+ unknown_filter = lambda p: p.startswith("choice")
def __init__(self, basepath, xsource, setup):
Bcfg2.Server.Plugin.Debuggable.__init__(self)
@@ -37,16 +92,18 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
self.xsource = xsource
self.setup = setup
self.essentialpkgs = set()
+ self.pkgnames = set()
try:
self.version = xsource.find('Version').text
except AttributeError:
self.version = None
- for key, tag in [('components', 'Component'), ('arches', 'Arch'),
- ('blacklist', 'Blacklist'),
- ('whitelist', 'Whitelist')]:
- setattr(self, key, [item.text for item in xsource.findall(tag)])
+ self.components = [item.text for item in xsource.findall('Component')]
+ self.arches = [item.text for item in xsource.findall('Arch')]
+ self.blacklist = [item.text for item in xsource.findall('Blacklist')]
+ self.whitelist = [item.text for item in xsource.findall('Whitelist')]
+
self.server_options = dict()
self.client_options = dict()
opts = xsource.findall("Options")
@@ -112,7 +169,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
else: # rawurl given
usettings = [dict(version=self.version, component=None,
arch=arch)]
-
+
for setting in usettings:
if not self.rawurl:
setting['baseurl'] = self.url
@@ -133,7 +190,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
g in self.arches)])))
def load_state(self):
- pass
+ data = open(self.cachefile)
+ (self.pkgnames, self.deps, self.provides,
+ self.essentialpkgs) = cPickle.load(data)
+
+ def save_state(self):
+ cache = open(self.cachefile, 'wb')
+ cPickle.dump((self.pkgnames, self.deps, self.provides,
+ self.essentialpkgs), cache, 2)
+ cache.close()
def setup_data(self, force_update=False):
should_read = True
@@ -234,14 +299,45 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
def escape_url(self, url):
return os.path.join(self.basepath, url.replace('/', '@'))
- def file_init(self):
- pass
-
def read_files(self):
pass
+ def process_files(self, deps, prov):
+ self.deps['global'] = dict()
+ self.provides['global'] = dict()
+ for barch in deps:
+ self.deps[barch] = dict()
+ self.provides[barch] = dict()
+ for pkgname in self.pkgnames:
+ pset = set()
+ for barch in deps:
+ if pkgname not in deps[barch]:
+ deps[barch][pkgname] = []
+ pset.add(tuple(deps[barch][pkgname]))
+ if len(pset) == 1:
+ self.deps['global'][pkgname] = pset.pop()
+ else:
+ for barch in deps:
+ self.deps[barch][pkgname] = deps[barch][pkgname]
+ provided = set()
+ for bprovided in list(prov.values()):
+ provided.update(set(bprovided))
+ for prov in provided:
+ prset = set()
+ for barch in prov:
+ if prov not in prov[barch]:
+ continue
+ prset.add(tuple(prov[barch].get(prov, ())))
+ if len(prset) == 1:
+ self.provides['global'][prov] = prset.pop()
+ else:
+ for barch in prov:
+ self.provides[barch][prov] = prov[barch].get(prov, ())
+ self.save_state()
+
def filter_unknown(self, unknown):
- pass
+ unknown.difference_update(set([u for u in unknown
+ if self.unknown_filter(u)]))
def update(self):
for url in self.urls:
@@ -286,8 +382,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable):
return self.provides[arch][required]
return []
- def is_package(self, metadata, _):
- return False
+ def is_package(self, metadata, pkg):
+ return (pkg in self.pkgnames and
+ pkg not in self.blacklist and
+ (len(self.whitelist) == 0 or pkg in self.whitelist))
def get_package(self, metadata, package):
return package
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index c942a1061..b8648fdde 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -558,6 +558,7 @@ class YumCollection(_Collection):
class YumSource(Source):
basegroups = ['yum', 'redhat', 'centos', 'fedora']
ptype = 'yum'
+ unknown_filter = lambda u: u.startswith("rpmlib")
def __init__(self, basepath, xsource, setup):
Source.__init__(self, basepath, xsource, setup)
@@ -632,20 +633,21 @@ class YumSource(Source):
rmdurl = '%srepodata/repomd.xml' % url
try:
repomd = fetch_url(rmdurl)
- xdata = lxml.etree.XML(repomd)
except ValueError:
self.logger.error("Packages: Bad url string %s" % rmdurl)
return []
- except URLError:
- err = sys.exc_info()[1]
- self.logger.error("Packages: Failed to fetch url %s. %s" %
- (rmdurl, err))
- return []
except HTTPError:
err = sys.exc_info()[1]
self.logger.error("Packages: Failed to fetch url %s. code=%s" %
(rmdurl, err.code))
return []
+ except URLError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Failed to fetch url %s. %s" %
+ (rmdurl, err))
+ return []
+ try:
+ xdata = lxml.etree.XML(repomd)
except lxml.etree.XMLSyntaxError:
err = sys.exc_info()[1]
self.logger.error("Packages: Failed to process metadata at %s: %s" %
@@ -764,17 +766,17 @@ class YumSource(Source):
filtered = set()
for unk in unknown:
try:
- if unk.startswith('rpmlib'):
+ if self.unknown_filter(unk):
filtered.update(unk)
except AttributeError:
try:
- if unk[0].startswith('rpmlib'):
+ if self.unknown_filter(unk[0]):
filtered.update(unk)
except (IndexError, AttributeError):
pass
+ unknown.difference_update(filtered)
else:
- filtered = set([u for u in unknown if u.startswith('rpmlib')])
- unknown.difference_update(filtered)
+ Source.filter_unknown(self, unknown)
def setup_data(self, force_update=False):
if not self.use_yum: