From 000a832751563cfe571363a27e293fd3d9db31a4 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 16 Jan 2022 07:43:05 +0100 Subject: Packages: Support different compression methods The new Reader classes implement different compression methods for the files parsed by the Packages backends. Each source can specify a default compression format. The user can configure a compression format per Source and the filename and extension for the metadata files are generated automatically. --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Source.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 86f7698f7..c80604d01 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -49,6 +49,7 @@ in your ``Source`` subclass. For an example of this kind of import os import re import sys +import Bcfg2.Options from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ @@ -111,6 +112,12 @@ class Source(Debuggable): # pylint: disable=R0902 #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None + #: The default compression format used by this Source class. This + #: is the file the package metadata files should be loaded. It is + #: used if a source has no custom compression format specified + #: in the :attr:`server_options`. + default_compression = 'None' + def __init__(self, basepath, xsource): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache @@ -328,6 +335,38 @@ class Source(Debuggable): # pylint: disable=R0902 self.conditions.append(lambda m, el=el: el.get("name") == m.hostname) + def _get_reader(self): + ctype = self.default_compression + if 'compression' in self.server_options: + ctype = self.server_options['compression'] + + for mod in Bcfg2.Options.setup.packages_readers: + if mod.__name__.endswith(".%s" % ctype.title()): + return getattr(mod, "%sReader" % ctype.title()) + + raise ValueError("Packages: Unknown compression type %s" % ctype) + + def _get_extension(self): + cls = self._get_reader() + if cls.extension is None: + raise ValueError("%s does not define an extension" % + cls.__name__) + return cls.extension + + def build_filename(self, basename): + extension = self._get_extension() + if extension == '': + return basename + return "%s.%s" % (basename, extension) + + def open_file(self, fname): + try: + cls = self._get_reader() + return cls(fname) + except IOError: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + @property def cachekey(self): """ A unique key for this source that will be used to generate -- cgit v1.2.3-1-g7c22 From 830007663cf2d83ccaa58f9c6caa8f461f6d8234 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 1 Nov 2012 00:06:17 +0100 Subject: Packages: Add priority to sources and sort them --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Source.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index c80604d01..b2422d84e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -195,6 +195,9 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "name" attribute from :attr:`xsource` self.name = None + #: The "priority" attribute from :attr:`xsource` + self.priority = xsource.get('priority', 500) + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -257,11 +260,13 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch, debsrc=self.debsrc) + arch=arch, debsrc=self.debsrc, + priority=self.priority) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch, debsrc=self.debsrc)] + arch=arch, debsrc=self.debsrc, + priority=self.priority)] for setting in usettings: if not self.rawurl: -- cgit v1.2.3-1-g7c22 From 18c024d69b373af53398157c658bcd2265d5d9c0 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 1 Mar 2013 01:07:05 +0100 Subject: Packages: Add 'pin' attribute --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Source.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index b2422d84e..2b7514b06 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -198,6 +198,9 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "priority" attribute from :attr:`xsource` self.priority = xsource.get('priority', 500) + #: The "pin" attribute from :attr:`xsource` + self.pin = xsource.get('pin', '') + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -261,12 +264,12 @@ class Source(Debuggable): # pylint: disable=R0902 if self.url: usettings = [dict(version=self.version, component=comp, arch=arch, debsrc=self.debsrc, - priority=self.priority) + priority=self.priority, pin=self.pin) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, arch=arch, debsrc=self.debsrc, - priority=self.priority)] + priority=self.priority, pin=self.pin)] for setting in usettings: if not self.rawurl: -- cgit v1.2.3-1-g7c22 From 8d0683c366afc6c5c6db39ea722fef37daa5ea6d Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 10 May 2016 16:27:52 +0200 Subject: Packages: Add repo options to additional_data --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Source.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 2b7514b06..9c9c9c5bb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -278,6 +278,8 @@ class Source(Debuggable): # pylint: disable=R0902 setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting setting['name'] = self.get_repo_name(setting) + setting['options'] = dict(server=self.server_options, + client=self.client_options) self.url_map.extend(usettings) def _init_attributes(self, xsource): -- cgit v1.2.3-1-g7c22 From 9cef2774875594d91d73a4cbc3cd6935bc992cec Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 16 Jan 2022 19:59:25 +0100 Subject: Packages: Add possibility to customize User-Agent Some mirror might block the default python urllib User-Agent. --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Source.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 9c9c9c5bb..b0c4bf44f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -57,7 +57,7 @@ from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ from Bcfg2.Server.Statistics import track_statistics -def fetch_url(url): +def fetch_url(url, opts): """ Return the content of the given URL. :param url: The URL to fetch content from. @@ -74,8 +74,14 @@ def fetch_url(url): url = mobj.group(1) + mobj.group(4) auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) auth.add_password(None, url, user, passwd) - install_opener(build_opener(auth)) - return urlopen(url).read() + req = build_opener(auth) + else: + req = build_opener() + + if 'user-agent' in opts: + req.addheaders = [('User-Agent', opts['user-agent'])] + + return req.open(url).read() class SourceInitError(Exception): @@ -735,7 +741,7 @@ class Source(Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'wb').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url, self.server_options)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise -- cgit v1.2.3-1-g7c22 From c22cd2a2fc8b909c08b1d8df8d5cc3909e8aeccc Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 16 Jan 2022 21:24:24 +0100 Subject: Packages: Allow to filter provided packages The provides packages of a source should be filtered by the Blacklist or Whitelist tags for the source. --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server/Plugins/Packages/Source.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index b0c4bf44f..574dbd851 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -589,6 +589,9 @@ class Source(Debuggable): # pylint: disable=R0902 self.logger.warning("%s provides no packages for %s" % (self, agrp)) continue + if (agrp in self.blacklist or + (len(self.whitelist) != 0 and agrp not in self.whitelist)): + continue for key, value in list(self.provides[agrp].items()): if key not in vdict: vdict[key] = set(value) @@ -815,7 +818,9 @@ class Source(Debuggable): # pylint: disable=R0902 :returns: list of strings """ for arch in self.get_arches(metadata): - if package in self.provides[arch]: + if (package in self.provides[arch] and + package not in self.blacklist and + (len(self.whitelist) == 0 or package in self.whitelist)): return self.provides[arch][package] return [] -- cgit v1.2.3-1-g7c22