From 000a832751563cfe571363a27e293fd3d9db31a4 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 16 Jan 2022 07:43:05 +0100 Subject: Packages: Support different compression methods The new Reader classes implement different compression methods for the files parsed by the Packages backends. Each source can specify a default compression format. The user can configure a compression format per Source and the filename and extension for the metadata files are generated automatically. --- src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 20 ++++++----- src/lib/Bcfg2/Server/Plugins/Packages/Pac.py | 12 +++++-- src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py | 25 ++++++++++---- .../Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py | 14 ++++++++ .../Bcfg2/Server/Plugins/Packages/Readers/Gzip.py | 11 ++++++ .../Bcfg2/Server/Plugins/Packages/Readers/None.py | 10 ++++++ .../Bcfg2/Server/Plugins/Packages/Readers/Xz.py | 11 ++++++ .../Server/Plugins/Packages/Readers/__init__.py | 36 ++++++++++++++++++++ src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 39 ++++++++++++++++++++++ src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 15 ++++++++- 10 files changed, 173 insertions(+), 20 deletions(-) create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py (limited to 'src/lib/Bcfg2') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 956cb9f51..cbbeb21eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -1,7 +1,6 @@ """ APT backend for :mod:`Bcfg2.Server.Plugins.Packages` """ import re -import gzip from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source @@ -68,19 +67,24 @@ class AptSource(Source): #: AptSource sets the ``type`` on Package entries to "deb" ptype = 'deb' + #: Most (3rd-party) debian repositories still only support "gzip". + default_compression = 'gzip' + @property def urls(self): """ A list of URLs to the base metadata file for each repository described by this source. """ + fname = self.build_filename('Packages') + if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: - rv.append("%sdists/%s/%s/binary-%s/Packages.gz" % - (self.url, self.version, part, arch)) + rv.append("%sdists/%s/%s/binary-%s/%s" % + (self.url, self.version, part, arch, fname)) return rv else: - return ["%sPackages.gz" % self.rawurl] + return ["%s%s" % (self.rawurl, fname)] def read_files(self): # pylint: disable=R0912 bdeps = dict() @@ -101,11 +105,8 @@ class AptSource(Source): bdeps[barch] = dict() brecs[barch] = dict() bprov[barch] = dict() - try: - reader = gzip.GzipFile(fname) - except IOError: - self.logger.error("Packages: Failed to read file %s" % fname) - raise + + reader = self.open_file(fname) for line in reader.readlines(): if not isinstance(line, str): line = line.decode('utf-8') @@ -150,5 +151,6 @@ class AptSource(Source): if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) + reader.close() self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 6fc084cc4..e3432c934 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -87,6 +87,9 @@ class PacSource(Source): #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' + #: The database of pacman repositories is compressed with "gzip" + default_compression = 'gzip' + def __init__(self, basepath, xsource): self.pacgroups = {} @@ -113,9 +116,10 @@ class PacSource(Source): if not self.rawurl: rv = [] for part in self.components: + filename = self.build_filename("%s.db.tar" % part) for arch in self.arches: - rv.append("%s%s/os/%s/%s.db.tar.gz" % - (self.url, part, arch, part)) + rv.append("%s%s/os/%s/%s" % + (self.url, part, arch, filename)) return rv else: raise Exception("PacSource : RAWUrl not supported (yet)") @@ -140,7 +144,8 @@ class PacSource(Source): bprov[barch] = {} try: self.debug_log("Packages: try to read %s" % fname) - tar = tarfile.open(fname, "r") + reader = self.open_file(fname) + tar = tarfile.open(fileobj=reader) except (IOError, tarfile.TarError): self.logger.error("Packages: Failed to read file %s" % fname) raise @@ -185,6 +190,7 @@ class PacSource(Source): self.pacgroups[group].append(pkgname) tar.close() + reader.close() self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py index 4938efb94..1fff3b7a7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -1,6 +1,5 @@ """ pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """ -import lzma import tarfile try: @@ -40,19 +39,30 @@ class PkgngSource(Source): #: PkgngSource sets the ``type`` on Package entries to "pkgng" ptype = 'pkgng' + #: The "packagesite" files of pkgng repositories are compressed + #: with "xz" + default_compression = 'xz' + + def _get_extension(self): + extension = super(PkgngSource, self)._get_extension() + if extension == '': + return 'tar' + return 't%s' % extension + @property def urls(self): """ A list of URLs to the base metadata file for each repository described by this source. """ + fname = self.build_filename("packagesite") if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: - rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" % - (self.url, self.version, arch, part)) + rv.append("%s/freebsd:%s:%s/%s/%s" % + (self.url, self.version, arch, part, fname)) return rv else: - return ["%s/packagesite.txz" % self.rawurl] + return ["%s/%s" % (self.rawurl, fname)] def read_files(self): bdeps = dict() @@ -70,12 +80,13 @@ class PkgngSource(Source): if barch not in bdeps: bdeps[barch] = dict() try: - tar = tarfile.open(fileobj=lzma.LZMAFile(fname)) - reader = tar.extractfile('packagesite.yaml') + reader = self.open_file(fname) + tar = tarfile.open(fileobj=reader) + packagesite = tar.extractfile('packagesite.yaml') except (IOError, tarfile.TarError): self.logger.error("Packages: Failed to read file %s" % fname) raise - for line in reader.readlines(): + for line in packagesite.readlines(): if not isinstance(line, str): line = line.decode('utf-8') pkg = json.loads(line) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py new file mode 100644 index 000000000..2267197ca --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py @@ -0,0 +1,14 @@ +""" Reader for bzip2 compressed package sources. """ + +import bz2 +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class Bzip2Reader(Reader): + extension = 'bz' + + def _open(self, filename): + return bz2.BZ2File(filename) + + def readable(self): + return True diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py new file mode 100644 index 000000000..8e67e2b33 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py @@ -0,0 +1,11 @@ +""" Reader for gzip compressed package sources. """ + +import gzip +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class GzipReader(Reader): + extension = 'gz' + + def _open(self, filename): + return gzip.GzipFile(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py new file mode 100644 index 000000000..2f7a18d84 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py @@ -0,0 +1,10 @@ +""" Reader for uncompressed package sources. """ + +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class NoneReader(Reader): + extension = '' + + def _open(self, filename): + return open(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py new file mode 100644 index 000000000..39a058767 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py @@ -0,0 +1,11 @@ +""" Reader for lzma compressed package sources. """ + +import lzma +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class XzReader(Reader): + extension = 'xz' + + def _open(self, filename): + return lzma.LZMAFile(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py new file mode 100644 index 000000000..51b4fb79c --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py @@ -0,0 +1,36 @@ +"""This module implements different readers for package files.""" + +from io import IOBase +from Bcfg2.Compat import walk_packages + + +def get_readers(): + """ Return all available packages readers. """ + return [m[1] # pylint: disable=C0103 + for m in walk_packages(path=__path__)] + + +class Reader(IOBase): + extension = None + + def __init__(self, name): + self.name = name + self._file = self._open(name) + + def _open(self, filename): + raise NotImplementedError + + def read(self, size=-1): + return self._file.read(size) + + def readable(self): + return self._file.readable() + + def readline(self, size=-1): + return self._file.readline(size) + + def readlines(self, hint=None): + return self._file.readlines(size) + + def writelines(self, lines): + self._unsupported("writelines") diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 86f7698f7..c80604d01 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -49,6 +49,7 @@ in your ``Source`` subclass. For an example of this kind of import os import re import sys +import Bcfg2.Options from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ @@ -111,6 +112,12 @@ class Source(Debuggable): # pylint: disable=R0902 #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None + #: The default compression format used by this Source class. This + #: is the file the package metadata files should be loaded. It is + #: used if a source has no custom compression format specified + #: in the :attr:`server_options`. + default_compression = 'None' + def __init__(self, basepath, xsource): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache @@ -328,6 +335,38 @@ class Source(Debuggable): # pylint: disable=R0902 self.conditions.append(lambda m, el=el: el.get("name") == m.hostname) + def _get_reader(self): + ctype = self.default_compression + if 'compression' in self.server_options: + ctype = self.server_options['compression'] + + for mod in Bcfg2.Options.setup.packages_readers: + if mod.__name__.endswith(".%s" % ctype.title()): + return getattr(mod, "%sReader" % ctype.title()) + + raise ValueError("Packages: Unknown compression type %s" % ctype) + + def _get_extension(self): + cls = self._get_reader() + if cls.extension is None: + raise ValueError("%s does not define an extension" % + cls.__name__) + return cls.extension + + def build_filename(self, basename): + extension = self._get_extension() + if extension == '': + return basename + return "%s.%s" % (basename, extension) + + def open_file(self, fname): + try: + cls = self._get_reader() + return cls(fname) + except IOError: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + @property def cachekey(self): """ A unique key for this source that will be used to generate diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 23ccd7b8e..1b53b1bb4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -14,6 +14,7 @@ from Bcfg2.Compat import urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Plugins.Packages.Readers import get_readers from Bcfg2.Server.Statistics import track_statistics @@ -36,6 +37,12 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction): fail_silently = True +class PackagesReadersAction(Bcfg2.Options.ComponentAction): + """ ComponentAction to load Packages readers """ + bases = ['Bcfg2.Server.Plugins.Packages.Readers'] + module = True + + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -82,7 +89,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, cf=("packages", "apt_config"), help="The default path for generated apt configs", default="/etc/apt/sources.list.d/" - "bcfg2-packages-generated-sources.list")] + "bcfg2-packages-generated-sources.list"), + Bcfg2.Options.Option( + cf=("packages", "readers"), dest="packages_readers", + help="Packages readers to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesReadersAction, + default=get_readers())] #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. -- cgit v1.2.3-1-g7c22