Packages: Support different compression methods

The new Reader classes implement different compression methods for the files
parsed by the Packages backends. Each source can specify a default compression
format. The user can configure a compression format per Source and the filename
and extension for the metadata files are generated automatically.

11 files changed, 174 insertions, 20 deletions

diff --git a/setup.py b/setup.py
index 5ff2ac003..383202490 100755
--- a/setup.py
+++ b/setup.py
@@ -47,6 +47,7 @@ setup(name="Bcfg2",
                 "Bcfg2.Server.Plugin",
                 "Bcfg2.Server.Plugins",
                 "Bcfg2.Server.Plugins.Packages",
+                "Bcfg2.Server.Plugins.Packages.Readers",
                 "Bcfg2.Server.Plugins.Cfg",
                 "Bcfg2.Server.Reports",
                 "Bcfg2.Server.Reports.reports",
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 956cb9f51..cbbeb21eb 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -1,7 +1,6 @@
 """ APT backend for :mod:`Bcfg2.Server.Plugins.Packages` """
 
 import re
-import gzip
 from Bcfg2.Server.Plugins.Packages.Collection import Collection
 from Bcfg2.Server.Plugins.Packages.Source import Source
 
@@ -68,19 +67,24 @@ class AptSource(Source):
     #: AptSource sets the ``type`` on Package entries to "deb"
     ptype = 'deb'
 
+    #: Most (3rd-party) debian repositories still only support "gzip".
+    default_compression = 'gzip'
+
     @property
     def urls(self):
         """ A list of URLs to the base metadata file for each
         repository described by this source. """
+        fname = self.build_filename('Packages')
+
         if not self.rawurl:
             rv = []
             for part in self.components:
                 for arch in self.arches:
-                    rv.append("%sdists/%s/%s/binary-%s/Packages.gz" %
-                              (self.url, self.version, part, arch))
+                    rv.append("%sdists/%s/%s/binary-%s/%s" %
+                              (self.url, self.version, part, arch, fname))
             return rv
         else:
-            return ["%sPackages.gz" % self.rawurl]
+            return ["%s%s" % (self.rawurl, fname)]
 
     def read_files(self):  # pylint: disable=R0912
         bdeps = dict()
@@ -101,11 +105,8 @@ class AptSource(Source):
                 bdeps[barch] = dict()
                 brecs[barch] = dict()
                 bprov[barch] = dict()
-            try:
-                reader = gzip.GzipFile(fname)
-            except IOError:
-                self.logger.error("Packages: Failed to read file %s" % fname)
-                raise
+
+            reader = self.open_file(fname)
             for line in reader.readlines():
                 if not isinstance(line, str):
                     line = line.decode('utf-8')
@@ -150,5 +151,6 @@ class AptSource(Source):
                         if dname not in bprov[barch]:
                             bprov[barch][dname] = set()
                         bprov[barch][dname].add(pkgname)
+            reader.close()
         self.process_files(bdeps, bprov, brecs)
     read_files.__doc__ = Source.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
index 6fc084cc4..e3432c934 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
@@ -87,6 +87,9 @@ class PacSource(Source):
     #: PacSource sets the ``type`` on Package entries to "pacman"
     ptype = 'pacman'
 
+    #: The database of pacman repositories is compressed with "gzip"
+    default_compression = 'gzip'
+
     def __init__(self, basepath, xsource):
         self.pacgroups = {}
 
@@ -113,9 +116,10 @@ class PacSource(Source):
         if not self.rawurl:
             rv = []
             for part in self.components:
+                filename = self.build_filename("%s.db.tar" % part)
                 for arch in self.arches:
-                    rv.append("%s%s/os/%s/%s.db.tar.gz" %
-                              (self.url, part, arch, part))
+                    rv.append("%s%s/os/%s/%s" %
+                              (self.url, part, arch, filename))
             return rv
         else:
             raise Exception("PacSource : RAWUrl not supported (yet)")
@@ -140,7 +144,8 @@ class PacSource(Source):
                 bprov[barch] = {}
             try:
                 self.debug_log("Packages: try to read %s" % fname)
-                tar = tarfile.open(fname, "r")
+                reader = self.open_file(fname)
+                tar = tarfile.open(fileobj=reader)
             except (IOError, tarfile.TarError):
                 self.logger.error("Packages: Failed to read file %s" % fname)
                 raise
@@ -185,6 +190,7 @@ class PacSource(Source):
                         self.pacgroups[group].append(pkgname)
 
             tar.close()
+            reader.close()
         self.process_files(bdeps, bprov, brecs)
     read_files.__doc__ = Source.read_files.__doc__
 
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
index 4938efb94..1fff3b7a7 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
@@ -1,6 +1,5 @@
 """ pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """
 
-import lzma
 import tarfile
 
 try:
@@ -40,19 +39,30 @@ class PkgngSource(Source):
     #: PkgngSource sets the ``type`` on Package entries to "pkgng"
     ptype = 'pkgng'
 
+    #: The "packagesite" files of pkgng repositories are compressed
+    #: with "xz"
+    default_compression = 'xz'
+
+    def _get_extension(self):
+        extension = super(PkgngSource, self)._get_extension()
+        if extension == '':
+            return 'tar'
+        return 't%s' % extension
+
     @property
     def urls(self):
         """ A list of URLs to the base metadata file for each
         repository described by this source. """
+        fname = self.build_filename("packagesite")
         if not self.rawurl:
             rv = []
             for part in self.components:
                 for arch in self.arches:
-                    rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" %
-                              (self.url, self.version, arch, part))
+                    rv.append("%s/freebsd:%s:%s/%s/%s" %
+                              (self.url, self.version, arch, part, fname))
             return rv
         else:
-            return ["%s/packagesite.txz" % self.rawurl]
+            return ["%s/%s" % (self.rawurl, fname)]
 
     def read_files(self):
         bdeps = dict()
@@ -70,12 +80,13 @@ class PkgngSource(Source):
             if barch not in bdeps:
                 bdeps[barch] = dict()
             try:
-                tar = tarfile.open(fileobj=lzma.LZMAFile(fname))
-                reader = tar.extractfile('packagesite.yaml')
+                reader = self.open_file(fname)
+                tar = tarfile.open(fileobj=reader)
+                packagesite = tar.extractfile('packagesite.yaml')
             except (IOError, tarfile.TarError):
                 self.logger.error("Packages: Failed to read file %s" % fname)
                 raise
-            for line in reader.readlines():
+            for line in packagesite.readlines():
                 if not isinstance(line, str):
                     line = line.decode('utf-8')
                 pkg = json.loads(line)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py
new file mode 100644
index 000000000..2267197ca
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py
@@ -0,0 +1,14 @@
+""" Reader for bzip2 compressed package sources. """
+
+import bz2
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class Bzip2Reader(Reader):
+    extension = 'bz'
+
+    def _open(self, filename):
+        return bz2.BZ2File(filename)
+
+    def readable(self):
+        return True
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py
new file mode 100644
index 000000000..8e67e2b33
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py
@@ -0,0 +1,11 @@
+""" Reader for gzip compressed package sources. """
+
+import gzip
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class GzipReader(Reader):
+    extension = 'gz'
+
+    def _open(self, filename):
+        return gzip.GzipFile(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py
new file mode 100644
index 000000000..2f7a18d84
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py
@@ -0,0 +1,10 @@
+""" Reader for uncompressed package sources. """
+
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class NoneReader(Reader):
+    extension = ''
+
+    def _open(self, filename):
+        return open(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py
new file mode 100644
index 000000000..39a058767
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py
@@ -0,0 +1,11 @@
+""" Reader for lzma compressed package sources. """
+
+import lzma
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class XzReader(Reader):
+    extension = 'xz'
+
+    def _open(self, filename):
+        return lzma.LZMAFile(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py
new file mode 100644
index 000000000..51b4fb79c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py
@@ -0,0 +1,36 @@
+"""This module implements different readers for package files."""
+
+from io import IOBase
+from Bcfg2.Compat import walk_packages
+
+
+def get_readers():
+    """ Return all available packages readers. """
+    return [m[1]  # pylint: disable=C0103
+            for m in walk_packages(path=__path__)]
+
+
+class Reader(IOBase):
+    extension = None
+
+    def __init__(self, name):
+        self.name = name
+        self._file = self._open(name)
+
+    def _open(self, filename):
+        raise NotImplementedError
+
+    def read(self, size=-1):
+        return self._file.read(size)
+
+    def readable(self):
+        return self._file.readable()
+
+    def readline(self, size=-1):
+        return self._file.readline(size)
+
+    def readlines(self, hint=None):
+        return self._file.readlines(size)
+
+    def writelines(self, lines):
+        self._unsupported("writelines")
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 86f7698f7..c80604d01 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -49,6 +49,7 @@ in your ``Source`` subclass.  For an example of this kind of
 import os
 import re
 import sys
+import Bcfg2.Options
 from Bcfg2.Logger import Debuggable
 from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \
     HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \
@@ -111,6 +112,12 @@ class Source(Debuggable):  # pylint: disable=R0902
     #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`.
     ptype = None
 
+    #: The default compression format used by this Source class. This
+    #: is the file the package metadata files should be loaded. It is
+    #: used if a source has no custom compression format specified
+    #: in the :attr:`server_options`.
+    default_compression = 'None'
+
     def __init__(self, basepath, xsource):  # pylint: disable=R0912
         """
         :param basepath: The base filesystem path under which cache
@@ -328,6 +335,38 @@ class Source(Debuggable):  # pylint: disable=R0902
                     self.conditions.append(lambda m, el=el:
                                            el.get("name") == m.hostname)
 
+    def _get_reader(self):
+        ctype = self.default_compression
+        if 'compression' in self.server_options:
+            ctype = self.server_options['compression']
+
+        for mod in Bcfg2.Options.setup.packages_readers:
+            if mod.__name__.endswith(".%s" % ctype.title()):
+                return getattr(mod, "%sReader" % ctype.title())
+
+        raise ValueError("Packages: Unknown compression type %s" % ctype)
+
+    def _get_extension(self):
+        cls = self._get_reader()
+        if cls.extension is None:
+            raise ValueError("%s does not define an extension" %
+                             cls.__name__)
+        return cls.extension
+
+    def build_filename(self, basename):
+        extension = self._get_extension()
+        if extension == '':
+            return basename
+        return "%s.%s" % (basename, extension)
+
+    def open_file(self, fname):
+        try:
+            cls = self._get_reader()
+            return cls(fname)
+        except IOError:
+            self.logger.error("Packages: Failed to read file %s" % fname)
+            raise
+
     @property
     def cachekey(self):
         """ A unique key for this source that will be used to generate
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 23ccd7b8e..1b53b1bb4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -14,6 +14,7 @@ from Bcfg2.Compat import urlopen, HTTPError, URLError
 from Bcfg2.Server.Plugins.Packages.Collection import Collection, \
     get_collection_class
 from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
+from Bcfg2.Server.Plugins.Packages.Readers import get_readers
 from Bcfg2.Server.Statistics import track_statistics
 
 
@@ -36,6 +37,12 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction):
     fail_silently = True
 
 
+class PackagesReadersAction(Bcfg2.Options.ComponentAction):
+    """ ComponentAction to load Packages readers """
+    bases = ['Bcfg2.Server.Plugins.Packages.Readers']
+    module = True
+
+
 class Packages(Bcfg2.Server.Plugin.Plugin,
                Bcfg2.Server.Plugin.StructureValidator,
                Bcfg2.Server.Plugin.Generator,
@@ -82,7 +89,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             cf=("packages", "apt_config"),
             help="The default path for generated apt configs",
             default="/etc/apt/sources.list.d/"
-            "bcfg2-packages-generated-sources.list")]
+            "bcfg2-packages-generated-sources.list"),
+        Bcfg2.Options.Option(
+            cf=("packages", "readers"), dest="packages_readers",
+            help="Packages readers to load",
+            type=Bcfg2.Options.Types.comma_list,
+            action=PackagesReadersAction,
+            default=get_readers())]
 
     #: Packages is an alternative to
     #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it.