From 000a832751563cfe571363a27e293fd3d9db31a4 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <asulfrian@zedat.fu-berlin.de>
Date: Sun, 16 Jan 2022 07:43:05 +0100
Subject: Packages: Support different compression methods

The new Reader classes implement different compression methods for the files
parsed by the Packages backends. Each source can specify a default compression
format. The user can configure a compression format per Source and the filename
and extension for the metadata files are generated automatically.
---
 src/lib/Bcfg2/Server/Plugins/Packages/Apt.py       | 20 ++++++-----
 src/lib/Bcfg2/Server/Plugins/Packages/Pac.py       | 12 +++++--
 src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py     | 25 ++++++++++----
 .../Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py | 14 ++++++++
 .../Bcfg2/Server/Plugins/Packages/Readers/Gzip.py  | 11 ++++++
 .../Bcfg2/Server/Plugins/Packages/Readers/None.py  | 10 ++++++
 .../Bcfg2/Server/Plugins/Packages/Readers/Xz.py    | 11 ++++++
 .../Server/Plugins/Packages/Readers/__init__.py    | 36 ++++++++++++++++++++
 src/lib/Bcfg2/Server/Plugins/Packages/Source.py    | 39 ++++++++++++++++++++++
 src/lib/Bcfg2/Server/Plugins/Packages/__init__.py  | 15 ++++++++-
 10 files changed, 173 insertions(+), 20 deletions(-)
 create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py
 create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py
 create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py
 create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py
 create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 956cb9f51..cbbeb21eb 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -1,7 +1,6 @@
 """ APT backend for :mod:`Bcfg2.Server.Plugins.Packages` """
 
 import re
-import gzip
 from Bcfg2.Server.Plugins.Packages.Collection import Collection
 from Bcfg2.Server.Plugins.Packages.Source import Source
 
@@ -68,19 +67,24 @@ class AptSource(Source):
     #: AptSource sets the ``type`` on Package entries to "deb"
     ptype = 'deb'
 
+    #: Most (3rd-party) debian repositories still only support "gzip".
+    default_compression = 'gzip'
+
     @property
     def urls(self):
         """ A list of URLs to the base metadata file for each
         repository described by this source. """
+        fname = self.build_filename('Packages')
+
         if not self.rawurl:
             rv = []
             for part in self.components:
                 for arch in self.arches:
-                    rv.append("%sdists/%s/%s/binary-%s/Packages.gz" %
-                              (self.url, self.version, part, arch))
+                    rv.append("%sdists/%s/%s/binary-%s/%s" %
+                              (self.url, self.version, part, arch, fname))
             return rv
         else:
-            return ["%sPackages.gz" % self.rawurl]
+            return ["%s%s" % (self.rawurl, fname)]
 
     def read_files(self):  # pylint: disable=R0912
         bdeps = dict()
@@ -101,11 +105,8 @@ class AptSource(Source):
                 bdeps[barch] = dict()
                 brecs[barch] = dict()
                 bprov[barch] = dict()
-            try:
-                reader = gzip.GzipFile(fname)
-            except IOError:
-                self.logger.error("Packages: Failed to read file %s" % fname)
-                raise
+
+            reader = self.open_file(fname)
             for line in reader.readlines():
                 if not isinstance(line, str):
                     line = line.decode('utf-8')
@@ -150,5 +151,6 @@ class AptSource(Source):
                         if dname not in bprov[barch]:
                             bprov[barch][dname] = set()
                         bprov[barch][dname].add(pkgname)
+            reader.close()
         self.process_files(bdeps, bprov, brecs)
     read_files.__doc__ = Source.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
index 6fc084cc4..e3432c934 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
@@ -87,6 +87,9 @@ class PacSource(Source):
     #: PacSource sets the ``type`` on Package entries to "pacman"
     ptype = 'pacman'
 
+    #: The database of pacman repositories is compressed with "gzip"
+    default_compression = 'gzip'
+
     def __init__(self, basepath, xsource):
         self.pacgroups = {}
 
@@ -113,9 +116,10 @@ class PacSource(Source):
         if not self.rawurl:
             rv = []
             for part in self.components:
+                filename = self.build_filename("%s.db.tar" % part)
                 for arch in self.arches:
-                    rv.append("%s%s/os/%s/%s.db.tar.gz" %
-                              (self.url, part, arch, part))
+                    rv.append("%s%s/os/%s/%s" %
+                              (self.url, part, arch, filename))
             return rv
         else:
             raise Exception("PacSource : RAWUrl not supported (yet)")
@@ -140,7 +144,8 @@ class PacSource(Source):
                 bprov[barch] = {}
             try:
                 self.debug_log("Packages: try to read %s" % fname)
-                tar = tarfile.open(fname, "r")
+                reader = self.open_file(fname)
+                tar = tarfile.open(fileobj=reader)
             except (IOError, tarfile.TarError):
                 self.logger.error("Packages: Failed to read file %s" % fname)
                 raise
@@ -185,6 +190,7 @@ class PacSource(Source):
                         self.pacgroups[group].append(pkgname)
 
             tar.close()
+            reader.close()
         self.process_files(bdeps, bprov, brecs)
     read_files.__doc__ = Source.read_files.__doc__
 
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
index 4938efb94..1fff3b7a7 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
@@ -1,6 +1,5 @@
 """ pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """
 
-import lzma
 import tarfile
 
 try:
@@ -40,19 +39,30 @@ class PkgngSource(Source):
     #: PkgngSource sets the ``type`` on Package entries to "pkgng"
     ptype = 'pkgng'
 
+    #: The "packagesite" files of pkgng repositories are compressed
+    #: with "xz"
+    default_compression = 'xz'
+
+    def _get_extension(self):
+        extension = super(PkgngSource, self)._get_extension()
+        if extension == '':
+            return 'tar'
+        return 't%s' % extension
+
     @property
     def urls(self):
         """ A list of URLs to the base metadata file for each
         repository described by this source. """
+        fname = self.build_filename("packagesite")
         if not self.rawurl:
             rv = []
             for part in self.components:
                 for arch in self.arches:
-                    rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" %
-                              (self.url, self.version, arch, part))
+                    rv.append("%s/freebsd:%s:%s/%s/%s" %
+                              (self.url, self.version, arch, part, fname))
             return rv
         else:
-            return ["%s/packagesite.txz" % self.rawurl]
+            return ["%s/%s" % (self.rawurl, fname)]
 
     def read_files(self):
         bdeps = dict()
@@ -70,12 +80,13 @@ class PkgngSource(Source):
             if barch not in bdeps:
                 bdeps[barch] = dict()
             try:
-                tar = tarfile.open(fileobj=lzma.LZMAFile(fname))
-                reader = tar.extractfile('packagesite.yaml')
+                reader = self.open_file(fname)
+                tar = tarfile.open(fileobj=reader)
+                packagesite = tar.extractfile('packagesite.yaml')
             except (IOError, tarfile.TarError):
                 self.logger.error("Packages: Failed to read file %s" % fname)
                 raise
-            for line in reader.readlines():
+            for line in packagesite.readlines():
                 if not isinstance(line, str):
                     line = line.decode('utf-8')
                 pkg = json.loads(line)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py
new file mode 100644
index 000000000..2267197ca
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py
@@ -0,0 +1,14 @@
+""" Reader for bzip2 compressed package sources. """
+
+import bz2
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class Bzip2Reader(Reader):
+    extension = 'bz'
+
+    def _open(self, filename):
+        return bz2.BZ2File(filename)
+
+    def readable(self):
+        return True
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py
new file mode 100644
index 000000000..8e67e2b33
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py
@@ -0,0 +1,11 @@
+""" Reader for gzip compressed package sources. """
+
+import gzip
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class GzipReader(Reader):
+    extension = 'gz'
+
+    def _open(self, filename):
+        return gzip.GzipFile(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py
new file mode 100644
index 000000000..2f7a18d84
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py
@@ -0,0 +1,10 @@
+""" Reader for uncompressed package sources. """
+
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class NoneReader(Reader):
+    extension = ''
+
+    def _open(self, filename):
+        return open(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py
new file mode 100644
index 000000000..39a058767
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py
@@ -0,0 +1,11 @@
+""" Reader for lzma compressed package sources. """
+
+import lzma
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class XzReader(Reader):
+    extension = 'xz'
+
+    def _open(self, filename):
+        return lzma.LZMAFile(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py
new file mode 100644
index 000000000..51b4fb79c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py
@@ -0,0 +1,36 @@
+"""This module implements different readers for package files."""
+
+from io import IOBase
+from Bcfg2.Compat import walk_packages
+
+
+def get_readers():
+    """ Return all available packages readers. """
+    return [m[1]  # pylint: disable=C0103
+            for m in walk_packages(path=__path__)]
+
+
+class Reader(IOBase):
+    extension = None
+
+    def __init__(self, name):
+        self.name = name
+        self._file = self._open(name)
+
+    def _open(self, filename):
+        raise NotImplementedError
+
+    def read(self, size=-1):
+        return self._file.read(size)
+
+    def readable(self):
+        return self._file.readable()
+
+    def readline(self, size=-1):
+        return self._file.readline(size)
+
+    def readlines(self, hint=None):
+        return self._file.readlines(size)
+
+    def writelines(self, lines):
+        self._unsupported("writelines")
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 86f7698f7..c80604d01 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -49,6 +49,7 @@ in your ``Source`` subclass.  For an example of this kind of
 import os
 import re
 import sys
+import Bcfg2.Options
 from Bcfg2.Logger import Debuggable
 from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \
     HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \
@@ -111,6 +112,12 @@ class Source(Debuggable):  # pylint: disable=R0902
     #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`.
     ptype = None
 
+    #: The default compression format used by this Source class. This
+    #: is the file the package metadata files should be loaded. It is
+    #: used if a source has no custom compression format specified
+    #: in the :attr:`server_options`.
+    default_compression = 'None'
+
     def __init__(self, basepath, xsource):  # pylint: disable=R0912
         """
         :param basepath: The base filesystem path under which cache
@@ -328,6 +335,38 @@ class Source(Debuggable):  # pylint: disable=R0902
                     self.conditions.append(lambda m, el=el:
                                            el.get("name") == m.hostname)
 
+    def _get_reader(self):
+        ctype = self.default_compression
+        if 'compression' in self.server_options:
+            ctype = self.server_options['compression']
+
+        for mod in Bcfg2.Options.setup.packages_readers:
+            if mod.__name__.endswith(".%s" % ctype.title()):
+                return getattr(mod, "%sReader" % ctype.title())
+
+        raise ValueError("Packages: Unknown compression type %s" % ctype)
+
+    def _get_extension(self):
+        cls = self._get_reader()
+        if cls.extension is None:
+            raise ValueError("%s does not define an extension" %
+                             cls.__name__)
+        return cls.extension
+
+    def build_filename(self, basename):
+        extension = self._get_extension()
+        if extension == '':
+            return basename
+        return "%s.%s" % (basename, extension)
+
+    def open_file(self, fname):
+        try:
+            cls = self._get_reader()
+            return cls(fname)
+        except IOError:
+            self.logger.error("Packages: Failed to read file %s" % fname)
+            raise
+
     @property
     def cachekey(self):
         """ A unique key for this source that will be used to generate
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 23ccd7b8e..1b53b1bb4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -14,6 +14,7 @@ from Bcfg2.Compat import urlopen, HTTPError, URLError
 from Bcfg2.Server.Plugins.Packages.Collection import Collection, \
     get_collection_class
 from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
+from Bcfg2.Server.Plugins.Packages.Readers import get_readers
 from Bcfg2.Server.Statistics import track_statistics
 
 
@@ -36,6 +37,12 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction):
     fail_silently = True
 
 
+class PackagesReadersAction(Bcfg2.Options.ComponentAction):
+    """ ComponentAction to load Packages readers """
+    bases = ['Bcfg2.Server.Plugins.Packages.Readers']
+    module = True
+
+
 class Packages(Bcfg2.Server.Plugin.Plugin,
                Bcfg2.Server.Plugin.StructureValidator,
                Bcfg2.Server.Plugin.Generator,
@@ -82,7 +89,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             cf=("packages", "apt_config"),
             help="The default path for generated apt configs",
             default="/etc/apt/sources.list.d/"
-            "bcfg2-packages-generated-sources.list")]
+            "bcfg2-packages-generated-sources.list"),
+        Bcfg2.Options.Option(
+            cf=("packages", "readers"), dest="packages_readers",
+            help="Packages readers to load",
+            type=Bcfg2.Options.Types.comma_list,
+            action=PackagesReadersAction,
+            default=get_readers())]
 
     #: Packages is an alternative to
     #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it.
-- 
cgit v1.2.3-1-g7c22


From 3842939e7d94373a8a4d4214072c9b24b1e32b6d Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <asulfrian@zedat.fu-berlin.de>
Date: Sun, 16 Jan 2022 08:40:09 +0100
Subject: Packages: Add "pyapt" source type

Pyapt is a new source that is using the apt python bindings to parse the
Packages files from debian repositories. Compared to the python implementation
it is faster and more robust. It will use the dependencies of the newest
version of a package from a specific source (because it can use the python
bindings to compare the version numbers).
---
 src/lib/Bcfg2/Server/Plugins/Packages/Apt.py      | 19 +++--
 src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py    | 93 +++++++++++++++++++++++
 src/lib/Bcfg2/Server/Plugins/Packages/__init__.py |  2 +-
 3 files changed, 105 insertions(+), 9 deletions(-)
 create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index cbbeb21eb..c3a3dc6ad 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -86,6 +86,16 @@ class AptSource(Source):
         else:
             return ["%s%s" % (self.rawurl, fname)]
 
+    def _get_arch(self, fname):
+        if not self.rawurl:
+            return [x
+                    for x in fname.split('@')
+                    if x.startswith('binary-')][0][7:]
+
+        # RawURL entries assume that they only have one <Arch></Arch>
+        # element and that it is the architecture of the source.
+        return self.arches[0]
+
     def read_files(self):  # pylint: disable=R0912
         bdeps = dict()
         brecs = dict()
@@ -93,14 +103,7 @@ class AptSource(Source):
         self.pkgnames = set()
         self.essentialpkgs = set()
         for fname in self.files:
-            if not self.rawurl:
-                barch = [x
-                         for x in fname.split('@')
-                         if x.startswith('binary-')][0][7:]
-            else:
-                # RawURL entries assume that they only have one <Arch></Arch>
-                # element and that it is the architecture of the source.
-                barch = self.arches[0]
+            barch = self._get_arch(fname)
             if barch not in bdeps:
                 bdeps[barch] = dict()
                 brecs[barch] = dict()
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py
new file mode 100644
index 000000000..5e0eb55be
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py
@@ -0,0 +1,93 @@
+"""
+APT backend for :mod:`Bcfg2.Server.Plugins.Packages` using
+apt_pkg python bindings.
+"""
+
+import apt_pkg
+from Bcfg2.Server.Plugins.Packages.Apt import AptCollection, AptSource
+
+
+class PyaptCollection(AptCollection):
+    """ Handle collections of PyAPT sources.  This is a no-op object
+    that simply inherits from
+    :class:`Bcfg2.Server.Plugins.Packages.Apt.AptCollection` and
+    overrides nothing.
+    """
+    pass
+
+
+class PyaptSource(AptSource):
+    """ Handle PyAPT sources """
+
+    def read_files(self):  # pylint: disable=R0912
+        bdeps = dict()
+        brecs = dict()
+        bprov = dict()
+        bvers = dict()
+        self.pkgnames = set()
+        self.essentialpkgs = set()
+        for fname in self.files:
+            barch = self._get_arch(fname)
+            if barch not in bdeps:
+                bdeps[barch] = dict()
+                brecs[barch] = dict()
+                bprov[barch] = dict()
+
+            apt_pkg.init_system()
+            with apt_pkg.TagFile(fname) as tagfile:
+                for section in tagfile:
+                    pkgname = section['Package']
+
+                    if pkgname in bvers:
+                        new = section['Version']
+                        old = bvers[pkgname]
+                        if apt_pkg.version_compare(new, old) <= 0:
+                            continue
+
+                    self.pkgnames.add(pkgname)
+                    bvers[pkgname] = section['Version']
+                    bdeps[barch][pkgname] = []
+                    brecs[barch][pkgname] = []
+
+                    if section.find_flag('Essential'):
+                        self.essentialpkgs.add(pkgname)
+
+                    for dep_type in ['Depends', 'Pre-Depends', 'Recommends']:
+                        dep_str = section.find(dep_type)
+                        if dep_str is None:
+                            continue
+
+                        vindex = 0
+                        for dep in apt_pkg.parse_depends(dep_str):
+                            if len(dep) > 1:
+                                cdeps = [cdep for (cdep, _, _) in dep]
+                                dyn_dname = "choice-%s-%s-%s" % (pkgname,
+                                                                 barch,
+                                                                 vindex)
+                                vindex += 1
+
+                                if dep_type == 'Recommends':
+                                    brecs[barch][pkgname].append(dyn_dname)
+                                else:
+                                    bdeps[barch][pkgname].append(dyn_dname)
+                                bprov[barch][dyn_dname] = set(cdeps)
+                            else:
+                                (raw_dep, _, _) = dep[0]
+                                if dep_type == 'Recommends':
+                                    brecs[barch][pkgname].append(raw_dep)
+                                else:
+                                    bdeps[barch][pkgname].append(raw_dep)
+
+                    provides = section.find('Provides')
+                    if provides is not None:
+                        provided_packages = [
+                                pkg
+                                for group in apt_pkg.parse_depends(provides)
+                                for (pkg, _, _) in group]
+                        for dname in provided_packages:
+                            if dname not in bprov[barch]:
+                                bprov[barch][dname] = set()
+                            bprov[barch][dname].add(pkgname)
+
+        self.process_files(bdeps, bprov, brecs)
+    read_files.__doc__ = AptSource.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 1b53b1bb4..fd9131db4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -64,7 +64,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             help="Packages backends to load",
             type=Bcfg2.Options.Types.comma_list,
             action=PackagesBackendAction,
-            default=['Yum', 'Apt', 'Pac', 'Pkgng']),
+            default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Pyapt']),
         Bcfg2.Options.PathOption(
             cf=("packages", "cache"), dest="packages_cache",
             help="Path to the Packages cache",
-- 
cgit v1.2.3-1-g7c22


From 830007663cf2d83ccaa58f9c6caa8f461f6d8234 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander@sulfrian.net>
Date: Thu, 1 Nov 2012 00:06:17 +0100
Subject: Packages: Add priority to sources and sort them

---
 src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 2 ++
 src/lib/Bcfg2/Server/Plugins/Packages/Source.py          | 9 +++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
index 1af046ec0..d982626d0 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
@@ -106,6 +106,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile):
                 source = self.source_from_xml(xsource)
                 if source is not None:
                     self.entries.append(source)
+            self.entries.sort(key=(lambda source: source.priority),
+                              reverse=True)
     Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """
 
         ``Index`` is responsible for calling :func:`source_from_xml`
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index c80604d01..b2422d84e 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -195,6 +195,9 @@ class Source(Debuggable):  # pylint: disable=R0902
         #: The "name" attribute from :attr:`xsource`
         self.name = None
 
+        #: The "priority" attribute from :attr:`xsource`
+        self.priority = xsource.get('priority', 500)
+
         #: A list of predicates that are used to determine if this
         #: source applies to a given
         #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
@@ -257,11 +260,13 @@ class Source(Debuggable):  # pylint: disable=R0902
         for arch in self.arches:
             if self.url:
                 usettings = [dict(version=self.version, component=comp,
-                                  arch=arch, debsrc=self.debsrc)
+                                  arch=arch, debsrc=self.debsrc,
+                                  priority=self.priority)
                              for comp in self.components]
             else:  # rawurl given
                 usettings = [dict(version=self.version, component=None,
-                                  arch=arch, debsrc=self.debsrc)]
+                                  arch=arch, debsrc=self.debsrc,
+                                  priority=self.priority)]
 
             for setting in usettings:
                 if not self.rawurl:
-- 
cgit v1.2.3-1-g7c22


From 18c024d69b373af53398157c658bcd2265d5d9c0 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander@sulfrian.net>
Date: Fri, 1 Mar 2013 01:07:05 +0100
Subject: Packages: Add 'pin' attribute

---
 src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index b2422d84e..2b7514b06 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -198,6 +198,9 @@ class Source(Debuggable):  # pylint: disable=R0902
         #: The "priority" attribute from :attr:`xsource`
         self.priority = xsource.get('priority', 500)
 
+        #: The "pin" attribute from :attr:`xsource`
+        self.pin = xsource.get('pin', '')
+
         #: A list of predicates that are used to determine if this
         #: source applies to a given
         #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
@@ -261,12 +264,12 @@ class Source(Debuggable):  # pylint: disable=R0902
             if self.url:
                 usettings = [dict(version=self.version, component=comp,
                                   arch=arch, debsrc=self.debsrc,
-                                  priority=self.priority)
+                                  priority=self.priority, pin=self.pin)
                              for comp in self.components]
             else:  # rawurl given
                 usettings = [dict(version=self.version, component=None,
                                   arch=arch, debsrc=self.debsrc,
-                                  priority=self.priority)]
+                                  priority=self.priority, pin=self.pin)]
 
             for setting in usettings:
                 if not self.rawurl:
-- 
cgit v1.2.3-1-g7c22


From 8d0683c366afc6c5c6db39ea722fef37daa5ea6d Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander@sulfrian.net>
Date: Tue, 10 May 2016 16:27:52 +0200
Subject: Packages: Add repo options to additional_data

---
 src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 2b7514b06..9c9c9c5bb 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -278,6 +278,8 @@ class Source(Debuggable):  # pylint: disable=R0902
                     setting['baseurl'] = self.rawurl
                 setting['url'] = baseurl % setting
                 setting['name'] = self.get_repo_name(setting)
+                setting['options'] = dict(server=self.server_options,
+                                          client=self.client_options)
             self.url_map.extend(usettings)
 
     def _init_attributes(self, xsource):
-- 
cgit v1.2.3-1-g7c22


From 9cef2774875594d91d73a4cbc3cd6935bc992cec Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <asulfrian@zedat.fu-berlin.de>
Date: Sun, 16 Jan 2022 19:59:25 +0100
Subject: Packages: Add possibility to customize User-Agent

Some mirror might block the default python urllib User-Agent.
---
 src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 9c9c9c5bb..b0c4bf44f 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -57,7 +57,7 @@ from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \
 from Bcfg2.Server.Statistics import track_statistics
 
 
-def fetch_url(url):
+def fetch_url(url, opts):
     """ Return the content of the given URL.
 
     :param url: The URL to fetch content from.
@@ -74,8 +74,14 @@ def fetch_url(url):
         url = mobj.group(1) + mobj.group(4)
         auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
         auth.add_password(None, url, user, passwd)
-        install_opener(build_opener(auth))
-    return urlopen(url).read()
+        req = build_opener(auth)
+    else:
+        req = build_opener()
+
+    if 'user-agent' in opts:
+        req.addheaders = [('User-Agent', opts['user-agent'])]
+
+    return req.open(url).read()
 
 
 class SourceInitError(Exception):
@@ -735,7 +741,7 @@ class Source(Debuggable):  # pylint: disable=R0902
             self.logger.info("Packages: Updating %s" % url)
             fname = self.escape_url(url)
             try:
-                open(fname, 'wb').write(fetch_url(url))
+                open(fname, 'wb').write(fetch_url(url, self.server_options))
             except ValueError:
                 self.logger.error("Packages: Bad url string %s" % url)
                 raise
-- 
cgit v1.2.3-1-g7c22


From c22cd2a2fc8b909c08b1d8df8d5cc3909e8aeccc Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <asulfrian@zedat.fu-berlin.de>
Date: Sun, 16 Jan 2022 21:24:24 +0100
Subject: Packages: Allow to filter provided packages

The provides packages of a source should be filtered by the Blacklist or
Whitelist tags for the source.
---
 src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index b0c4bf44f..574dbd851 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -589,6 +589,9 @@ class Source(Debuggable):  # pylint: disable=R0902
                 self.logger.warning("%s provides no packages for %s" %
                                     (self, agrp))
                 continue
+            if (agrp in self.blacklist or
+                    (len(self.whitelist) != 0 and agrp not in self.whitelist)):
+                continue
             for key, value in list(self.provides[agrp].items()):
                 if key not in vdict:
                     vdict[key] = set(value)
@@ -815,7 +818,9 @@ class Source(Debuggable):  # pylint: disable=R0902
         :returns: list of strings
         """
         for arch in self.get_arches(metadata):
-            if package in self.provides[arch]:
+            if (package in self.provides[arch] and
+                    package not in self.blacklist and
+                    (len(self.whitelist) == 0 or package in self.whitelist)):
                 return self.provides[arch][package]
         return []
 
-- 
cgit v1.2.3-1-g7c22


From 4e1a98aec6afa273bda82e3f840d3ed27e42b3be Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander@sulfrian.net>
Date: Tue, 11 Jun 2013 10:10:41 +0200
Subject: PkgVars: Add new plugin to set vars for packages

This plugins allows the setting of varius flags per package. It should
be used f.e. to specify pinnings for debian packages.
---
 src/lib/Bcfg2/Server/Lint/Validate.py              |  3 +-
 .../Bcfg2/Server/Plugins/Packages/Collection.py    | 27 ++++++++--
 src/lib/Bcfg2/Server/Plugins/Packages/Yum.py       |  5 +-
 src/lib/Bcfg2/Server/Plugins/Packages/__init__.py  |  8 ++-
 src/lib/Bcfg2/Server/Plugins/PkgVars.py            | 59 ++++++++++++++++++++++
 5 files changed, 94 insertions(+), 8 deletions(-)
 create mode 100644 src/lib/Bcfg2/Server/Plugins/PkgVars.py

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index d6f18afbb..146f18b0c 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -61,7 +61,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
              "AWSTags/config.xml": "awstags.xsd",
              "NagiosGen/config.xml": "nagiosgen.xsd",
              "FileProbes/config.xml": "fileprobes.xsd",
-             "GroupLogic/groups.xml": "grouplogic.xsd"
+             "GroupLogic/groups.xml": "grouplogic.xsd",
+             "PkgVars/*.xml": "pkgvars.xsd"
              }
 
         self.filelists = {}
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index 004e27874..ccb526a19 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -289,7 +289,7 @@ class Collection(list, Debuggable):
         return any(source.is_virtual_package(self.metadata, package)
                    for source in self)
 
-    def get_deps(self, package, recs=None):
+    def get_deps(self, package, recs=None, pinnings=None):
         """ Get a list of the dependencies of the given package.
 
         The base implementation simply aggregates the results of
@@ -297,16 +297,35 @@ class Collection(list, Debuggable):
 
         :param package: The name of the symbol, but see :ref:`pkg-objects`
         :type package: string
+        :param pinnings: Mapping from package names to source names.
+        :type pinnings: dict
         :returns: list of strings, but see :ref:`pkg-objects`
         """
         recommended = None
         if recs and package in recs:
             recommended = recs[package]
 
+        pin_found = False
+        pin_source = None
+        if pinnings and package in pinnings:
+            pin_source = pinnings[package]
+
         for source in self:
+            if pin_source and pin_source != source.name:
+                continue
+            pin_found = True
+
             if source.is_package(self.metadata, package):
                 return source.get_deps(self.metadata, package, recommended)
 
+        if not pin_found:
+            if pin_source:
+                self.logger.error("Packages: Source '%s' for package '%s' not found" %
+                                  (pin_source, package))
+            else:
+                self.logger.error("Packages: No source found for package '%s'" %
+                                  package);
+
         return []
 
     def get_essential(self):
@@ -471,12 +490,14 @@ class Collection(list, Debuggable):
 
     @track_statistics()
     def complete(self, packagelist,  # pylint: disable=R0912,R0914
-                 recommended=None):
+                 recommended=None, pinnings=None):
         """ Build a complete list of all packages and their dependencies.
 
         :param packagelist: Set of initial packages computed from the
                             specification.
         :type packagelist: set of strings, but see :ref:`pkg-objects`
+        :param pinnings: Mapping from package names to source names.
+        :type pinnings: dict
         :returns: tuple of sets - The first element contains a set of
                   strings (but see :ref:`pkg-objects`) describing the
                   complete package list, and the second element is a
@@ -535,7 +556,7 @@ class Collection(list, Debuggable):
                 self.debug_log("Packages: handling package requirement %s" %
                                (current,))
                 packages.add(current)
-                deps = self.get_deps(current, recommended)
+                deps = self.get_deps(current, recommended, pinnings)
                 newdeps = set(deps).difference(examined)
                 if newdeps:
                     self.debug_log("Packages: Package %s added requirements %s"
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 846fb89cd..acb11f1ab 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -850,7 +850,7 @@ class YumCollection(Collection):
         return new
 
     @track_statistics()
-    def complete(self, packagelist, recommended=None):
+    def complete(self, packagelist, recommended=None, pinnings=None):
         """ Build a complete list of all packages and their dependencies.
 
         When using the Python yum libraries, this defers to the
@@ -868,7 +868,8 @@ class YumCollection(Collection):
                   resolved.
         """
         if not self.use_yum:
-            return Collection.complete(self, packagelist, recommended)
+            return Collection.complete(self, packagelist, recommended,
+                                       pinnings)
 
         lock = FileLock(os.path.join(self.cachefile, "lock"))
         slept = 0
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index fd9131db4..d888af965 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -328,6 +328,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         groups = []
         recommended = dict()
 
+        pinned_src = dict()
+        if hasattr(metadata, 'PkgVars'):
+            pinned_src = metadata.PkgVars['pin']
+
         for struct in structures:
             for pkg in struct.xpath('//Package | //BoundPackage'):
                 if pkg.get("name"):
@@ -369,11 +373,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         base.update(collection.get_essential())
 
         # check for this set of packages in the package cache
-        pkey = hash(tuple(base))
+        pkey = hash((tuple(base), tuple(recommended), tuple(pinned_src)))
         pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets",
                                           collection.cachekey)
         if pkey not in pcache:
-            pcache[pkey] = collection.complete(base, recommended)
+            pcache[pkey] = collection.complete(base, recommended, pinned_src)
         packages, unknown = pcache[pkey]
         if unknown:
             self.logger.info("Packages: Got %d unknown entries" % len(unknown))
diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
new file mode 100644
index 000000000..a085ea17e
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
@@ -0,0 +1,59 @@
+import os
+import re
+import sys
+import copy
+import logging
+import lxml.etree
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger('Bcfg2.Plugins.PkgVars')
+vars = ['pin', 'use', 'keywords']
+
+class PkgVarsFile(Bcfg2.Server.Plugin.StructFile):
+    def get_additional_data(self, meta):
+        data = self.Match(meta)
+        results = {}
+        for d in data:
+            name = d.get('name', '')
+
+            for v in vars:
+                value = d.get(v, None)
+                if value:
+                    results[v][name] = value
+
+        return results
+
+class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
+    __child__ = PkgVarsFile
+    patterns = re.compile(r'.*\.xml$')
+
+    def get_additional_data(self, meta):
+        results = {}
+        for v in vars:
+            results[v] = {}
+
+        for files in self.entries:
+            new = self.entries[files].get_additional_data(meta)
+            for x in vars:
+                results[x].update(new[x])
+
+        return results
+
+class PkgVars(Bcfg2.Server.Plugin.Plugin,
+              Bcfg2.Server.Plugin.Connector):
+    name = 'PkgVars'
+    version = '$Revision$'
+
+    def __init__(self, core):
+        Bcfg2.Server.Plugin.Plugin.__init__(self, core)
+        Bcfg2.Server.Plugin.Connector.__init__(self)
+        try:
+            self.store = PkgVarsDirectoryBacked(self.data)
+        except OSError:
+            e = sys.exc_info()[1]
+            self.logger.error("Error while creating PkgVars store: %s %s" %
+                              (e.strerror, e.filename))
+            raise Bcfg2.Server.Plugin.PluginInitError
+
+    def get_additional_data(self, meta):
+        return self.store.get_additional_data(meta)
-- 
cgit v1.2.3-1-g7c22


From ef568d29698c00bf2c02c99a34b98e6b8ca96653 Mon Sep 17 00:00:00 2001
From: Alexander Sulfrian <alexander@sulfrian.net>
Date: Sun, 10 Mar 2013 23:07:22 +0100
Subject: PkgVars: Add support for multiple values

If multiple values specified for one package all values are joined
together in a set.
---
 src/lib/Bcfg2/Server/Plugins/Packages/Collection.py |  4 ++--
 src/lib/Bcfg2/Server/Plugins/PkgVars.py             | 10 ++++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

(limited to 'src/lib')

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index ccb526a19..e0d6e1fc3 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -311,7 +311,7 @@ class Collection(list, Debuggable):
             pin_source = pinnings[package]
 
         for source in self:
-            if pin_source and pin_source != source.name:
+            if pin_source and source.name not in pin_source:
                 continue
             pin_found = True
 
@@ -321,7 +321,7 @@ class Collection(list, Debuggable):
         if not pin_found:
             if pin_source:
                 self.logger.error("Packages: Source '%s' for package '%s' not found" %
-                                  (pin_source, package))
+                                  (' or '.join(pin_source), package))
             else:
                 self.logger.error("Packages: No source found for package '%s'" %
                                   package);
diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
index a085ea17e..9a2649d02 100644
--- a/src/lib/Bcfg2/Server/Plugins/PkgVars.py
+++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
@@ -19,7 +19,12 @@ class PkgVarsFile(Bcfg2.Server.Plugin.StructFile):
             for v in vars:
                 value = d.get(v, None)
                 if value:
-                    results[v][name] = value
+                    if v not in results:
+                        results[v] = {}
+                    if name not in results[v]:
+                        results[v][name] = set()
+
+                    results[v][name].add(value)
 
         return results
 
@@ -35,7 +40,8 @@ class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
         for files in self.entries:
             new = self.entries[files].get_additional_data(meta)
             for x in vars:
-                results[x].update(new[x])
+                if x in new:
+                    results[x].update(new[x])
 
         return results
 
-- 
cgit v1.2.3-1-g7c22