From 3842939e7d94373a8a4d4214072c9b24b1e32b6d Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 16 Jan 2022 08:40:09 +0100 Subject: Packages: Add "pyapt" source type Pyapt is a new source that is using the apt python bindings to parse the Packages files from debian repositories. Compared to the python implementation it is faster and more robust. It will use the dependencies of the newest version of a package from a specific source (because it can use the python bindings to compare the version numbers). --- src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 19 +++-- src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py | 93 +++++++++++++++++++++++ src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 +- 3 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index cbbeb21eb..c3a3dc6ad 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -86,6 +86,16 @@ class AptSource(Source): else: return ["%s%s" % (self.rawurl, fname)] + def _get_arch(self, fname): + if not self.rawurl: + return [x + for x in fname.split('@') + if x.startswith('binary-')][0][7:] + + # RawURL entries assume that they only have one + # element and that it is the architecture of the source. + return self.arches[0] + def read_files(self): # pylint: disable=R0912 bdeps = dict() brecs = dict() @@ -93,14 +103,7 @@ class AptSource(Source): self.pkgnames = set() self.essentialpkgs = set() for fname in self.files: - if not self.rawurl: - barch = [x - for x in fname.split('@') - if x.startswith('binary-')][0][7:] - else: - # RawURL entries assume that they only have one - # element and that it is the architecture of the source. - barch = self.arches[0] + barch = self._get_arch(fname) if barch not in bdeps: bdeps[barch] = dict() brecs[barch] = dict() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py new file mode 100644 index 000000000..5e0eb55be --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py @@ -0,0 +1,93 @@ +""" +APT backend for :mod:`Bcfg2.Server.Plugins.Packages` using +apt_pkg python bindings. +""" + +import apt_pkg +from Bcfg2.Server.Plugins.Packages.Apt import AptCollection, AptSource + + +class PyaptCollection(AptCollection): + """ Handle collections of PyAPT sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Apt.AptCollection` and + overrides nothing. + """ + pass + + +class PyaptSource(AptSource): + """ Handle PyAPT sources """ + + def read_files(self): # pylint: disable=R0912 + bdeps = dict() + brecs = dict() + bprov = dict() + bvers = dict() + self.pkgnames = set() + self.essentialpkgs = set() + for fname in self.files: + barch = self._get_arch(fname) + if barch not in bdeps: + bdeps[barch] = dict() + brecs[barch] = dict() + bprov[barch] = dict() + + apt_pkg.init_system() + with apt_pkg.TagFile(fname) as tagfile: + for section in tagfile: + pkgname = section['Package'] + + if pkgname in bvers: + new = section['Version'] + old = bvers[pkgname] + if apt_pkg.version_compare(new, old) <= 0: + continue + + self.pkgnames.add(pkgname) + bvers[pkgname] = section['Version'] + bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] + + if section.find_flag('Essential'): + self.essentialpkgs.add(pkgname) + + for dep_type in ['Depends', 'Pre-Depends', 'Recommends']: + dep_str = section.find(dep_type) + if dep_str is None: + continue + + vindex = 0 + for dep in apt_pkg.parse_depends(dep_str): + if len(dep) > 1: + cdeps = [cdep for (cdep, _, _) in dep] + dyn_dname = "choice-%s-%s-%s" % (pkgname, + barch, + vindex) + vindex += 1 + + if dep_type == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) + bprov[barch][dyn_dname] = set(cdeps) + else: + (raw_dep, _, _) = dep[0] + if dep_type == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) + + provides = section.find('Provides') + if provides is not None: + provided_packages = [ + pkg + for group in apt_pkg.parse_depends(provides) + for (pkg, _, _) in group] + for dname in provided_packages: + if dname not in bprov[barch]: + bprov[barch][dname] = set() + bprov[barch][dname].add(pkgname) + + self.process_files(bdeps, bprov, brecs) + read_files.__doc__ = AptSource.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 1b53b1bb4..fd9131db4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -64,7 +64,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, action=PackagesBackendAction, - default=['Yum', 'Apt', 'Pac', 'Pkgng']), + default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Pyapt']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", -- cgit v1.2.3-1-g7c22