Packages: Add "pyapt" source type

Pyapt is a new source that is using the apt python bindings to parse the
Packages files from debian repositories. Compared to the python implementation
it is faster and more robust. It will use the dependencies of the newest
version of a package from a specific source (because it can use the python
bindings to compare the version numbers).

3 files changed, 105 insertions, 9 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index cbbeb21eb..c3a3dc6ad 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -86,6 +86,16 @@ class AptSource(Source):
         else:
             return ["%s%s" % (self.rawurl, fname)]
 
+    def _get_arch(self, fname):
+        if not self.rawurl:
+            return [x
+                    for x in fname.split('@')
+                    if x.startswith('binary-')][0][7:]
+
+        # RawURL entries assume that they only have one <Arch></Arch>
+        # element and that it is the architecture of the source.
+        return self.arches[0]
+
     def read_files(self):  # pylint: disable=R0912
         bdeps = dict()
         brecs = dict()
@@ -93,14 +103,7 @@ class AptSource(Source):
         self.pkgnames = set()
         self.essentialpkgs = set()
         for fname in self.files:
-            if not self.rawurl:
-                barch = [x
-                         for x in fname.split('@')
-                         if x.startswith('binary-')][0][7:]
-            else:
-                # RawURL entries assume that they only have one <Arch></Arch>
-                # element and that it is the architecture of the source.
-                barch = self.arches[0]
+            barch = self._get_arch(fname)
             if barch not in bdeps:
                 bdeps[barch] = dict()
                 brecs[barch] = dict()
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py
new file mode 100644
index 000000000..5e0eb55be
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py
@@ -0,0 +1,93 @@
+"""
+APT backend for :mod:`Bcfg2.Server.Plugins.Packages` using
+apt_pkg python bindings.
+"""
+
+import apt_pkg
+from Bcfg2.Server.Plugins.Packages.Apt import AptCollection, AptSource
+
+
+class PyaptCollection(AptCollection):
+    """ Handle collections of PyAPT sources.  This is a no-op object
+    that simply inherits from
+    :class:`Bcfg2.Server.Plugins.Packages.Apt.AptCollection` and
+    overrides nothing.
+    """
+    pass
+
+
+class PyaptSource(AptSource):
+    """ Handle PyAPT sources """
+
+    def read_files(self):  # pylint: disable=R0912
+        bdeps = dict()
+        brecs = dict()
+        bprov = dict()
+        bvers = dict()
+        self.pkgnames = set()
+        self.essentialpkgs = set()
+        for fname in self.files:
+            barch = self._get_arch(fname)
+            if barch not in bdeps:
+                bdeps[barch] = dict()
+                brecs[barch] = dict()
+                bprov[barch] = dict()
+
+            apt_pkg.init_system()
+            with apt_pkg.TagFile(fname) as tagfile:
+                for section in tagfile:
+                    pkgname = section['Package']
+
+                    if pkgname in bvers:
+                        new = section['Version']
+                        old = bvers[pkgname]
+                        if apt_pkg.version_compare(new, old) <= 0:
+                            continue
+
+                    self.pkgnames.add(pkgname)
+                    bvers[pkgname] = section['Version']
+                    bdeps[barch][pkgname] = []
+                    brecs[barch][pkgname] = []
+
+                    if section.find_flag('Essential'):
+                        self.essentialpkgs.add(pkgname)
+
+                    for dep_type in ['Depends', 'Pre-Depends', 'Recommends']:
+                        dep_str = section.find(dep_type)
+                        if dep_str is None:
+                            continue
+
+                        vindex = 0
+                        for dep in apt_pkg.parse_depends(dep_str):
+                            if len(dep) > 1:
+                                cdeps = [cdep for (cdep, _, _) in dep]
+                                dyn_dname = "choice-%s-%s-%s" % (pkgname,
+                                                                 barch,
+                                                                 vindex)
+                                vindex += 1
+
+                                if dep_type == 'Recommends':
+                                    brecs[barch][pkgname].append(dyn_dname)
+                                else:
+                                    bdeps[barch][pkgname].append(dyn_dname)
+                                bprov[barch][dyn_dname] = set(cdeps)
+                            else:
+                                (raw_dep, _, _) = dep[0]
+                                if dep_type == 'Recommends':
+                                    brecs[barch][pkgname].append(raw_dep)
+                                else:
+                                    bdeps[barch][pkgname].append(raw_dep)
+
+                    provides = section.find('Provides')
+                    if provides is not None:
+                        provided_packages = [
+                                pkg
+                                for group in apt_pkg.parse_depends(provides)
+                                for (pkg, _, _) in group]
+                        for dname in provided_packages:
+                            if dname not in bprov[barch]:
+                                bprov[barch][dname] = set()
+                            bprov[barch][dname].add(pkgname)
+
+        self.process_files(bdeps, bprov, brecs)
+    read_files.__doc__ = AptSource.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 1b53b1bb4..fd9131db4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -64,7 +64,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             help="Packages backends to load",
             type=Bcfg2.Options.Types.comma_list,
             action=PackagesBackendAction,
-            default=['Yum', 'Apt', 'Pac', 'Pkgng']),
+            default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Pyapt']),
         Bcfg2.Options.PathOption(
             cf=("packages", "cache"), dest="packages_cache",
             help="Path to the Packages cache",