Merge branch 'packages'

19 files changed, 471 insertions, 44 deletions

diff --git a/schemas/packages.xsd b/schemas/packages.xsd
index 849be41e2..4a9881a8f 100644
--- a/schemas/packages.xsd
+++ b/schemas/packages.xsd
@@ -226,6 +226,25 @@
         </xsd:documentation>
       </xsd:annotation>
     </xsd:attribute>
+    <xsd:attribute type="xsd:integer" name="priority">
+      <xsd:annotation>
+        <xsd:documentation>
+          The priority of the source. This is used to order the
+          sources. After sorting, the first source, that could
+          deliver the package, is used. If not supplied the default
+          priority is 500.
+        </xsd:documentation>
+      </xsd:annotation>
+    </xsd:attribute>
+    <xsd:attribute type="xsd:string" name="pin">
+      <xsd:annotation>
+        <xsd:documentation>
+          Extra information for pinning. This information is used
+          to differ between the sources. Should be used in the
+          supported format of apt.
+        </xsd:documentation>
+      </xsd:annotation>
+    </xsd:attribute>
     <xsd:attributeGroup ref="py:genshiAttrs"/>
   </xsd:complexType>
 
diff --git a/schemas/pkgvars.xsd b/schemas/pkgvars.xsd
new file mode 100644
index 000000000..dbd02726d
--- /dev/null
+++ b/schemas/pkgvars.xsd
@@ -0,0 +1,43 @@
+<xsd:schema xmlns:xsd='http://www.w3.org/2001/XMLSchema'
+            xmlns:py="http://genshi.edgewall.org/">
+
+  <xsd:annotation>
+    <xsd:documentation>
+      XML-Schema-Definition für PkgVars/*.xml
+      Alexander Sulfrian
+    </xsd:documentation>
+  </xsd:annotation>
+
+  <xsd:import namespace="http://genshi.edgewall.org/"
+              schemaLocation="genshi.xsd"/>
+
+  <xsd:complexType name='pkgVarType'>
+    <xsd:attribute type='xsd:string' name='name'/>
+
+    <xsd:attribute type='xsd:string' name='pin'/>
+    <xsd:attribute type='xsd:string' name='use'/>
+    <xsd:attribute type='xsd:string' name='keywords'/>
+
+    <xsd:attributeGroup ref="py:genshiAttrs"/>
+  </xsd:complexType>
+
+  <xsd:complexType name='containerType'>
+    <xsd:choice maxOccurs='unbounded'>
+      <xsd:element name='Package' type='pkgVarType'/>
+      <xsd:element name='Client' type='containerType'/>
+      <xsd:element name='Group' type='containerType'/>
+    </xsd:choice>
+    <xsd:attribute name='name' type='xsd:string' use='required'/>
+    <xsd:attribute name='negate' type='xsd:boolean'/>
+  </xsd:complexType>
+
+  <xsd:complexType name='pkgVarsType'>
+    <xsd:choice minOccurs='0' maxOccurs='unbounded'>
+      <xsd:element name='Package' type='pkgVarType'/>
+      <xsd:element name='Client' type='containerType'/>
+      <xsd:element name='Group' type='containerType'/>
+    </xsd:choice>
+  </xsd:complexType>
+
+  <xsd:element name='PkgVars' type='pkgVarsType'/>
+</xsd:schema>
diff --git a/setup.py b/setup.py
index 5ff2ac003..383202490 100755
--- a/setup.py
+++ b/setup.py
@@ -47,6 +47,7 @@ setup(name="Bcfg2",
                 "Bcfg2.Server.Plugin",
                 "Bcfg2.Server.Plugins",
                 "Bcfg2.Server.Plugins.Packages",
+                "Bcfg2.Server.Plugins.Packages.Readers",
                 "Bcfg2.Server.Plugins.Cfg",
                 "Bcfg2.Server.Reports",
                 "Bcfg2.Server.Reports.reports",
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index d6f18afbb..146f18b0c 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -61,7 +61,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
              "AWSTags/config.xml": "awstags.xsd",
              "NagiosGen/config.xml": "nagiosgen.xsd",
              "FileProbes/config.xml": "fileprobes.xsd",
-             "GroupLogic/groups.xml": "grouplogic.xsd"
+             "GroupLogic/groups.xml": "grouplogic.xsd",
+             "PkgVars/*.xml": "pkgvars.xsd"
              }
 
         self.filelists = {}
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 956cb9f51..c3a3dc6ad 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -1,7 +1,6 @@
 """ APT backend for :mod:`Bcfg2.Server.Plugins.Packages` """
 
 import re
-import gzip
 from Bcfg2.Server.Plugins.Packages.Collection import Collection
 from Bcfg2.Server.Plugins.Packages.Source import Source
 
@@ -68,19 +67,34 @@ class AptSource(Source):
     #: AptSource sets the ``type`` on Package entries to "deb"
     ptype = 'deb'
 
+    #: Most (3rd-party) debian repositories still only support "gzip".
+    default_compression = 'gzip'
+
     @property
     def urls(self):
         """ A list of URLs to the base metadata file for each
         repository described by this source. """
+        fname = self.build_filename('Packages')
+
         if not self.rawurl:
             rv = []
             for part in self.components:
                 for arch in self.arches:
-                    rv.append("%sdists/%s/%s/binary-%s/Packages.gz" %
-                              (self.url, self.version, part, arch))
+                    rv.append("%sdists/%s/%s/binary-%s/%s" %
+                              (self.url, self.version, part, arch, fname))
             return rv
         else:
-            return ["%sPackages.gz" % self.rawurl]
+            return ["%s%s" % (self.rawurl, fname)]
+
+    def _get_arch(self, fname):
+        if not self.rawurl:
+            return [x
+                    for x in fname.split('@')
+                    if x.startswith('binary-')][0][7:]
+
+        # RawURL entries assume that they only have one <Arch></Arch>
+        # element and that it is the architecture of the source.
+        return self.arches[0]
 
     def read_files(self):  # pylint: disable=R0912
         bdeps = dict()
@@ -89,23 +103,13 @@ class AptSource(Source):
         self.pkgnames = set()
         self.essentialpkgs = set()
         for fname in self.files:
-            if not self.rawurl:
-                barch = [x
-                         for x in fname.split('@')
-                         if x.startswith('binary-')][0][7:]
-            else:
-                # RawURL entries assume that they only have one <Arch></Arch>
-                # element and that it is the architecture of the source.
-                barch = self.arches[0]
+            barch = self._get_arch(fname)
             if barch not in bdeps:
                 bdeps[barch] = dict()
                 brecs[barch] = dict()
                 bprov[barch] = dict()
-            try:
-                reader = gzip.GzipFile(fname)
-            except IOError:
-                self.logger.error("Packages: Failed to read file %s" % fname)
-                raise
+
+            reader = self.open_file(fname)
             for line in reader.readlines():
                 if not isinstance(line, str):
                     line = line.decode('utf-8')
@@ -150,5 +154,6 @@ class AptSource(Source):
                         if dname not in bprov[barch]:
                             bprov[barch][dname] = set()
                         bprov[barch][dname].add(pkgname)
+            reader.close()
         self.process_files(bdeps, bprov, brecs)
     read_files.__doc__ = Source.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index 004e27874..e0d6e1fc3 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -289,7 +289,7 @@ class Collection(list, Debuggable):
         return any(source.is_virtual_package(self.metadata, package)
                    for source in self)
 
-    def get_deps(self, package, recs=None):
+    def get_deps(self, package, recs=None, pinnings=None):
         """ Get a list of the dependencies of the given package.
 
         The base implementation simply aggregates the results of
@@ -297,16 +297,35 @@ class Collection(list, Debuggable):
 
         :param package: The name of the symbol, but see :ref:`pkg-objects`
         :type package: string
+        :param pinnings: Mapping from package names to source names.
+        :type pinnings: dict
         :returns: list of strings, but see :ref:`pkg-objects`
         """
         recommended = None
         if recs and package in recs:
             recommended = recs[package]
 
+        pin_found = False
+        pin_source = None
+        if pinnings and package in pinnings:
+            pin_source = pinnings[package]
+
         for source in self:
+            if pin_source and source.name not in pin_source:
+                continue
+            pin_found = True
+
             if source.is_package(self.metadata, package):
                 return source.get_deps(self.metadata, package, recommended)
 
+        if not pin_found:
+            if pin_source:
+                self.logger.error("Packages: Source '%s' for package '%s' not found" %
+                                  (' or '.join(pin_source), package))
+            else:
+                self.logger.error("Packages: No source found for package '%s'" %
+                                  package);
+
         return []
 
     def get_essential(self):
@@ -471,12 +490,14 @@ class Collection(list, Debuggable):
 
     @track_statistics()
     def complete(self, packagelist,  # pylint: disable=R0912,R0914
-                 recommended=None):
+                 recommended=None, pinnings=None):
         """ Build a complete list of all packages and their dependencies.
 
         :param packagelist: Set of initial packages computed from the
                             specification.
         :type packagelist: set of strings, but see :ref:`pkg-objects`
+        :param pinnings: Mapping from package names to source names.
+        :type pinnings: dict
         :returns: tuple of sets - The first element contains a set of
                   strings (but see :ref:`pkg-objects`) describing the
                   complete package list, and the second element is a
@@ -535,7 +556,7 @@ class Collection(list, Debuggable):
                 self.debug_log("Packages: handling package requirement %s" %
                                (current,))
                 packages.add(current)
-                deps = self.get_deps(current, recommended)
+                deps = self.get_deps(current, recommended, pinnings)
                 newdeps = set(deps).difference(examined)
                 if newdeps:
                     self.debug_log("Packages: Package %s added requirements %s"
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
index 6fc084cc4..e3432c934 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
@@ -87,6 +87,9 @@ class PacSource(Source):
     #: PacSource sets the ``type`` on Package entries to "pacman"
     ptype = 'pacman'
 
+    #: The database of pacman repositories is compressed with "gzip"
+    default_compression = 'gzip'
+
     def __init__(self, basepath, xsource):
         self.pacgroups = {}
 
@@ -113,9 +116,10 @@ class PacSource(Source):
         if not self.rawurl:
             rv = []
             for part in self.components:
+                filename = self.build_filename("%s.db.tar" % part)
                 for arch in self.arches:
-                    rv.append("%s%s/os/%s/%s.db.tar.gz" %
-                              (self.url, part, arch, part))
+                    rv.append("%s%s/os/%s/%s" %
+                              (self.url, part, arch, filename))
             return rv
         else:
             raise Exception("PacSource : RAWUrl not supported (yet)")
@@ -140,7 +144,8 @@ class PacSource(Source):
                 bprov[barch] = {}
             try:
                 self.debug_log("Packages: try to read %s" % fname)
-                tar = tarfile.open(fname, "r")
+                reader = self.open_file(fname)
+                tar = tarfile.open(fileobj=reader)
             except (IOError, tarfile.TarError):
                 self.logger.error("Packages: Failed to read file %s" % fname)
                 raise
@@ -185,6 +190,7 @@ class PacSource(Source):
                         self.pacgroups[group].append(pkgname)
 
             tar.close()
+            reader.close()
         self.process_files(bdeps, bprov, brecs)
     read_files.__doc__ = Source.read_files.__doc__
 
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
index 1af046ec0..d982626d0 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
@@ -106,6 +106,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile):
                 source = self.source_from_xml(xsource)
                 if source is not None:
                     self.entries.append(source)
+            self.entries.sort(key=(lambda source: source.priority),
+                              reverse=True)
     Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """
 
         ``Index`` is responsible for calling :func:`source_from_xml`
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
index 55dd4e488..5248ad896 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py
@@ -1,6 +1,5 @@
 """ pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """
 
-import lzma
 import tarfile
 
 try:
@@ -40,19 +39,30 @@ class PkgngSource(Source):
     #: PkgngSource sets the ``type`` on Package entries to "pkgng"
     ptype = 'pkgng'
 
+    #: The "packagesite" files of pkgng repositories are compressed
+    #: with "xz"
+    default_compression = 'xz'
+
+    def _get_extension(self):
+        extension = super(PkgngSource, self)._get_extension()
+        if extension == '':
+            return 'tar'
+        return 't%s' % extension
+
     @property
     def urls(self):
         """ A list of URLs to the base metadata file for each
         repository described by this source. """
+        fname = self.build_filename("packagesite")
         if not self.rawurl:
             rv = []
             for part in self.components:
                 for arch in self.arches:
-                    rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" %
-                              (self.url, self.version, arch, part))
+                    rv.append("%s/freebsd:%s:%s/%s/%s" %
+                              (self.url, self.version, arch, part, fname))
             return rv
         else:
-            return ["%s/packagesite.txz" % self.rawurl]
+            return ["%s/%s" % (self.rawurl, fname)]
 
     def read_files(self):
         bdeps = dict()
@@ -70,12 +80,13 @@ class PkgngSource(Source):
             if barch not in bdeps:
                 bdeps[barch] = dict()
             try:
-                tar = tarfile.open(fileobj=lzma.LZMAFile(fname))
-                reader = tar.extractfile('packagesite.yaml')
+                reader = self.open_file(fname)
+                tar = tarfile.open(fileobj=reader)
+                packagesite = tar.extractfile('packagesite.yaml')
             except (IOError, tarfile.TarError):
                 self.logger.error("Packages: Failed to read file %s" % fname)
                 raise
-            for line in reader.readlines():
+            for line in packagesite.readlines():
                 pkg = json.loads(unicode(line, errors='ignore'))
                 pkgname = pkg['name']
                 self.pkgnames.add(pkgname)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py
new file mode 100644
index 000000000..5e0eb55be
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py
@@ -0,0 +1,93 @@
+"""
+APT backend for :mod:`Bcfg2.Server.Plugins.Packages` using
+apt_pkg python bindings.
+"""
+
+import apt_pkg
+from Bcfg2.Server.Plugins.Packages.Apt import AptCollection, AptSource
+
+
+class PyaptCollection(AptCollection):
+    """ Handle collections of PyAPT sources.  This is a no-op object
+    that simply inherits from
+    :class:`Bcfg2.Server.Plugins.Packages.Apt.AptCollection` and
+    overrides nothing.
+    """
+    pass
+
+
+class PyaptSource(AptSource):
+    """ Handle PyAPT sources """
+
+    def read_files(self):  # pylint: disable=R0912
+        bdeps = dict()
+        brecs = dict()
+        bprov = dict()
+        bvers = dict()
+        self.pkgnames = set()
+        self.essentialpkgs = set()
+        for fname in self.files:
+            barch = self._get_arch(fname)
+            if barch not in bdeps:
+                bdeps[barch] = dict()
+                brecs[barch] = dict()
+                bprov[barch] = dict()
+
+            apt_pkg.init_system()
+            with apt_pkg.TagFile(fname) as tagfile:
+                for section in tagfile:
+                    pkgname = section['Package']
+
+                    if pkgname in bvers:
+                        new = section['Version']
+                        old = bvers[pkgname]
+                        if apt_pkg.version_compare(new, old) <= 0:
+                            continue
+
+                    self.pkgnames.add(pkgname)
+                    bvers[pkgname] = section['Version']
+                    bdeps[barch][pkgname] = []
+                    brecs[barch][pkgname] = []
+
+                    if section.find_flag('Essential'):
+                        self.essentialpkgs.add(pkgname)
+
+                    for dep_type in ['Depends', 'Pre-Depends', 'Recommends']:
+                        dep_str = section.find(dep_type)
+                        if dep_str is None:
+                            continue
+
+                        vindex = 0
+                        for dep in apt_pkg.parse_depends(dep_str):
+                            if len(dep) > 1:
+                                cdeps = [cdep for (cdep, _, _) in dep]
+                                dyn_dname = "choice-%s-%s-%s" % (pkgname,
+                                                                 barch,
+                                                                 vindex)
+                                vindex += 1
+
+                                if dep_type == 'Recommends':
+                                    brecs[barch][pkgname].append(dyn_dname)
+                                else:
+                                    bdeps[barch][pkgname].append(dyn_dname)
+                                bprov[barch][dyn_dname] = set(cdeps)
+                            else:
+                                (raw_dep, _, _) = dep[0]
+                                if dep_type == 'Recommends':
+                                    brecs[barch][pkgname].append(raw_dep)
+                                else:
+                                    bdeps[barch][pkgname].append(raw_dep)
+
+                    provides = section.find('Provides')
+                    if provides is not None:
+                        provided_packages = [
+                                pkg
+                                for group in apt_pkg.parse_depends(provides)
+                                for (pkg, _, _) in group]
+                        for dname in provided_packages:
+                            if dname not in bprov[barch]:
+                                bprov[barch][dname] = set()
+                            bprov[barch][dname].add(pkgname)
+
+        self.process_files(bdeps, bprov, brecs)
+    read_files.__doc__ = AptSource.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py
new file mode 100644
index 000000000..2267197ca
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py
@@ -0,0 +1,14 @@
+""" Reader for bzip2 compressed package sources. """
+
+import bz2
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class Bzip2Reader(Reader):
+    extension = 'bz'
+
+    def _open(self, filename):
+        return bz2.BZ2File(filename)
+
+    def readable(self):
+        return True
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py
new file mode 100644
index 000000000..8e67e2b33
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py
@@ -0,0 +1,11 @@
+""" Reader for gzip compressed package sources. """
+
+import gzip
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class GzipReader(Reader):
+    extension = 'gz'
+
+    def _open(self, filename):
+        return gzip.GzipFile(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py
new file mode 100644
index 000000000..2f7a18d84
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py
@@ -0,0 +1,10 @@
+""" Reader for uncompressed package sources. """
+
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class NoneReader(Reader):
+    extension = ''
+
+    def _open(self, filename):
+        return open(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py
new file mode 100644
index 000000000..39a058767
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py
@@ -0,0 +1,11 @@
+""" Reader for lzma compressed package sources. """
+
+import lzma
+from Bcfg2.Server.Plugins.Packages.Readers import Reader
+
+
+class XzReader(Reader):
+    extension = 'xz'
+
+    def _open(self, filename):
+        return lzma.LZMAFile(filename)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py
new file mode 100644
index 000000000..51b4fb79c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py
@@ -0,0 +1,36 @@
+"""This module implements different readers for package files."""
+
+from io import IOBase
+from Bcfg2.Compat import walk_packages
+
+
+def get_readers():
+    """ Return all available packages readers. """
+    return [m[1]  # pylint: disable=C0103
+            for m in walk_packages(path=__path__)]
+
+
+class Reader(IOBase):
+    extension = None
+
+    def __init__(self, name):
+        self.name = name
+        self._file = self._open(name)
+
+    def _open(self, filename):
+        raise NotImplementedError
+
+    def read(self, size=-1):
+        return self._file.read(size)
+
+    def readable(self):
+        return self._file.readable()
+
+    def readline(self, size=-1):
+        return self._file.readline(size)
+
+    def readlines(self, hint=None):
+        return self._file.readlines(size)
+
+    def writelines(self, lines):
+        self._unsupported("writelines")
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 86f7698f7..574dbd851 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -49,6 +49,7 @@ in your ``Source`` subclass.  For an example of this kind of
 import os
 import re
 import sys
+import Bcfg2.Options
 from Bcfg2.Logger import Debuggable
 from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \
     HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \
@@ -56,7 +57,7 @@ from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \
 from Bcfg2.Server.Statistics import track_statistics
 
 
-def fetch_url(url):
+def fetch_url(url, opts):
     """ Return the content of the given URL.
 
     :param url: The URL to fetch content from.
@@ -73,8 +74,14 @@ def fetch_url(url):
         url = mobj.group(1) + mobj.group(4)
         auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
         auth.add_password(None, url, user, passwd)
-        install_opener(build_opener(auth))
-    return urlopen(url).read()
+        req = build_opener(auth)
+    else:
+        req = build_opener()
+
+    if 'user-agent' in opts:
+        req.addheaders = [('User-Agent', opts['user-agent'])]
+
+    return req.open(url).read()
 
 
 class SourceInitError(Exception):
@@ -111,6 +118,12 @@ class Source(Debuggable):  # pylint: disable=R0902
     #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`.
     ptype = None
 
+    #: The default compression format used by this Source class. This
+    #: is the file the package metadata files should be loaded. It is
+    #: used if a source has no custom compression format specified
+    #: in the :attr:`server_options`.
+    default_compression = 'None'
+
     def __init__(self, basepath, xsource):  # pylint: disable=R0912
         """
         :param basepath: The base filesystem path under which cache
@@ -188,6 +201,12 @@ class Source(Debuggable):  # pylint: disable=R0902
         #: The "name" attribute from :attr:`xsource`
         self.name = None
 
+        #: The "priority" attribute from :attr:`xsource`
+        self.priority = xsource.get('priority', 500)
+
+        #: The "pin" attribute from :attr:`xsource`
+        self.pin = xsource.get('pin', '')
+
         #: A list of predicates that are used to determine if this
         #: source applies to a given
         #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
@@ -250,11 +269,13 @@ class Source(Debuggable):  # pylint: disable=R0902
         for arch in self.arches:
             if self.url:
                 usettings = [dict(version=self.version, component=comp,
-                                  arch=arch, debsrc=self.debsrc)
+                                  arch=arch, debsrc=self.debsrc,
+                                  priority=self.priority, pin=self.pin)
                              for comp in self.components]
             else:  # rawurl given
                 usettings = [dict(version=self.version, component=None,
-                                  arch=arch, debsrc=self.debsrc)]
+                                  arch=arch, debsrc=self.debsrc,
+                                  priority=self.priority, pin=self.pin)]
 
             for setting in usettings:
                 if not self.rawurl:
@@ -263,6 +284,8 @@ class Source(Debuggable):  # pylint: disable=R0902
                     setting['baseurl'] = self.rawurl
                 setting['url'] = baseurl % setting
                 setting['name'] = self.get_repo_name(setting)
+                setting['options'] = dict(server=self.server_options,
+                                          client=self.client_options)
             self.url_map.extend(usettings)
 
     def _init_attributes(self, xsource):
@@ -328,6 +351,38 @@ class Source(Debuggable):  # pylint: disable=R0902
                     self.conditions.append(lambda m, el=el:
                                            el.get("name") == m.hostname)
 
+    def _get_reader(self):
+        ctype = self.default_compression
+        if 'compression' in self.server_options:
+            ctype = self.server_options['compression']
+
+        for mod in Bcfg2.Options.setup.packages_readers:
+            if mod.__name__.endswith(".%s" % ctype.title()):
+                return getattr(mod, "%sReader" % ctype.title())
+
+        raise ValueError("Packages: Unknown compression type %s" % ctype)
+
+    def _get_extension(self):
+        cls = self._get_reader()
+        if cls.extension is None:
+            raise ValueError("%s does not define an extension" %
+                             cls.__name__)
+        return cls.extension
+
+    def build_filename(self, basename):
+        extension = self._get_extension()
+        if extension == '':
+            return basename
+        return "%s.%s" % (basename, extension)
+
+    def open_file(self, fname):
+        try:
+            cls = self._get_reader()
+            return cls(fname)
+        except IOError:
+            self.logger.error("Packages: Failed to read file %s" % fname)
+            raise
+
     @property
     def cachekey(self):
         """ A unique key for this source that will be used to generate
@@ -534,6 +589,9 @@ class Source(Debuggable):  # pylint: disable=R0902
                 self.logger.warning("%s provides no packages for %s" %
                                     (self, agrp))
                 continue
+            if (agrp in self.blacklist or
+                    (len(self.whitelist) != 0 and agrp not in self.whitelist)):
+                continue
             for key, value in list(self.provides[agrp].items()):
                 if key not in vdict:
                     vdict[key] = set(value)
@@ -686,7 +744,7 @@ class Source(Debuggable):  # pylint: disable=R0902
             self.logger.info("Packages: Updating %s" % url)
             fname = self.escape_url(url)
             try:
-                open(fname, 'wb').write(fetch_url(url))
+                open(fname, 'wb').write(fetch_url(url, self.server_options))
             except ValueError:
                 self.logger.error("Packages: Bad url string %s" % url)
                 raise
@@ -760,7 +818,9 @@ class Source(Debuggable):  # pylint: disable=R0902
         :returns: list of strings
         """
         for arch in self.get_arches(metadata):
-            if package in self.provides[arch]:
+            if (package in self.provides[arch] and
+                    package not in self.blacklist and
+                    (len(self.whitelist) == 0 or package in self.whitelist)):
                 return self.provides[arch][package]
         return []
 
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 846fb89cd..acb11f1ab 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -850,7 +850,7 @@ class YumCollection(Collection):
         return new
 
     @track_statistics()
-    def complete(self, packagelist, recommended=None):
+    def complete(self, packagelist, recommended=None, pinnings=None):
         """ Build a complete list of all packages and their dependencies.
 
         When using the Python yum libraries, this defers to the
@@ -868,7 +868,8 @@ class YumCollection(Collection):
                   resolved.
         """
         if not self.use_yum:
-            return Collection.complete(self, packagelist, recommended)
+            return Collection.complete(self, packagelist, recommended,
+                                       pinnings)
 
         lock = FileLock(os.path.join(self.cachefile, "lock"))
         slept = 0
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 95b4baa3e..1a9673891 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -14,6 +14,7 @@ from Bcfg2.Compat import urlopen, HTTPError, URLError
 from Bcfg2.Server.Plugins.Packages.Collection import Collection, \
     get_collection_class
 from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
+from Bcfg2.Server.Plugins.Packages.Readers import get_readers
 from Bcfg2.Server.Statistics import track_statistics
 
 
@@ -36,6 +37,12 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction):
     fail_silently = True
 
 
+class PackagesReadersAction(Bcfg2.Options.ComponentAction):
+    """ ComponentAction to load Packages readers """
+    bases = ['Bcfg2.Server.Plugins.Packages.Readers']
+    module = True
+
+
 class Packages(Bcfg2.Server.Plugin.Plugin,
                Bcfg2.Server.Plugin.StructureValidator,
                Bcfg2.Server.Plugin.Generator,
@@ -57,7 +64,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             help="Packages backends to load",
             type=Bcfg2.Options.Types.comma_list,
             action=PackagesBackendAction,
-            default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Dummy']),
+            default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Dummy', 'Pyapt']),
         Bcfg2.Options.PathOption(
             cf=("packages", "cache"), dest="packages_cache",
             help="Path to the Packages cache",
@@ -82,7 +89,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             cf=("packages", "apt_config"),
             help="The default path for generated apt configs",
             default="/etc/apt/sources.list.d/"
-            "bcfg2-packages-generated-sources.list")]
+            "bcfg2-packages-generated-sources.list"),
+        Bcfg2.Options.Option(
+            cf=("packages", "readers"), dest="packages_readers",
+            help="Packages readers to load",
+            type=Bcfg2.Options.Types.comma_list,
+            action=PackagesReadersAction,
+            default=get_readers())]
 
     #: Packages is an alternative to
     #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it.
@@ -315,6 +328,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         groups = []
         recommended = dict()
 
+        pinned_src = dict()
+        if hasattr(metadata, 'PkgVars'):
+            pinned_src = metadata.PkgVars['pin']
+
         for struct in structures:
             for pkg in struct.xpath('//Package | //BoundPackage'):
                 if pkg.get("name"):
@@ -356,11 +373,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         base.update(collection.get_essential())
 
         # check for this set of packages in the package cache
-        pkey = hash(tuple(base))
+        pkey = hash((tuple(base), tuple(recommended), tuple(pinned_src)))
         pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets",
                                           collection.cachekey)
         if pkey not in pcache:
-            pcache[pkey] = collection.complete(base, recommended)
+            pcache[pkey] = collection.complete(base, recommended, pinned_src)
         packages, unknown = pcache[pkey]
         if unknown:
             self.logger.info("Packages: Got %d unknown entries" % len(unknown))
diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
new file mode 100644
index 000000000..9a2649d02
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
@@ -0,0 +1,65 @@
+import os
+import re
+import sys
+import copy
+import logging
+import lxml.etree
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger('Bcfg2.Plugins.PkgVars')
+vars = ['pin', 'use', 'keywords']
+
+class PkgVarsFile(Bcfg2.Server.Plugin.StructFile):
+    def get_additional_data(self, meta):
+        data = self.Match(meta)
+        results = {}
+        for d in data:
+            name = d.get('name', '')
+
+            for v in vars:
+                value = d.get(v, None)
+                if value:
+                    if v not in results:
+                        results[v] = {}
+                    if name not in results[v]:
+                        results[v][name] = set()
+
+                    results[v][name].add(value)
+
+        return results
+
+class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
+    __child__ = PkgVarsFile
+    patterns = re.compile(r'.*\.xml$')
+
+    def get_additional_data(self, meta):
+        results = {}
+        for v in vars:
+            results[v] = {}
+
+        for files in self.entries:
+            new = self.entries[files].get_additional_data(meta)
+            for x in vars:
+                if x in new:
+                    results[x].update(new[x])
+
+        return results
+
+class PkgVars(Bcfg2.Server.Plugin.Plugin,
+              Bcfg2.Server.Plugin.Connector):
+    name = 'PkgVars'
+    version = '$Revision$'
+
+    def __init__(self, core):
+        Bcfg2.Server.Plugin.Plugin.__init__(self, core)
+        Bcfg2.Server.Plugin.Connector.__init__(self)
+        try:
+            self.store = PkgVarsDirectoryBacked(self.data)
+        except OSError:
+            e = sys.exc_info()[1]
+            self.logger.error("Error while creating PkgVars store: %s %s" %
+                              (e.strerror, e.filename))
+            raise Bcfg2.Server.Plugin.PluginInitError
+
+    def get_additional_data(self, meta):
+        return self.store.get_additional_data(meta)