Allow to run directly from a git checkout (#1037)

Signed-off-by: Sol Jerome <sol.jerome@gmail.com>

8 files changed, 1943 insertions, 0 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
new file mode 100644
index 000000000..f76bf7fa1
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -0,0 +1,139 @@
+import re
+import gzip
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+from Bcfg2.Server.Plugins.Packages.Source import Source
+from Bcfg2.Bcfg2Py3k import cPickle, file
+
+class AptCollection(Collection):
+    def get_group(self, group):
+        self.logger.warning("Packages: Package groups are not supported by APT")
+        return []
+
+class AptSource(Source):
+    basegroups = ['apt', 'debian', 'ubuntu', 'nexenta']
+    ptype = 'deb'
+
+    def __init__(self, basepath, xsource, config):
+        Source.__init__(self, basepath, xsource, config)
+        self.pkgnames = set()
+
+        self.url_map = [{'rawurl': self.rawurl, 'url': self.url,
+                         'version': self.version,
+                         'components': self.components, 'arches': self.arches}]
+
+    def save_state(self):
+        cache = file(self.cachefile, 'wb')
+        cPickle.dump((self.pkgnames, self.deps, self.provides),
+                     cache, 2)
+        cache.close()
+
+    def load_state(self):
+        data = file(self.cachefile)
+        self.pkgnames, self.deps, self.provides = cPickle.load(data)
+
+    def filter_unknown(self, unknown):
+        filtered = set([u for u in unknown if u.startswith('choice')])
+        unknown.difference_update(filtered)
+
+    def get_urls(self):
+        if not self.rawurl:
+            rv = []
+            for part in self.components:
+                for arch in self.arches:
+                    rv.append("%sdists/%s/%s/binary-%s/Packages.gz" %
+                              (self.url, self.version, part, arch))
+            return rv
+        else:
+            return ["%sPackages.gz" % self.rawurl]
+    urls = property(get_urls)
+
+    def read_files(self):
+        bdeps = dict()
+        bprov = dict()
+        if self.recommended:
+            depfnames = ['Depends', 'Pre-Depends', 'Recommends']
+        else:
+            depfnames = ['Depends', 'Pre-Depends']
+        for fname in self.files:
+            if not self.rawurl:
+                barch = [x
+                         for x in fname.split('@')
+                         if x.startswith('binary-')][0][7:]
+            else:
+                # RawURL entries assume that they only have one <Arch></Arch>
+                # element and that it is the architecture of the source.
+                barch = self.arches[0]
+            if barch not in bdeps:
+                bdeps[barch] = dict()
+                bprov[barch] = dict()
+            try:
+                reader = gzip.GzipFile(fname)
+            except:
+                self.logger.error("Packages: Failed to read file %s" % fname)
+                raise
+            for line in reader.readlines():
+                words = str(line.strip()).split(':', 1)
+                if words[0] == 'Package':
+                    pkgname = words[1].strip().rstrip()
+                    self.pkgnames.add(pkgname)
+                    bdeps[barch][pkgname] = []
+                elif words[0] in depfnames:
+                    vindex = 0
+                    for dep in words[1].split(','):
+                        if '|' in dep:
+                            cdeps = [re.sub('\s+', '',
+                                            re.sub('\(.*\)', '', cdep))
+                                     for cdep in dep.split('|')]
+                            dyn_dname = "choice-%s-%s-%s" % (pkgname,
+                                                             barch,
+                                                             vindex)
+                            vindex += 1
+                            bdeps[barch][pkgname].append(dyn_dname)
+                            bprov[barch][dyn_dname] = set(cdeps)
+                        else:
+                            raw_dep = re.sub('\(.*\)', '', dep)
+                            raw_dep = raw_dep.rstrip().strip()
+                            bdeps[barch][pkgname].append(raw_dep)
+                elif words[0] == 'Provides':
+                    for pkg in words[1].split(','):
+                        dname = pkg.rstrip().strip()
+                        if dname not in bprov[barch]:
+                            bprov[barch][dname] = set()
+                        bprov[barch][dname].add(pkgname)
+
+        self.deps['global'] = dict()
+        self.provides['global'] = dict()
+        for barch in bdeps:
+            self.deps[barch] = dict()
+            self.provides[barch] = dict()
+        for pkgname in self.pkgnames:
+            pset = set()
+            for barch in bdeps:
+                if pkgname not in bdeps[barch]:
+                    bdeps[barch][pkgname] = []
+                pset.add(tuple(bdeps[barch][pkgname]))
+            if len(pset) == 1:
+                self.deps['global'][pkgname] = pset.pop()
+            else:
+                for barch in bdeps:
+                    self.deps[barch][pkgname] = bdeps[barch][pkgname]
+        provided = set()
+        for bprovided in list(bprov.values()):
+            provided.update(set(bprovided))
+        for prov in provided:
+            prset = set()
+            for barch in bprov:
+                if prov not in bprov[barch]:
+                    continue
+                prset.add(tuple(bprov[barch].get(prov, ())))
+            if len(prset) == 1:
+                self.provides['global'][prov] = prset.pop()
+            else:
+                for barch in bprov:
+                    self.provides[barch][prov] = bprov[barch].get(prov, ())
+        self.save_state()
+
+    def is_package(self, _, pkg):
+        return (pkg in self.pkgnames and
+                pkg not in self.blacklist and
+                (len(self.whitelist) == 0 or pkg in self.whitelist))
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
new file mode 100644
index 000000000..59c50fe8b
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -0,0 +1,343 @@
+import copy
+import logging
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger(__name__)
+
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
+
+# we have to cache Collection objects so that calling Packages.Refresh
+# or .Reload can tell the collection objects to clean up their cache,
+# but we don't actually use the cache to return a Collection object
+# when one is requested, because that prevents new machines from
+# working, since a Collection object gets created by
+# get_additional_data(), which is called for all clients at server
+# startup.  (It would also prevent machines that change groups from
+# working properly; e.g., if you reinstall a machine with a new OS,
+# then returning a cached Collection object would give the wrong
+# sources to that client.)
+collections = dict()
+
+class Collection(Bcfg2.Server.Plugin.Debuggable):
+    def __init__(self, metadata, sources, basepath, debug=False):
+        """ don't call this directly; use the factory function """
+        Bcfg2.Server.Plugin.Debuggable.__init__(self)
+        self.debug_flag = debug
+        self.metadata = metadata
+        self.sources = sources
+        self.basepath = basepath
+        self.virt_pkgs = dict()
+
+        try:
+            self.config = sources[0].config
+            self.cachepath = sources[0].basepath
+            self.ptype = sources[0].ptype
+        except IndexError:
+            self.config = None
+            self.cachepath = None
+            self.ptype = "unknown"
+
+        self.cachefile = None
+
+    @property
+    def cachekey(self):
+        return md5(self.get_config()).hexdigest()
+
+    def get_config(self):
+        self.logger.error("Packages: Cannot generate config for host with "
+                          "multiple source types (%s)" % self.metadata.hostname)
+        return ""
+
+    def get_relevant_groups(self):
+        groups = []
+        for source in self.sources:
+            groups.extend(source.get_relevant_groups(self.metadata))
+        return sorted(list(set(groups)))
+
+    @property
+    def basegroups(self):
+        groups = set()
+        for source in self.sources:
+            groups.update(source.basegroups)
+        return list(groups)
+
+    @property
+    def cachefiles(self):
+        cachefiles = set([self.cachefile])
+        for source in self.sources:
+            cachefiles.add(source.cachefile)
+        return list(cachefiles)
+
+    def get_group(self, group, ptype=None):
+        for source in self.sources:
+            pkgs = source.get_group(self.metadata, group, ptype=ptype)
+            if pkgs:
+                return pkgs
+        self.logger.warning("Packages: '%s' is not a valid group" % group)
+        return []
+
+    def is_package(self, package):
+        for source in self.sources:
+            if source.is_package(self.metadata, package):
+                return True
+        return False
+
+    def is_virtual_package(self, package):
+        for source in self.sources:
+            if source.is_virtual_package(self.metadata, package):
+                return True
+        return False
+
+    def get_deps(self, package):
+        for source in self.sources:
+            if source.is_package(self.metadata, package):
+                return source.get_deps(self.metadata, package)
+        return []
+
+    def get_provides(self, package):
+        for source in self.sources:
+            providers = source.get_provides(self.metadata, package)
+            if providers:
+                return providers
+        return []
+
+    def get_vpkgs(self):
+        """ get virtual packages """
+        vpkgs = dict()
+        for source in self.sources:
+            s_vpkgs = source.get_vpkgs(self.metadata)
+            for name, prov_set in list(s_vpkgs.items()):
+                if name not in vpkgs:
+                    vpkgs[name] = set(prov_set)
+                else:
+                    vpkgs[name].update(prov_set)
+        return vpkgs
+
+    def filter_unknown(self, unknown):
+        for source in self.sources:
+            source.filter_unknown(unknown)
+
+    def magic_groups_match(self):
+        for source in self.sources:
+            if source.magic_groups_match(self.metadata):
+                return True
+
+    def build_extra_structures(self, independent):
+        pass
+
+    def get_additional_data(self):
+        sdata = []
+        for source in self.sources:
+            sdata.extend(copy.deepcopy(source.url_map))
+        return sdata
+
+    def setup_data(self, force_update=False):
+        """ do any collection-level data setup tasks """
+        pass
+
+    def complete(self, packagelist):
+        '''Build the transitive closure of all package dependencies
+
+        Arguments:
+        packageslist - set of package names
+        returns => (set(packages), set(unsatisfied requirements))
+        '''
+
+        # setup vpkg cache
+        pgrps = tuple(self.get_relevant_groups())
+        if pgrps not in self.virt_pkgs:
+            self.virt_pkgs[pgrps] = self.get_vpkgs()
+        vpkg_cache = self.virt_pkgs[pgrps]
+
+        # unclassified is set of unsatisfied requirements (may be pkg
+        # for vpkg)
+        unclassified = set(packagelist)
+        vpkgs = set()
+        both = set()
+        pkgs = set(packagelist)
+
+        packages = set()
+        examined = set()
+        unknown = set()
+
+        final_pass = False
+        really_done = False
+        # do while unclassified or vpkgs or both or pkgs
+        while unclassified or pkgs or both or final_pass:
+            if really_done:
+                break
+            if len(unclassified) + len(pkgs) + len(both) == 0:
+                # one more pass then exit
+                really_done = True
+
+            while unclassified:
+                current = unclassified.pop()
+                examined.add(current)
+                is_pkg = False
+                if self.is_package(current):
+                    is_pkg = True
+
+                is_vpkg = current in vpkg_cache
+
+                if is_pkg and is_vpkg:
+                    both.add(current)
+                elif is_pkg and not is_vpkg:
+                    pkgs.add(current)
+                elif is_vpkg and not is_pkg:
+                    vpkgs.add(current)
+                elif not is_vpkg and not is_pkg:
+                    unknown.add(current)
+
+            while pkgs:
+                # direct packages; current can be added, and all deps
+                # should be resolved
+                current = pkgs.pop()
+                self.debug_log("Packages: handling package requirement %s" %
+                               current)
+                packages.add(current)
+                deps = self.get_deps(current)
+                newdeps = set(deps).difference(examined)
+                if newdeps:
+                    self.debug_log("Packages: Package %s added requirements %s"
+                                   % (current, newdeps))
+                unclassified.update(newdeps)
+
+            satisfied_vpkgs = set()
+            for current in vpkgs:
+                # virtual dependencies, satisfied if one of N in the
+                # config, or can be forced if only one provider
+                if len(vpkg_cache[current]) == 1:
+                    self.debug_log("Packages: requirement %s satisfied by %s" %
+                                   (current, vpkg_cache[current]))
+                    unclassified.update(vpkg_cache[current].difference(examined))
+                    satisfied_vpkgs.add(current)
+                else:
+                    satisfiers = [item for item in vpkg_cache[current]
+                                  if item in packages]
+                    self.debug_log("Packages: requirement %s satisfied by %s" %
+                                   (current, satisfiers))
+                    satisfied_vpkgs.add(current)
+            vpkgs.difference_update(satisfied_vpkgs)
+
+            satisfied_both = set()
+            for current in both:
+                # packages that are both have virtual providers as
+                # well as a package with that name. allow use of virt
+                # through explicit specification, then fall back to
+                # forcing current on last pass
+                satisfiers = [item for item in vpkg_cache[current]
+                              if item in packages]
+                if satisfiers:
+                    self.debug_log("Packages: requirement %s satisfied by %s" %
+                                   (current, satisfiers))
+                    satisfied_both.add(current)
+                elif current in packagelist or final_pass:
+                    pkgs.add(current)
+                    satisfied_both.add(current)
+            both.difference_update(satisfied_both)
+
+            if len(unclassified) + len(pkgs) == 0:
+                final_pass = True
+            else:
+                final_pass = False
+
+            self.filter_unknown(unknown)
+
+        return packages, unknown
+
+    def __len__(self):
+        return len(self.sources)
+
+    def __getitem__(self, item):
+        return self.sources[item]
+
+    def __setitem__(self, item, value):
+        self.sources[item] = value
+
+    def __delitem__(self, item):
+        del self.sources[item]
+
+    def append(self, item):
+        self.sources.append(item)
+
+    def count(self):
+        return self.sources.count()
+
+    def index(self, item):
+        return self.sources.index(item)
+
+    def extend(self, items):
+        self.sources.extend(items)
+
+    def insert(self, index, item):
+        self.sources.insert(index, item)
+
+    def pop(self, index=None):
+        self.sources.pop(index)
+
+    def remove(self, item):
+        self.sources.remove(item)
+
+    def sort(self, cmp=None, key=None, reverse=False):
+        self.sources.sort(cmp, key, reverse)
+
+def clear_cache():
+    global collections
+    collections = dict()
+
+def factory(metadata, sources, basepath, debug=False):
+    global collections
+
+    if not sources.loaded:
+        # if sources.xml has not received a FAM event yet, defer;
+        # instantiate a dummy Collection object
+        return Collection(metadata, [], basepath)
+
+    sclasses = set()
+    relevant = list()
+
+    for source in sources:
+        if source.applies(metadata):
+            relevant.append(source)
+            sclasses.update([source.__class__])
+
+    if len(sclasses) > 1:
+        logger.warning("Packages: Multiple source types found for %s: %s" %
+                       ",".join([s.__name__ for s in sclasses]))
+        cclass = Collection
+    elif len(sclasses) == 0:
+        # you'd think this should be a warning, but it happens all the
+        # freaking time if you have a) machines in your clients.xml
+        # that do not have the proper groups set up yet (e.g., if you
+        # have multiple Bcfg2 servers and Packages-relevant groups set
+        # by probes); and b) templates that query all or multiple
+        # machines (e.g., with metadata.query.all_clients())
+        if debug:
+            logger.error("Packages: No sources found for %s" %
+                         metadata.hostname)
+        cclass = Collection
+    else:
+        stype = sclasses.pop().__name__.replace("Source", "")
+        try:
+            module = \
+                getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" %
+                                   stype.title()).Server.Plugins.Packages,
+                        stype.title())
+            cclass = getattr(module, "%sCollection" % stype.title())
+        except ImportError:
+            logger.error("Packages: Unknown source type %s" % stype)
+        except AttributeError:
+            logger.warning("Packages: No collection class found for %s sources"
+                           % stype)
+
+    if debug:
+        logger.error("Packages: Using %s for Collection of sources for %s" %
+                     (cclass.__name__, metadata.hostname))
+
+    collection = cclass(metadata, relevant, basepath, debug=debug)
+    collections[metadata.hostname] = collection
+    return collection
+
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
new file mode 100644
index 000000000..486c094a6
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py
@@ -0,0 +1,120 @@
+import gzip
+import tarfile
+from Bcfg2.Bcfg2Py3k import cPickle, file
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+from Bcfg2.Server.Plugins.Packages.Source import Source
+
+class PacCollection(Collection):
+    def get_group(self, group):
+        self.logger.warning("Packages: Package groups are not supported by APT")
+        return []
+
+class PacSource(Source):
+    basegroups = ['arch', 'parabola']
+    ptype = 'pacman'
+
+    def __init__(self, basepath, xsource, config):
+        Source.__init__(self, basepath, xsource, config)
+        self.pkgnames = set()
+
+        self.url_map = [{'rawurl': self.rawurl, 'url': self.url,
+                         'version': self.version,
+                         'components': self.components, 'arches': self.arches}]
+
+    def save_state(self):
+        cache = file(self.cachefile, 'wb')
+        cPickle.dump((self.pkgnames, self.deps, self.provides),
+                     cache, 2)
+        cache.close()
+
+    def load_state(self):
+        data = file(self.cachefile)
+        self.pkgnames, self.deps, self.provides = cPickle.load(data)
+
+    def filter_unknown(self, unknown):
+        filtered = set([u for u in unknown if u.startswith('choice')])
+        unknown.difference_update(filtered)
+
+    def get_urls(self):
+        if not self.rawurl:
+            rv = []
+            for part in self.components:
+                for arch in self.arches:
+                    rv.append("%s%s/os/%s/%s.db.tar.gz" %
+                              (self.url, part, arch, part))
+            return rv
+        else:
+            raise Exception("PacSource : RAWUrl not supported (yet)")
+    urls = property(get_urls)
+
+    def read_files(self):
+        bdeps = dict()
+        bprov = dict()
+
+        if self.recommended:
+            depfnames = ['Depends', 'Pre-Depends', 'Recommends']
+        else:
+            depfnames = ['Depends', 'Pre-Depends']
+
+        for fname in self.files:
+            if not self.rawurl:
+                barch = [x for x in fname.split('@') if x in self.arches][0]
+            else:
+                # RawURL entries assume that they only have one <Arch></Arch>
+                # element and that it is the architecture of the source.
+                barch = self.arches[0]
+
+            if barch not in bdeps:
+                bdeps[barch] = dict()
+                bprov[barch] = dict()
+            try:
+                self.debug_log("Packages: try to read %s" % fname)
+                tar = tarfile.open(fname, "r")
+                reader = gzip.GzipFile(fname)
+            except:
+                self.logger.error("Packages: Failed to read file %s" % fname)
+                raise
+
+            for tarinfo in tar:
+                if tarinfo.isdir():
+                    self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0])
+                    self.debug_log("Packages: added %s" %
+                                   tarinfo.name.rsplit("-", 2)[0])
+            tar.close()
+
+        self.deps['global'] = dict()
+        self.provides['global'] = dict()
+        for barch in bdeps:
+            self.deps[barch] = dict()
+            self.provides[barch] = dict()
+        for pkgname in self.pkgnames:
+            pset = set()
+            for barch in bdeps:
+                if pkgname not in bdeps[barch]:
+                    bdeps[barch][pkgname] = []
+                pset.add(tuple(bdeps[barch][pkgname]))
+            if len(pset) == 1:
+                self.deps['global'][pkgname] = pset.pop()
+            else:
+                for barch in bdeps:
+                    self.deps[barch][pkgname] = bdeps[barch][pkgname]
+        provided = set()
+        for bprovided in list(bprov.values()):
+            provided.update(set(bprovided))
+        for prov in provided:
+            prset = set()
+            for barch in bprov:
+                if prov not in bprov[barch]:
+                    continue
+                prset.add(tuple(bprov[barch].get(prov, ())))
+            if len(prset) == 1:
+                self.provides['global'][prov] = prset.pop()
+            else:
+                for barch in bprov:
+                    self.provides[barch][prov] = bprov[barch].get(prov, ())
+        self.save_state()
+
+    def is_package(self, _, pkg):
+        return (pkg in self.pkgnames and
+                pkg not in self.blacklist and
+                (len(self.whitelist) == 0 or pkg in self.whitelist))
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py
new file mode 100644
index 000000000..3846c06ce
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py
@@ -0,0 +1,15 @@
+import Bcfg2.Server.Plugin
+
+class PackagesConfig(Bcfg2.Server.Plugin.SimpleConfig):
+    _required = False
+
+    def Index(self):
+        """ Build local data structures """
+        Bcfg2.Server.Plugin.SimpleConfig.Index(self)
+
+        if hasattr(self.plugin, "sources") and self.plugin.sources.loaded:
+            # only reload Packages plugin if sources have been loaded.
+            # otherwise, this is getting called on server startup, and
+            # we have to wait until all sources have been indexed
+            # before we can call Packages.Reload()
+            self.plugin.Reload()
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
new file mode 100644
index 000000000..3511cfc3d
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
@@ -0,0 +1,97 @@
+import os
+import sys
+import lxml.etree
+import Bcfg2.Server.Plugin
+from Bcfg2.Server.Plugins.Packages.Source import SourceInitError
+
+class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked,
+                      Bcfg2.Server.Plugin.StructFile,
+                      Bcfg2.Server.Plugin.Debuggable):
+    __identifier__ = None
+
+    def __init__(self, filename, cachepath, fam, packages, config):
+        Bcfg2.Server.Plugin.Debuggable.__init__(self)
+        try:
+            Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self,
+                                                             filename,
+                                                             fam)
+        except OSError:
+            err = sys.exc_info()[1]
+            msg = "Packages: Failed to read configuration file: %s" % err
+            if not os.path.exists(self.name):
+                msg += " Have you created it?"
+            self.logger.error(msg)
+            raise Bcfg2.Server.Plugin.PluginInitError(msg)
+        Bcfg2.Server.Plugin.StructFile.__init__(self, filename)
+        self.cachepath = cachepath
+        self.config = config
+        if not os.path.exists(self.cachepath):
+            # create cache directory if needed
+            try:
+                os.makedirs(self.cachepath)
+            except OSError:
+                err = sys.exc_info()[1]
+                self.logger.error("Could not create Packages cache at %s: %s" %
+                                  (self.cachepath, err))
+        self.pkg_obj = packages
+        self.parsed = set()
+        self.loaded = False
+
+    def toggle_debug(self):
+        Bcfg2.Server.Plugin.Debuggable.toggle_debug(self)
+        for source in self.entries:
+            source.toggle_debug()
+
+    def HandleEvent(self, event=None):
+        Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event)
+        if event.filename != self.name:
+            self.parsed.add(os.path.basename(event.filename))
+
+        if sorted(list(self.parsed)) == sorted(self.extras):
+            self.logger.info("Reloading Packages plugin")
+            self.pkg_obj.Reload()
+            self.loaded = True
+
+    def Index(self):
+        Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self)
+        self.entries = []
+        for xsource in self.xdata.findall('.//Source'):
+            source = self.source_from_xml(xsource)
+            if source is not None:
+                self.entries.append(source)
+
+    def source_from_xml(self, xsource):
+        """ create a *Source object from its XML representation in
+        sources.xml """
+        stype = xsource.get("type")
+        if stype is None:
+            self.logger.error("Packages: No type specified for source, "
+                              "skipping")
+            return None
+
+        try:
+            module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" %
+                                        stype.title()).Server.Plugins.Packages,
+                             stype.title())
+            cls = getattr(module, "%sSource" % stype.title())
+        except (ImportError, AttributeError):
+            self.logger.error("Packages: Unknown source type %s" % stype)
+            return None
+
+        try:
+            source = cls(self.cachepath, xsource, self.config)
+        except SourceInitError:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: %s" % err)
+            source = None
+
+        return source
+
+    def __getitem__(self, key):
+        return self.entries[key]
+
+    def __repr__(self):
+        return "PackagesSources: %s" % repr(self.entries)
+
+    def __str__(self):
+        return "PackagesSources: %s" % str(self.entries)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
new file mode 100644
index 000000000..b33fde60a
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -0,0 +1,278 @@
+import os
+import re
+import sys
+import base64
+import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \
+     HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \
+     urlopen, file, cPickle
+
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
+
+def fetch_url(url):
+    if '@' in url:
+        mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url)
+        if not mobj:
+            raise ValueError
+        user = mobj.group(2)
+        passwd = mobj.group(3)
+        url = mobj.group(1) + mobj.group(4)
+        auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm())
+        auth.add_password(None, url, user, passwd)
+        install_opener(build_opener(auth))
+    return urlopen(url).read()
+
+
+class SourceInitError(Exception):
+    pass
+
+
+class Source(Bcfg2.Server.Plugin.Debuggable):
+    mrepo_re = re.compile(r'/RPMS\.([^/]+)')
+    pulprepo_re = re.compile(r'pulp/repos/([^/]+)')
+    genericrepo_re = re.compile(r'https?://[^/]+/(.+?)/?$')
+    basegroups = []
+
+    def __init__(self, basepath, xsource, config):
+        Bcfg2.Server.Plugin.Debuggable.__init__(self)
+        self.basepath = basepath
+        self.xsource = xsource
+        self.config = config
+
+        try:
+            self.version = xsource.find('Version').text
+        except AttributeError:
+            pass
+
+        for key, tag in [('components', 'Component'), ('arches', 'Arch'),
+                         ('blacklist', 'Blacklist'),
+                         ('whitelist', 'Whitelist')]:
+            self.__dict__[key] = [item.text for item in xsource.findall(tag)]
+
+        self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]
+
+        self.recommended = xsource.get('recommended', 'false').lower() == 'true'
+
+        self.rawurl = xsource.get('rawurl', '')
+        if self.rawurl and not self.rawurl.endswith("/"):
+            self.rawurl += "/"
+        self.url = xsource.get('url', '')
+        if self.url and not self.url.endswith("/"):
+            self.url += "/"
+        self.version = xsource.get('version', '')
+
+        # build the set of conditions to see if this source applies to
+        # a given set of metadata
+        self.conditions = []
+        self.groups = [] # provided for some limited backwards compat
+        for el in xsource.iterancestors():
+            if el.tag == "Group":
+                if el.get("negate", "false").lower() == "true":
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") not in m.groups)
+                else:
+                    self.groups.append(el.get("name"))
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") in m.groups)
+            elif el.tag == "Client":
+                if el.get("negate", "false").lower() == "true":
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") != m.hostname)
+                else:
+                    self.conditions.append(lambda m, el=el:
+                                           el.get("name") == m.hostname)
+
+        self.deps = dict()
+        self.provides = dict()
+
+        self.cachefile = os.path.join(self.basepath,
+                                      "cache-%s" % self.cachekey)
+        self.url_map = []
+
+    @property
+    def cachekey(self):
+        return md5(cPickle.dumps([self.version, self.components, self.url,
+                                  self.rawurl, self.arches])).hexdigest()
+
+    def get_relevant_groups(self, metadata):
+        return sorted(list(set([g for g in metadata.groups
+                                if (g in self.basegroups or
+                                    g in self.groups or
+                                    g in self.arches)])))
+
+    def load_state(self):
+        pass
+
+    def setup_data(self, force_update=False):
+        should_read = True
+        should_download = False
+        if os.path.exists(self.cachefile):
+            try:
+                self.load_state()
+                should_read = False
+            except:
+                self.logger.error("Packages: Cachefile %s load failed; "
+                                  "falling back to file read" % self.cachefile)
+        if should_read:
+            try:
+                self.read_files()
+            except:
+                self.logger.error("Packages: File read failed; "
+                                  "falling back to file download")
+                should_download = True
+
+        if should_download or force_update:
+            try:
+                self.update()
+                self.read_files()
+            except:
+                self.logger.error("Packages: Failed to load data for Source "
+                                  "of %s. Some Packages will be missing." %
+                                  self.urls)
+
+    def get_repo_name(self, url_map):
+        # try to find a sensible name for a repo
+        if url_map['component']:
+            rname = url_map['component']
+        else:
+            name = None
+            for repo_re in (self.mrepo_re,
+                            self.pulprepo_re,
+                            self.genericrepo_re):
+                match = repo_re.search(url_map['url'])
+                if match:
+                    break
+            if name is None:
+                # couldn't figure out the name from the URL or URL map
+                # (which probably means its a screwy URL), so we just
+                # generate a random one
+                name = base64.b64encode(os.urandom(16))[:-2]
+            rname = "%s-%s" % (self.groups[0], name)
+        # see yum/__init__.py in the yum source, lines 441-449, for
+        # the source of this regex.  yum doesn't like anything but
+        # string.ascii_letters, string.digits, and [-_.:].  There
+        # doesn't seem to be a reason for this, because yum.
+        return re.sub(r'[^A-Za-z0-9-_.:]', '-', rname)
+
+    def __str__(self):
+        if self.rawurl:
+            return "%s at %s" % (self.__class__.__name__, self.rawurl)
+        elif self.url:
+            return "%s at %s" % (self.__class__.__name__, self.url)
+        else:
+            return self.__class__.__name__
+
+    def get_urls(self):
+        return []
+    urls = property(get_urls)
+
+    def get_files(self):
+        return [self.escape_url(url) for url in self.urls]
+    files = property(get_files)
+
+    def get_vpkgs(self, metadata):
+        agroups = ['global'] + [a for a in self.arches
+                                if a in metadata.groups]
+        vdict = dict()
+        for agrp in agroups:
+            for key, value in list(self.provides[agrp].items()):
+                if key not in vdict:
+                    vdict[key] = set(value)
+                else:
+                    vdict[key].update(value)
+        return vdict
+
+    def is_virtual_package(self, metadata, package):
+        """ called to determine if a package is a virtual package.
+        this is only invoked if the package is not listed in the dict
+        returned by get_vpkgs """
+        return False
+
+    def escape_url(self, url):
+        return os.path.join(self.basepath, url.replace('/', '@'))
+
+    def file_init(self):
+        pass
+
+    def read_files(self):
+        pass
+
+    def filter_unknown(self, unknown):
+        pass
+
+    def update(self):
+        for url in self.urls:
+            self.logger.info("Packages: Updating %s" % url)
+            fname = self.escape_url(url)
+            try:
+                data = fetch_url(url)
+                file(fname, 'w').write(data)
+            except ValueError:
+                self.logger.error("Packages: Bad url string %s" % url)
+                raise
+            except HTTPError:
+                err = sys.exc_info()[1]
+                self.logger.error("Packages: Failed to fetch url %s. HTTP "
+                                  "response code=%s" % (url, err.code))
+                raise
+
+    def applies(self, metadata):
+        # check base groups
+        if not self.magic_groups_match(metadata):
+            return False
+
+        # check Group/Client tags from sources.xml
+        for condition in self.conditions:
+            if not condition(metadata):
+                return False
+
+        return True
+
+    def get_arches(self, metadata):
+        return ['global'] + [a for a in self.arches if a in metadata.groups]
+
+    def get_deps(self, metadata, pkgname):
+        for arch in self.get_arches(metadata):
+            if pkgname in self.deps[arch]:
+                return self.deps[arch][pkgname]
+        return []
+
+    def get_provides(self, metadata, required):
+        for arch in self.get_arches(metadata):
+            if required in self.provides[arch]:
+                return self.provides[arch][required]
+        return []
+
+    def is_package(self, metadata, _):
+        return False
+
+    def get_package(self, metadata, package):
+        return package
+
+    def get_group(self, metadata, group, ptype=None):
+        return []
+
+    def magic_groups_match(self, metadata):
+        """ check to see if this source applies to the given host
+        metadata by checking 'magic' (base) groups only, or if magic
+        groups are off """
+        # we always check that arch matches
+        found_arch = False
+        for arch in self.arches:
+            if arch in metadata.groups:
+                found_arch = True
+                break
+        if not found_arch:
+            return False
+
+        if self.config.getboolean("global", "magic_groups",
+                                  default=True) == False:
+            return True
+        else:
+            for group in self.basegroups:
+                if group in metadata.groups:
+                    return True
+            return False
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
new file mode 100644
index 000000000..b39b6aed2
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -0,0 +1,688 @@
+import os
+import sys
+import time
+import copy
+import glob
+import socket
+import random
+import logging
+import threading
+import lxml.etree
+from UserDict import DictMixin
+from subprocess import Popen, PIPE, STDOUT
+import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \
+     fetch_url
+
+logger = logging.getLogger(__name__)
+
+try:
+    from pulp.client.consumer.config import ConsumerConfig
+    from pulp.client.api.repository import RepositoryAPI
+    from pulp.client.api.consumer import ConsumerAPI
+    from pulp.client.api import server
+    has_pulp = True
+except ImportError:
+    has_pulp = False
+
+try:
+    import yum
+    has_yum = True
+except ImportError:
+    has_yum = False
+    logger.info("Packages: No yum libraries found; forcing use of internal "
+                "dependency resolver")
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+XP = '{http://linux.duke.edu/metadata/common}'
+RP = '{http://linux.duke.edu/metadata/rpm}'
+RPO = '{http://linux.duke.edu/metadata/repo}'
+FL = '{http://linux.duke.edu/metadata/filelists}'
+
+PULPSERVER = None
+PULPCONFIG = None
+
+
+def _setup_pulp(config):
+    global PULPSERVER, PULPCONFIG
+    if not has_pulp:
+        logger.error("Packages: Cannot create Pulp collection: Pulp libraries not "
+                     "found")
+        raise Bcfg2.Server.Plugin.PluginInitError
+
+    if PULPSERVER is None:
+        try:
+            username = config.get("pulp", "username")
+            password = config.get("pulp", "password")
+        except ConfigParser.NoSectionError:
+            logger.error("Packages: No [pulp] section found in Packages/packages.conf")
+            raise Bcfg2.Server.Plugin.PluginInitError
+        except ConfigParser.NoOptionError:
+            err = sys.exc_info()[1]
+            logger.error("Packages: Required option not found in "
+                         "Packages/packages.conf: %s" % err)
+            raise Bcfg2.Server.Plugin.PluginInitError
+
+        PULPCONFIG = ConsumerConfig()
+        serveropts = PULPCONFIG.server
+
+        PULPSERVER = server.PulpServer(serveropts['host'],
+                                       int(serveropts['port']),
+                                       serveropts['scheme'],
+                                       serveropts['path'])
+        PULPSERVER.set_basic_auth_credentials(username, password)
+        server.set_active_server(PULPSERVER)
+    return PULPSERVER
+
+
+class YumCollection(Collection):
+    # options that are included in the [yum] section but that should
+    # not be included in the temporary yum.conf we write out
+    option_blacklist = ["use_yum_libraries", "helper"]
+
+    def __init__(self, metadata, sources, basepath, debug=False):
+        Collection.__init__(self, metadata, sources, basepath, debug=debug)
+        self.keypath = os.path.join(self.basepath, "keys")
+
+        if len(sources):
+            config = sources[0].config
+            self.use_yum = has_yum and config.getboolean("yum",
+                                                         "use_yum_libraries",
+                                                         default=False)
+        else:
+            self.use_yum = False
+
+        if self.use_yum:
+            self.cachefile = os.path.join(self.cachepath,
+                                         "cache-%s" % self.cachekey)
+            if not os.path.exists(self.cachefile):
+                os.mkdir(self.cachefile)
+
+            self.configdir = os.path.join(self.basepath, "yum")
+            if not os.path.exists(self.configdir):
+                os.mkdir(self.configdir)
+            self.cfgfile = os.path.join(self.configdir,
+                                        "%s-yum.conf" % self.cachekey)
+            self.write_config()
+
+            self.helper = self.config.get("yum", "helper",
+                                          default="/usr/sbin/bcfg2-yum-helper")
+        if has_pulp:
+            _setup_pulp(self.config)
+
+    def write_config(self):
+        if not os.path.exists(self.cfgfile):
+            yumconf = self.get_config(raw=True)
+            yumconf.add_section("main")
+
+            mainopts = dict(cachedir=self.cachefile,
+                            keepcache="0",
+                            sslverify="0",
+                            debuglevel="0",
+                            reposdir="/dev/null")
+            try:
+                for opt in self.config.options("yum"):
+                    if opt not in self.option_blacklist:
+                        mainopts[opt] = self.config.get("yum", opt)
+            except ConfigParser.NoSectionError:
+                pass
+
+            for opt, val in list(mainopts.items()):
+                yumconf.set("main", opt, val)
+
+            yumconf.write(open(self.cfgfile, 'w'))
+
+    def get_config(self, raw=False):
+        config = ConfigParser.SafeConfigParser()
+        for source in self.sources:
+            # get_urls() loads url_map as a side-effect
+            source.get_urls()
+            for url_map in source.url_map:
+                if url_map['arch'] in self.metadata.groups:
+                    basereponame = source.get_repo_name(url_map)
+                    reponame = basereponame
+
+                    added = False
+                    while not added:
+                        try:
+                            config.add_section(reponame)
+                            added = True
+                        except ConfigParser.DuplicateSectionError:
+                            match = re.match("-(\d)", reponame)
+                            if match:
+                                rid = int(match.group(1)) + 1
+                            else:
+                                rid = 1
+                            reponame = "%s-%d" % (basereponame, rid)
+
+                    config.set(reponame, "name", reponame)
+                    config.set(reponame, "baseurl", url_map['url'])
+                    config.set(reponame, "enabled", "1")
+                    if len(source.gpgkeys):
+                        config.set(reponame, "gpgcheck", "1")
+                        config.set(reponame, "gpgkey",
+                                   " ".join(source.gpgkeys))
+                    else:
+                        config.set(reponame, "gpgcheck", "0")
+
+                    if len(source.blacklist):
+                        config.set(reponame, "exclude",
+                                   " ".join(source.blacklist))
+                    if len(source.whitelist):
+                        config.set(reponame, "includepkgs",
+                                   " ".join(source.whitelist))
+
+        if raw:
+            return config
+        else:
+            # configparser only writes to file, so we have to use a
+            # StringIO object to get the data out as a string
+            buf = StringIO()
+            config.write(buf)
+            return "# This config was generated automatically by the Bcfg2 " \
+                   "Packages plugin\n\n" + buf.getvalue()
+
+    def build_extra_structures(self, independent):
+        """ build list of gpg keys to be added to the specification by
+        validate_structures() """
+        needkeys = set()
+        for source in self.sources:
+            for key in source.gpgkeys:
+                needkeys.add(key)
+
+        if len(needkeys):
+            if has_yum:
+                # this must be be has_yum, not use_yum, because
+                # regardless of whether the user wants to use the yum
+                # resolver we want to include gpg key data
+                keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey",
+                                            type=self.ptype, origin='Packages')
+            else:
+                self.logger.warning("GPGKeys were specified for yum sources in "
+                                    "sources.xml, but no yum libraries were "
+                                    "found")
+                self.logger.warning("GPG key version/release data cannot be "
+                                    "determined automatically")
+                self.logger.warning("Install yum libraries, or manage GPG keys "
+                                    "manually")
+                keypkg = None
+
+            for key in needkeys:
+                # figure out the path of the key on the client
+                keydir = self.config.get("global", "gpg_keypath",
+                                         default="/etc/pki/rpm-gpg")
+                remotekey = os.path.join(keydir, os.path.basename(key))
+                localkey = os.path.join(self.keypath, os.path.basename(key))
+                kdata = open(localkey).read()
+
+                # copy the key to the client
+                keypath = lxml.etree.Element("BoundPath", name=remotekey,
+                                             encoding='ascii',
+                                             owner='root', group='root',
+                                             type='file', perms='0644',
+                                             important='true')
+                keypath.text = kdata
+
+                # hook to add version/release info if possible
+                self._add_gpg_instances(keypkg, kdata, localkey, remotekey)
+                independent.append(keypath)
+            if keypkg is not None:
+                independent.append(keypkg)
+
+        # see if there are any pulp sources to handle
+        has_pulp_sources = False
+        for source in self.sources:
+            if source.pulp_id:
+                has_pulp_sources = True
+                break
+
+        if has_pulp_sources:
+            consumerapi = ConsumerAPI()
+            consumer = self._get_pulp_consumer(consumerapi=consumerapi)
+            if consumer is None:
+                consumer = consumerapi.create(self.metadata.hostname,
+                                              self.metadata.hostname)
+                lxml.etree.SubElement(independent, "BoundAction",
+                                      name="pulp-update", timing="pre",
+                                      when="always", status="check",
+                                      command="pulp-consumer consumer update")
+
+            for source in self.sources:
+                # each pulp source can only have one arch, so we don't
+                # have to check the arch in url_map
+                if (source.pulp_id and
+                    source.pulp_id not in consumer['repoids']):
+                    consumerapi.bind(self.metadata.hostname, source.pulp_id)
+
+            crt = lxml.etree.SubElement(independent, "BoundPath",
+                                        name="/etc/pki/consumer/cert.pem",
+                                        type="file", owner="root",
+                                        group="root", perms="0644")
+            crt.text = consumerapi.certificate(self.metadata.hostname)
+
+    def _get_pulp_consumer(self, consumerapi=None):
+        if consumerapi is None:
+            consumerapi = ConsumerAPI()
+        consumer = None
+        try:
+            consumer = consumerapi.consumer(self.metadata.hostname)
+        except server.ServerRequestError:
+            # consumer does not exist
+            pass
+        except socket.error:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: Could not contact Pulp server: %s" %
+                              err)
+        except:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: Unknown error querying Pulp server: %s"
+                              % err)
+        return consumer
+
+    def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey):
+        """ add gpg keys to the specification to ensure they get
+        installed """
+        # this must be be has_yum, not use_yum, because regardless of
+        # whether the user wants to use the yum resolver we want to
+        # include gpg key data
+        if not has_yum:
+            return
+
+        try:
+            kinfo = yum.misc.getgpgkeyinfo(keydata)
+            version = yum.misc.keyIdToRPMVer(kinfo['keyid'])
+            release = yum.misc.keyIdToRPMVer(kinfo['timestamp'])
+
+            lxml.etree.SubElement(keyentry, 'Instance',
+                                  version=version,
+                                  release=release,
+                                  simplefile=remotekey)
+        except ValueError:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: Could not read GPG key %s: %s" %
+                              (localkey, err))
+
+    def is_package(self, package):
+        if not self.use_yum:
+            return Collection.is_package(self, package)
+        elif isinstance(package, tuple):
+            if package[1] is None and package[2] == (None, None, None):
+                package = package[0]
+            else:
+                return None
+        else:
+            # this should really never get called; it's just provided
+            # for API completeness
+            return self.call_helper("is_package", package)
+
+    def is_virtual_package(self, package):
+        if not self.use_yum:
+            return Collection.is_virtual_package(self, package)
+        else:
+            # this should really never get called; it's just provided
+            # for API completeness
+            return self.call_helper("is_virtual_package", package)
+
+    def get_deps(self, package):
+        if not self.use_yum:
+            return Collection.get_deps(self, package)
+        else:
+            # this should really never get called; it's just provided
+            # for API completeness
+            return self.call_helper("get_deps", package)
+
+    def get_provides(self, required, all=False, silent=False):
+        if not self.use_yum:
+            return Collection.get_provides(self, package)
+        else:
+            # this should really never get called; it's just provided
+            # for API completeness
+            return self.call_helper("get_provides", package)
+
+    def get_group(self, group, ptype="default"):
+        if not self.use_yum:
+            self.logger.warning("Packages: Package groups are not supported by "
+                                "Bcfg2's internal Yum dependency generator")
+            return []
+
+        if group.startswith("@"):
+            group = group[1:]
+
+        pkgs = self.call_helper("get_group", dict(group=group, type=ptype))
+        return pkgs
+
+    def complete(self, packagelist):
+        if not self.use_yum:
+            return Collection.complete(self, packagelist)
+
+        packages = set()
+        unknown = set(packagelist)
+
+        if unknown:
+            result = \
+                self.call_helper("complete",
+                                 dict(packages=list(unknown),
+                                      groups=list(self.get_relevant_groups())))
+            if result and "packages" in result and "unknown" in result:
+                # we stringify every package because it gets returned
+                # in unicode; set.update() doesn't work if some
+                # elements are unicode and other are strings.  (I.e.,
+                # u'foo' and 'foo' get treated as unique elements.)
+                packages.update([str(p) for p in result['packages']])
+                unknown = set([str(p) for p in result['unknown']])
+
+            self.filter_unknown(unknown)
+
+        return packages, unknown
+
+    def call_helper(self, command, input=None):
+        """ Make a call to bcfg2-yum-helper.  The yum libs have
+        horrific memory leaks, so apparently the right way to get
+        around that in long-running processes it to have a short-lived
+        helper.  No, seriously -- check out the yum-updatesd code.
+        It's pure madness. """
+        # it'd be nice if we could change this to be more verbose if
+        # -v was given to bcfg2-server, but Collection objects don't
+        # get the 'setup' variable, so we don't know how verbose
+        # bcfg2-server is.  It'd also be nice if we could tell yum to
+        # log to syslog.  So would a unicorn.
+        cmd = [self.helper, "-c", self.cfgfile]
+        if self.debug_flag:
+            cmd.append("-v")
+        cmd.append(command)
+        self.debug_log("Packages: running %s" % " ".join(cmd))
+        try:
+            helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        except OSError:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: Failed to execute %s: %s" %
+                              (" ".join(cmd), err))
+            return None
+
+        if input:
+            idata = json.dumps(input)
+            (stdout, stderr) = helper.communicate(idata)
+        else:
+            (stdout, stderr) = helper.communicate()
+        rv = helper.wait()
+        if rv:
+            self.logger.error("Packages: error running bcfg2-yum-helper "
+                              "(returned %d): %s" % (rv, stderr))
+        elif self.debug_flag:
+            self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" %
+                           stderr)
+        try:
+            return json.loads(stdout)
+        except ValueError:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: error reading bcfg2-yum-helper "
+                              "output: %s" % err)
+            return None
+
+    def setup_data(self, force_update=False):
+        if not self.use_yum:
+            return Collection.setup_data(self, force_update)
+
+        if force_update:
+            # we call this twice: one to clean up data from the old
+            # config, and once to clean up data from the new config
+            self.call_helper("clean")
+
+        os.unlink(self.cfgfile)
+        self.write_config()
+
+        if force_update:
+            self.call_helper("clean")
+
+
+class YumSource(Source):
+    basegroups = ['yum', 'redhat', 'centos', 'fedora']
+    ptype = 'yum'
+
+    def __init__(self, basepath, xsource, config):
+        Source.__init__(self, basepath, xsource, config)
+        self.pulp_id = None
+        if has_pulp and xsource.get("pulp_id"):
+            self.pulp_id = xsource.get("pulp_id")
+
+            _setup_pulp(self.config)
+            repoapi = RepositoryAPI()
+            try:
+                self.repo = repoapi.repository(self.pulp_id)
+                self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key)
+                                for key in repoapi.listkeys(self.pulp_id)]
+            except server.ServerRequestError:
+                err = sys.exc_info()[1]
+                if err[0] == 401:
+                    msg = "Packages: Error authenticating to Pulp: %s" % err[1]
+                elif err[0] == 404:
+                    msg = "Packages: Pulp repo id %s not found: %s" % \
+                          (self.pulp_id, err[1])
+                else:
+                    msg = "Packages: Error %d fetching pulp repo %s: %s" % \
+                          (err[0], self.pulp_id, err[1])
+                raise SourceInitError(msg)
+            except socket.error:
+                err = sys.exc_info()[1]
+                raise SourceInitError("Could not contact Pulp server: %s" % err)
+            except:
+                err = sys.exc_info()[1]
+                raise SourceInitError("Unknown error querying Pulp server: %s" %
+                                      err)
+            self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'],
+                                     self.repo['relative_path'])
+            self.arches = [self.repo['arch']]
+
+        if not self.rawurl:
+            self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/"
+        else:
+            self.baseurl = self.rawurl
+        self.packages = dict()
+        self.deps = dict([('global', dict())])
+        self.provides = dict([('global', dict())])
+        self.filemap = dict([(x, dict())
+                             for x in ['global'] + self.arches])
+        self.needed_paths = set()
+        self.file_to_arch = dict()
+
+        self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries",
+                                                     default=False)
+
+    def save_state(self):
+        if not self.use_yum:
+            cache = file(self.cachefile, 'wb')
+            cPickle.dump((self.packages, self.deps, self.provides,
+                          self.filemap, self.url_map), cache, 2)
+            cache.close()
+
+
+    def load_state(self):
+        if not self.use_yum:
+            data = file(self.cachefile)
+            (self.packages, self.deps, self.provides,
+             self.filemap, self.url_map) = cPickle.load(data)
+
+    def get_urls(self):
+        surls = list()
+        self.url_map = []
+        for arch in self.arches:
+            if self.url:
+                usettings = [{'version':self.version, 'component':comp,
+                              'arch':arch}
+                             for comp in self.components]
+            else: # rawurl given
+                usettings = [{'version':self.version, 'component':None,
+                              'arch':arch}]
+
+            for setting in usettings:
+                setting['url'] = self.baseurl % setting
+                self.url_map.append(copy.deepcopy(setting))
+            surls.append((arch, [setting['url'] for setting in usettings]))
+        urls = []
+        for (sarch, surl_list) in surls:
+            for surl in surl_list:
+                urls.extend(self._get_urls_from_repodata(surl, sarch))
+        return urls
+    urls = property(get_urls)
+
+    def _get_urls_from_repodata(self, url, arch):
+        if self.use_yum:
+            return [url]
+
+        rmdurl = '%srepodata/repomd.xml' % url
+        try:
+            repomd = fetch_url(rmdurl)
+            xdata = lxml.etree.XML(repomd)
+        except ValueError:
+            self.logger.error("Packages: Bad url string %s" % rmdurl)
+            return []
+        except HTTPError:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: Failed to fetch url %s. code=%s" %
+                              (rmdurl, err.code))
+            return []
+        except lxml.etree.XMLSyntaxError:
+            err = sys.exc_info()[1]
+            self.logger.error("Packages: Failed to process metadata at %s: %s" %
+                              (rmdurl, err))
+            return []
+
+        urls = []
+        for elt in xdata.findall(RPO + 'data'):
+            if elt.get('type') in ['filelists', 'primary']:
+                floc = elt.find(RPO + 'location')
+                fullurl = url + floc.get('href')
+                urls.append(fullurl)
+                self.file_to_arch[self.escape_url(fullurl)] = arch
+        return urls
+
+    def read_files(self):
+        # we have to read primary.xml first, and filelists.xml afterwards;
+        primaries = list()
+        filelists = list()
+        for fname in self.files:
+            if fname.endswith('primary.xml.gz'):
+                primaries.append(fname)
+            elif fname.endswith('filelists.xml.gz'):
+                filelists.append(fname)
+
+        for fname in primaries:
+            farch = self.file_to_arch[fname]
+            fdata = lxml.etree.parse(fname).getroot()
+            self.parse_primary(fdata, farch)
+        for fname in filelists:
+            farch = self.file_to_arch[fname]
+            fdata = lxml.etree.parse(fname).getroot()
+            self.parse_filelist(fdata, farch)
+
+        # merge data
+        sdata = list(self.packages.values())
+        try:
+            self.packages['global'] = copy.deepcopy(sdata.pop())
+        except IndexError:
+            logger.error("Packages: No packages in repo")
+        while sdata:
+            self.packages['global'] = \
+                self.packages['global'].intersection(sdata.pop())
+
+        for key in self.packages:
+            if key == 'global':
+                continue
+            self.packages[key] = \
+                self.packages[key].difference(self.packages['global'])
+        self.save_state()
+
+    def parse_filelist(self, data, arch):
+        if arch not in self.filemap:
+            self.filemap[arch] = dict()
+        for pkg in data.findall(FL + 'package'):
+            for fentry in pkg.findall(FL + 'file'):
+                if fentry.text in self.needed_paths:
+                    if fentry.text in self.filemap[arch]:
+                        self.filemap[arch][fentry.text].add(pkg.get('name'))
+                    else:
+                        self.filemap[arch][fentry.text] = \
+                            set([pkg.get('name')])
+
+    def parse_primary(self, data, arch):
+        if arch not in self.packages:
+            self.packages[arch] = set()
+        if arch not in self.deps:
+            self.deps[arch] = dict()
+        if arch not in self.provides:
+            self.provides[arch] = dict()
+        for pkg in data.getchildren():
+            if not pkg.tag.endswith('package'):
+                continue
+            pkgname = pkg.find(XP + 'name').text
+            self.packages[arch].add(pkgname)
+
+            pdata = pkg.find(XP + 'format')
+            self.deps[arch][pkgname] = set()
+            pre = pdata.find(RP + 'requires')
+            if pre is not None:
+                for entry in pre.getchildren():
+                    self.deps[arch][pkgname].add(entry.get('name'))
+                    if entry.get('name').startswith('/'):
+                        self.needed_paths.add(entry.get('name'))
+            pro = pdata.find(RP + 'provides')
+            if pro != None:
+                for entry in pro.getchildren():
+                    prov = entry.get('name')
+                    if prov not in self.provides[arch]:
+                        self.provides[arch][prov] = list()
+                    self.provides[arch][prov].append(pkgname)
+
+    def is_package(self, metadata, item):
+        arch = [a for a in self.arches if a in metadata.groups]
+        if not arch:
+            return False
+        return ((item in self.packages['global'] or
+                 item in self.packages[arch[0]]) and
+                item not in self.blacklist and
+                (len(self.whitelist) == 0 or item in self.whitelist))
+
+    def get_vpkgs(self, metadata):
+        if self.use_yum:
+            return dict()
+
+        rv = Source.get_vpkgs(self, metadata)
+        for arch, fmdata in list(self.filemap.items()):
+            if arch not in metadata.groups and arch != 'global':
+                continue
+            for filename, pkgs in list(fmdata.items()):
+                rv[filename] = pkgs
+        return rv
+
+    def filter_unknown(self, unknown):
+        if self.use_yum:
+            filtered = set()
+            for unk in unknown:
+                try:
+                    if unk.startswith('rpmlib'):
+                        filtered.update(unk)
+                except AttributeError:
+                    try:
+                        if unk[0].startswith('rpmlib'):
+                            filtered.update(unk)
+                    except (IndexError, AttributeError):
+                        pass
+        else:
+            filtered = set([u for u in unknown if u.startswith('rpmlib')])
+        unknown.difference_update(filtered)
+
+    def setup_data(self, force_update=False):
+        if not self.use_yum:
+            Source.setup_data(self, force_update=force_update)
+
+    def get_repo_name(self, url_map):
+        if self.pulp_id:
+            return self.pulp_id
+        else:
+            return Source.get_repo_name(self, url_map)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
new file mode 100644
index 000000000..da5832e90
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -0,0 +1,263 @@
+import os
+import sys
+import time
+import copy
+import glob
+import shutil
+import lxml.etree
+import Bcfg2.Logger
+import Bcfg2.Server.Plugin
+from Bcfg2.Bcfg2Py3k import ConfigParser, urlopen
+from Bcfg2.Server.Plugins.Packages import Collection
+from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
+from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig
+
+class Packages(Bcfg2.Server.Plugin.Plugin,
+               Bcfg2.Server.Plugin.StructureValidator,
+               Bcfg2.Server.Plugin.Generator,
+               Bcfg2.Server.Plugin.Connector):
+    name = 'Packages'
+    conflicts = ['Pkgmgr']
+    experimental = True
+    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload']
+
+    def __init__(self, core, datastore):
+        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+        Bcfg2.Server.Plugin.StructureValidator.__init__(self)
+        Bcfg2.Server.Plugin.Generator.__init__(self)
+        Bcfg2.Server.Plugin.Connector.__init__(self)
+        Bcfg2.Server.Plugin.Probing.__init__(self)
+
+        self.sentinels = set()
+        self.cachepath = os.path.join(self.data, 'cache')
+        self.keypath = os.path.join(self.data, 'keys')
+        if not os.path.exists(self.keypath):
+            # create key directory if needed
+            os.makedirs(self.keypath)
+
+        # set up config files
+        self.config = PackagesConfig(self)
+        self.sources = PackagesSources(os.path.join(self.data, "sources.xml"),
+                                       self.cachepath, core.fam, self,
+                                       self.config)
+
+    def toggle_debug(self):
+        Bcfg2.Server.Plugin.Plugin.toggle_debug(self)
+        self.sources.toggle_debug()
+
+    @property
+    def disableResolver(self):
+        try:
+            return not self.config.getboolean("global", "resolver")
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            return False
+        except ValueError:
+            # for historical reasons we also accept "enabled" and
+            # "disabled", which are not handled according to the
+            # Python docs but appear to be handled properly by
+            # ConfigParser in at least some versions
+            return self.config.get("global", "resolver",
+                                   default="enabled").lower() == "disabled"
+
+    @property
+    def disableMetaData(self):
+        try:
+            return not self.config.getboolean("global", "resolver")
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            return False
+        except ValueError:
+            # for historical reasons we also accept "enabled" and
+            # "disabled"
+            return self.config.get("global", "metadata",
+                                   default="enabled").lower() == "disabled"
+
+    def create_config(self, entry, metadata):
+        """ create yum/apt config for the specified host """
+        attrib = {'encoding': 'ascii',
+                  'owner': 'root',
+                  'group': 'root',
+                  'type': 'file',
+                  'perms': '0644'}
+
+        collection = self._get_collection(metadata)
+        entry.text = collection.get_config()
+        for (key, value) in list(attrib.items()):
+            entry.attrib.__setitem__(key, value)
+
+    def HandleEntry(self, entry, metadata):
+        if entry.tag == 'Package':
+            collection = self._get_collection(metadata)
+            entry.set('version', 'auto')
+            entry.set('type', collection.ptype)
+        elif entry.tag == 'Path':
+            if (entry.get("name") == self.config.get("global", "yum_config",
+                                                     default="") or
+                entry.get("name") == self.config.get("global", "apt_config",
+                                                     default="")):
+                self.create_config(entry, metadata)
+
+    def HandlesEntry(self, entry, metadata):
+        if entry.tag == 'Package':
+            if self.config.getboolean("global", "magic_groups",
+                                      default=True) == True:
+                collection = self._get_collection(metadata)
+                if collection.magic_groups_match():
+                    return True
+            else:
+                return True
+        elif entry.tag == 'Path':
+            # managed entries for yum/apt configs
+            if (entry.get("name") == self.config.get("global", "yum_config",
+                                                     default="") or
+                entry.get("name") == self.config.get("global", "apt_config",
+                                                     default="")):
+                return True
+        return False
+
+    def validate_structures(self, metadata, structures):
+        '''Ensure client configurations include all needed prerequisites
+
+        Arguments:
+        metadata - client metadata instance
+        structures - a list of structure-stage entry combinations
+        '''
+        collection = self._get_collection(metadata)
+        indep = lxml.etree.Element('Independent')
+        self._build_packages(metadata, indep, structures,
+                             collection=collection)
+        collection.build_extra_structures(indep)
+        structures.append(indep)
+
+    def _build_packages(self, metadata, independent, structures,
+                        collection=None):
+        """ build list of packages that need to be included in the
+        specification by validate_structures() """
+        if self.disableResolver:
+            # Config requests no resolver
+            return
+
+        if collection is None:
+            collection = self._get_collection(metadata)
+        # initial is the set of packages that are explicitly specified
+        # in the configuration
+        initial = set()
+        # base is the set of initial packages with groups expanded
+        base = set()
+        to_remove = []
+        for struct in structures:
+            for pkg in struct.xpath('//Package | //BoundPackage'):
+                if pkg.get("name"):
+                    initial.add(pkg.get("name"))
+                elif pkg.get("group"):
+                    try:
+                        if pkg.get("type"):
+                            gpkgs = collection.get_group(pkg.get("group"),
+                                                         ptype=pkg.get("type"))
+                        else:
+                            gpkgs = collection.get_group(pkg.get("group"))
+                        base.update(gpkgs)
+                    except TypeError:
+                        raise
+                        self.logger.error("Could not resolve group %s" %
+                                          pkg.get("group"))
+                    to_remove.append(pkg)
+                else:
+                    self.logger.error("Packages: Malformed Package: %s" %
+                                      lxml.etree.tostring(pkg))
+        base.update(initial)
+        for el in to_remove:
+            el.getparent().remove(el)
+
+        packages, unknown = collection.complete(base)
+        if unknown:
+            self.logger.info("Packages: Got %d unknown entries" % len(unknown))
+            self.logger.info("Packages: %s" % list(unknown))
+        newpkgs = list(packages.difference(initial))
+        self.debug_log("Packages: %d initial, %d complete, %d new" %
+                       (len(initial), len(packages), len(newpkgs)))
+        newpkgs.sort()
+        for pkg in newpkgs:
+            lxml.etree.SubElement(independent, 'BoundPackage', name=pkg,
+                                  version='auto', type=collection.ptype,
+                                  origin='Packages')
+
+    def Refresh(self):
+        '''Packages.Refresh() => True|False\nReload configuration
+        specification and download sources\n'''
+        self._load_config(force_update=True)
+        return True
+
+    def Reload(self):
+        '''Packages.Refresh() => True|False\nReload configuration
+        specification and sources\n'''
+        self._load_config()
+        return True
+
+    def _load_config(self, force_update=False):
+        '''
+        Load the configuration data and setup sources
+
+        Keyword args:
+            force_update    Force downloading repo data
+        '''
+        self._load_sources(force_update)
+        self._load_gpg_keys(force_update)
+
+    def _load_sources(self, force_update):
+        """ Load sources from the config """
+        self.sentinels = set()
+        cachefiles = set()
+
+        for collection in list(Collection.collections.values()):
+            cachefiles.update(collection.cachefiles)
+            if not self.disableMetaData:
+                collection.setup_data(force_update)
+            self.sentinels.update(collection.basegroups)
+
+        Collection.clear_cache()
+
+        for source in self.sources:
+            cachefiles.add(source.cachefile)
+            if not self.disableMetaData:
+                source.setup_data(force_update)
+
+        for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")):
+            if cfile not in cachefiles:
+                try:
+                    if os.path.isdir(cfile):
+                        shutil.rmtree(cfile)
+                    else:
+                        os.unlink(cfile)
+                except OSError:
+                    err = sys.exc_info()[1]
+                    self.logger.error("Packages: Could not remove cache file "
+                                      "%s: %s" % (cfile, err))
+
+    def _load_gpg_keys(self, force_update):
+        """ Load gpg keys from the config """
+        keyfiles = []
+        keys = []
+        for source in self.sources:
+            for key in source.gpgkeys:
+                localfile = os.path.join(self.keypath,
+                                         os.path.basename(key.rstrip("/")))
+                if localfile not in keyfiles:
+                    keyfiles.append(localfile)
+                if ((force_update and key not in keys) or
+                    not os.path.exists(localfile)):
+                    self.logger.info("Packages: Downloading and parsing %s" % key)
+                    response = urlopen(key)
+                    open(localfile, 'w').write(response.read())
+                    keys.append(key)
+
+        for kfile in glob.glob(os.path.join(self.keypath, "*")):
+            if kfile not in keyfiles:
+                os.unlink(kfile)
+
+    def _get_collection(self, metadata):
+        return Collection.factory(metadata, self.sources, self.data,
+                                  debug=self.debug_flag)
+
+    def get_additional_data(self, metadata):
+        collection = self._get_collection(metadata)
+        return dict(sources=collection.get_additional_data())