Plugins/Packages: ability to overwrite recommended flag per package

7 files changed, 75 insertions, 25 deletions

diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt
index 8b317552f..594f50ae4 100644
--- a/doc/server/plugins/generators/packages.txt
+++ b/doc/server/plugins/generators/packages.txt
@@ -217,10 +217,6 @@ something like this:
 
         <Source type="apt" recommended="true" ...>
 
-    .. warning:: You must regenerate the Packages cache when adding or
-                 removing the recommended attribute (``bcfg2-admin xcmd
-                 Packages.Refresh``).
-
     .. [#f1] Bcfg2 will by default add **Essential** packages to the
              client specification. You can disable this behavior by
              setting the :xml:attribute:`SourceType:essential`
diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd
index c76c52824..7ad7606b2 100644
--- a/schemas/pkgtype.xsd
+++ b/schemas/pkgtype.xsd
@@ -54,6 +54,15 @@
         </xsd:documentation>
       </xsd:annotation>
     </xsd:attribute>
+    <xsd:attribute name="recommended" type="xsd:boolean">
+      <xsd:annotation>
+        <xsd:documentation>
+          Whether also the recommended packages should be installed.
+          This is currently only used with the :ref:`APT
+          &lt;client-tools-apt&gt;` driver.
+        </xsd:documentation>
+      </xsd:annotation>
+    </xsd:attribute>
     <xsd:attributeGroup ref="py:genshiAttrs"/>
   </xsd:complexType>
 
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index c1d53a78e..434e175d8 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -71,11 +71,9 @@ class AptSource(Source):
 
     def read_files(self):
         bdeps = dict()
+        brecs = dict()
         bprov = dict()
         self.essentialpkgs = set()
-        depfnames = ['Depends', 'Pre-Depends']
-        if self.recommended:
-            depfnames.append('Recommends')
         for fname in self.files:
             if not self.rawurl:
                 barch = [x
@@ -87,6 +85,7 @@ class AptSource(Source):
                 barch = self.arches[0]
             if barch not in bdeps:
                 bdeps[barch] = dict()
+                brecs[barch] = dict()
                 bprov[barch] = dict()
             try:
                 reader = gzip.GzipFile(fname)
@@ -101,9 +100,10 @@ class AptSource(Source):
                     pkgname = words[1].strip().rstrip()
                     self.pkgnames.add(pkgname)
                     bdeps[barch][pkgname] = []
+                    brecs[barch][pkgname] = []
                 elif words[0] == 'Essential' and self.essential:
                     self.essentialpkgs.add(pkgname)
-                elif words[0] in depfnames:
+                elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']:
                     vindex = 0
                     for dep in words[1].split(','):
                         if '|' in dep:
@@ -114,17 +114,24 @@ class AptSource(Source):
                                                              barch,
                                                              vindex)
                             vindex += 1
-                            bdeps[barch][pkgname].append(dyn_dname)
+
+                            if words[0] == 'Recommends':
+                                brecs[barch][pkgname].append(dyn_dname)
+                            else:
+                                bdeps[barch][pkgname].append(dyn_dname)
                             bprov[barch][dyn_dname] = set(cdeps)
                         else:
                             raw_dep = re.sub(r'\(.*\)', '', dep)
                             raw_dep = raw_dep.rstrip().strip()
-                            bdeps[barch][pkgname].append(raw_dep)
+                            if words[0] == 'Recommends':
+                                brecs[barch][pkgname].append(raw_dep)
+                            else:
+                                bdeps[barch][pkgname].append(raw_dep)
                 elif words[0] == 'Provides':
                     for pkg in words[1].split(','):
                         dname = pkg.rstrip().strip()
                         if dname not in bprov[barch]:
                             bprov[barch][dname] = set()
                         bprov[barch][dname].add(pkgname)
-        self.process_files(bdeps, bprov)
+        self.process_files(bdeps, bprov, brecs)
     read_files.__doc__ = Source.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index 8b20df58a..9bd384167 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -289,7 +289,7 @@ class Collection(list, Debuggable):
         return any(source.is_virtual_package(self.metadata, package)
                    for source in self)
 
-    def get_deps(self, package):
+    def get_deps(self, package, recs=None):
         """ Get a list of the dependencies of the given package.
 
         The base implementation simply aggregates the results of
@@ -299,9 +299,14 @@ class Collection(list, Debuggable):
         :type package: string
         :returns: list of strings, but see :ref:`pkg-objects`
         """
+        recommended = None
+        if recs and package in recs:
+            recommended = recs[package]
+
         for source in self:
             if source.is_package(self.metadata, package):
-                return source.get_deps(self.metadata, package)
+                return source.get_deps(self.metadata, package, recommended)
+
         return []
 
     def get_essential(self):
@@ -465,7 +470,7 @@ class Collection(list, Debuggable):
         return list(complete.difference(initial))
 
     @track_statistics()
-    def complete(self, packagelist):  # pylint: disable=R0912,R0914
+    def complete(self, packagelist, recommended=None):  # pylint: disable=R0912,R0914
         """ Build a complete list of all packages and their dependencies.
 
         :param packagelist: Set of initial packages computed from the
@@ -529,7 +534,7 @@ class Collection(list, Debuggable):
                 self.debug_log("Packages: handling package requirement %s" %
                                (current,))
                 packages.add(current)
-                deps = self.get_deps(current)
+                deps = self.get_deps(current, recommended)
                 newdeps = set(deps).difference(examined)
                 if newdeps:
                     self.debug_log("Packages: Package %s added requirements %s"
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 4b6130f72..eadc4e075 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -246,6 +246,10 @@ class Source(Debuggable):  # pylint: disable=R0902
         #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
         self.provides = dict()
 
+        #: A dict of ``<package name>`` -> ``<list of recommended
+        #: symbols>``.  This will not necessarily be populated.
+        self.recommends = dict()
+
         #: The file (or directory) used for this source's cache data
         self.cachefile = os.path.join(self.basepath,
                                       "cache-%s" % self.cachekey)
@@ -310,7 +314,7 @@ class Source(Debuggable):  # pylint: disable=R0902
         :raises: cPickle.UnpicklingError - If the saved data is corrupt """
         data = open(self.cachefile, 'rb')
         (self.pkgnames, self.deps, self.provides,
-         self.essentialpkgs) = cPickle.load(data)
+         self.essentialpkgs, self.recommends) = cPickle.load(data)
 
     def save_state(self):
         """ Save state to :attr:`cachefile`.  If caching and
@@ -318,7 +322,7 @@ class Source(Debuggable):  # pylint: disable=R0902
         does not need to be implemented. """
         cache = open(self.cachefile, 'wb')
         cPickle.dump((self.pkgnames, self.deps, self.provides,
-                      self.essentialpkgs), cache, 2)
+                      self.essentialpkgs, self.recommends), cache, 2)
         cache.close()
 
     @track_statistics()
@@ -513,13 +517,13 @@ class Source(Debuggable):  # pylint: disable=R0902
         as its final step."""
         pass
 
-    def process_files(self, dependencies, provides):
+    def process_files(self, dependencies, provides, recommends=dict()):
         """ Given dicts of depends and provides generated by
         :func:`read_files`, this generates :attr:`deps` and
         :attr:`provides` and calls :func:`save_state` to save the
         cached data to disk.
 
-        Both arguments are dicts of dicts of lists.  Keys are the
+        All arguments are dicts of dicts of lists.  Keys are the
         arches of packages contained in this source; values are dicts
         whose keys are package names and values are lists of either
         dependencies for each package the symbols provided by each
@@ -531,14 +535,20 @@ class Source(Debuggable):  # pylint: disable=R0902
         :param provides: A dict of symbols provided by packages in
                         this repository.
         :type provides: dict; see above.
+        :param recommends: A dict of recommended dependencies
+                           found for this source.
+        :type recommends: dict; see above.
         """
         self.deps['global'] = dict()
+        self.recommends['global'] = dict()
         self.provides['global'] = dict()
         for barch in dependencies:
             self.deps[barch] = dict()
+            self.recommends[barch] = dict()
             self.provides[barch] = dict()
         for pkgname in self.pkgnames:
             pset = set()
+            rset = set()
             for barch in dependencies:
                 if pkgname not in dependencies[barch]:
                     dependencies[barch][pkgname] = []
@@ -548,6 +558,17 @@ class Source(Debuggable):  # pylint: disable=R0902
             else:
                 for barch in dependencies:
                     self.deps[barch][pkgname] = dependencies[barch][pkgname]
+
+            for barch in recommends:
+                if pkgname not in recommends[barch]:
+                    recommends[barch][pkgname] = []
+                rset.add(tuple(recommends[barch][pkgname]))
+            if len(rset) == 1:
+                self.recommends['global'][pkgname] = rset.pop()
+            else:
+                for barch in recommends:
+                    self.recommends[barch][pkgname] = recommends[barch][pkgname]
+
         provided = set()
         for bprovided in list(provides.values()):
             provided.update(set(bprovided))
@@ -655,17 +676,24 @@ class Source(Debuggable):  # pylint: disable=R0902
         """
         return ['global'] + [a for a in self.arches if a in metadata.groups]
 
-    def get_deps(self, metadata, package):
+    def get_deps(self, metadata, package, recommended=None):
         """ Get a list of the dependencies of the given package.
 
         :param package: The name of the symbol
         :type package: string
         :returns: list of strings
         """
+        recs = []
+        if ((recommended is None and self.recommended) or
+            (recommended and recommended.lower() == 'true')):
+            for arch in self.get_arches(metadata):
+                if package in self.recommends[arch]:
+                    recs.extend(self.recommends[arch][package])
+
         for arch in self.get_arches(metadata):
             if package in self.deps[arch]:
-                return self.deps[arch][package]
-        return []
+                recs.extend(self.deps[arch][package])
+        return recs
 
     def get_provides(self, metadata, package):
         """ Get a list of all symbols provided by the given package.
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 2dc44d0c4..f26ded4c5 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -857,7 +857,7 @@ class YumCollection(Collection):
         return new
 
     @track_statistics()
-    def complete(self, packagelist):
+    def complete(self, packagelist, recommended=None):
         """ Build a complete list of all packages and their dependencies.
 
         When using the Python yum libraries, this defers to the
@@ -875,7 +875,7 @@ class YumCollection(Collection):
                   resolved.
         """
         if not self.use_yum:
-            return Collection.complete(self, packagelist)
+            return Collection.complete(self, packagelist, recommended)
 
         lock = FileLock(os.path.join(self.cachefile, "lock"))
         slept = 0
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 49f64bdf3..59cd3aafe 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -357,10 +357,15 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         initial = set()
         to_remove = []
         groups = []
+        recommended = dict()
+
         for struct in structures:
             for pkg in struct.xpath('//Package | //BoundPackage'):
                 if pkg.get("name"):
                     initial.update(collection.packages_from_entry(pkg))
+
+                    if pkg.get("recommended"):
+                        recommended[pkg.get("name")] = pkg.get("recommended")
                 elif pkg.get("group"):
                     groups.append((pkg.get("group"),
                                    pkg.get("type")))
@@ -399,7 +404,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets",
                                           collection.cachekey)
         if pkey not in pcache:
-            pcache[pkey] = collection.complete(base)
+            pcache[pkey] = collection.complete(base, recommended)
         packages, unknown = pcache[pkey]
         if unknown:
             self.logger.info("Packages: Got %d unknown entries" % len(unknown))