batch up get_group calls in Packages to improve performance (#1101)

4 files changed, 45 insertions, 14 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index 9cea9da48..1e269167b 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -104,6 +104,14 @@ class Collection(Bcfg2.Server.Plugin.Debuggable):
             cachefiles.add(source.cachefile)
         return list(cachefiles)
 
+    def get_groups(self, grouplist):
+        """ provided since some backends may be able to query multiple
+        groups at once faster than serially """
+        rv = dict()
+        for group, ptype in grouplist:
+            rv[group] = self.get_group(group, ptype)
+        return rv
+
     def get_group(self, group, ptype=None):
         for source in self.sources:
             pkgs = source.get_group(self.metadata, group, ptype=ptype)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 779614dbb..a1499e263 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -367,6 +367,22 @@ class YumCollection(Collection):
             # for API completeness
             return self.call_helper("get_provides", package)
 
+    def get_groups(self, grouplist):
+        if not self.use_yum:
+            self.logger.warning("Packages: Package groups are not supported by "
+                                "Bcfg2's internal Yum dependency generator")
+            return []
+
+        gdicts = []
+        for group, ptype in grouplist:
+            if group.startswith("@"):
+                group = group[1:]
+            if not ptype:
+                ptype = "default"
+            gdicts.append(dict(group=group, type=ptype))
+
+        return self.call_helper("get_groups", gdicts)
+
     def get_group(self, group, ptype="default"):
         if not self.use_yum:
             self.logger.warning("Packages: Package groups are not supported by "
@@ -376,8 +392,7 @@ class YumCollection(Collection):
         if group.startswith("@"):
             group = group[1:]
 
-        pkgs = self.call_helper("get_group", dict(group=group, type=ptype))
-        return pkgs
+        return self.call_helper("get_group", dict(group=group, type=ptype))
 
     def complete(self, packagelist):
         if not self.use_yum:
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index e76f648d0..31670d3a3 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -167,26 +167,23 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         # essential pkgs are those marked as such by the distribution
         essential = collection.get_essential()
         to_remove = []
+        groups = []
         for struct in structures:
             for pkg in struct.xpath('//Package | //BoundPackage'):
                 if pkg.get("name"):
                     initial.add(pkg.get("name"))
                 elif pkg.get("group"):
-                    try:
-                        if pkg.get("type"):
-                            gpkgs = collection.get_group(pkg.get("group"),
-                                                         ptype=pkg.get("type"))
-                        else:
-                            gpkgs = collection.get_group(pkg.get("group"))
-                        base.update(gpkgs)
-                    except TypeError:
-                        raise
-                        self.logger.error("Could not resolve group %s" %
-                                          pkg.get("group"))
+                    groups.append((pkg.get("group"),
+                                   pkg.get("type")))
                     to_remove.append(pkg)
                 else:
                     self.logger.error("Packages: Malformed Package: %s" %
                                       lxml.etree.tostring(pkg))
+
+        gpkgs = collection.get_groups(groups)
+        for group, pkgs in gpkgs.items():
+            base.update(pkgs)
+
         base.update(initial | essential)
         for el in to_remove:
             el.getparent().remove(el)
diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper
index 2da7c6336..186eefe7a 100755
--- a/src/sbin/bcfg2-yum-helper
+++ b/src/sbin/bcfg2-yum-helper
@@ -134,7 +134,7 @@ class DepSolver(object):
             if self.yumbase.comps.has_group(group):
                 group = self.yumbase.comps.return_group(group)
             else:
-                self.logger.warning("%s is not a valid group" % group)
+                self.logger.error("%s is not a valid group" % group)
                 return []
         except yum.Errors.GroupsError:
             err = sys.exc_info()[1]
@@ -365,6 +365,17 @@ def main():
         else:
             packages = depsolver.get_group(data['group'])
         print json.dumps(list(packages))
+    elif cmd == "get_groups":
+        data = json.loads(sys.stdin.read())
+        rv = dict()
+        for gdata in data:
+            if "type" in gdata:
+                packages = depsolver.get_group(gdata['group'],
+                                               ptype=gdata['type'])
+            else:
+                packages = depsolver.get_group(gdata['group'])
+            rv[gdata['group']] = list(packages)
+        print json.dumps(rv)
     elif cmd == "is_package":
         package = json.loads(sys.stdin.read())
         print json.dumps(getattr(depsolver, cmd)(package))