improved Packages debug info, added packageresolve and packagesources bcfg2-info commands

4 files changed, 125 insertions, 75 deletions

diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt
index 701195ba9..93b5308be 100644
--- a/doc/server/plugins/generators/packages.txt
+++ b/doc/server/plugins/generators/packages.txt
@@ -526,50 +526,50 @@ as consumers, and will be bound to the appropriate repositories.
 Debugging unexpected behavior
 =============================
 
+.. versionadded:: 1.2.1
+
 Using bcfg2-info
 ----------------
 
 The dependency resolver used in Packages can be run in debug mode::
 
-
-    $ bcfg2-info
+    $ bcfg2-info packageresolve foo.example.com bcfg2-server zlib
     ...
-    Handled 20 events in 0.004s
-    > debug
-    dropping to python interpreter; press ^D to resume
-    ...
-    (debug_shell)
-    >>> m = self.build_metadata('ubik3')
-    >>> self.plugins['Packages'].complete(m, ['ssh'], debug=True)
-    Package ssh: adding new deps ['openssh-client', 'openssh-server']
-    Package openssh-server: adding new deps ['libc6', 'libcomerr2', 'libkrb53', 'libpam0g', 'libselinux1', 'libssl0.9.8
-    ', 'libwrap0', 'zlib1g', 'debconf', 'libpam-runtime', 'libpam-modules', 'adduser', 'dpkg', 'lsb-base']
-    Package debconf: adding new deps ['debconf-i18n']
-    Package libpam-modules: adding new deps ['libdb4.7']
-    Package openssh-client: adding new deps ['libedit2', 'libncurses5', 'passwd']
-    Package lsb-base: adding new deps ['sed', 'ncurses-bin']
-    Package adduser: adding new deps ['perl-base']
-    Package debconf-i18n: adding new deps ['liblocale-gettext-perl', 'libtext-iconv-perl', 'libtext-wrapi18n-perl', 'libtext-charwidth-perl']
-    Package passwd: adding new deps ['debianutils']
-    Package libtext-charwidth-perl: adding new deps ['perlapi-5.10.0']
-    VPackage perlapi-5.10.0: got provides ['perl-base']
-    Package libkrb53: adding new deps ['libkeyutils1']
-    Package libtext-iconv-perl: adding new deps ['perlapi-5.10.0']
-    Package libc6: adding new deps ['libgcc1', 'findutils']
-    Package libgcc1: adding new deps ['gcc-4.3-base']
-    (set(['debconf', 'libgcc1', 'lsb-base', 'libtext-wrapi18n-perl', 'libtext-iconv-perl', 'sed', 'passwd', 'findutils', 'libpam0g', 'openssh-client', 'debconf-i18n', 'libselinux1', 'zlib1g', 'adduser', 'libwrap0', 'ncurses-bin', 'libssl0.9.8', 'liblocale-gettext-perl', 'libkeyutils1', 'libpam-runtime', 'libpam-modules', 'openssh-server', 'libkrb53', 'ssh', 'libncurses5', 'libc6', 'libedit2', 'libcomerr2', 'dpkg', 'perl-base', 'libdb4.7', 'libtext-charwidth-perl', 'gcc-4.3-base', 'debianutils']), set([]), 'deb')
+    2 initial packages
+        bcfg2-server
+        zlib
+    54 new packages added
+        sqlite
+        less
+        libxml2
+        expat
+        ...
+    1 unknown packages
+        libglib-2.0.so.0()(64bit)
 
 This will show why the resolver is acting as it is. Replace
-``"ubik3"`` and ``['ssh']`` with a client name and list of packages,
-respectively. Also, a more polished interface to this functionality is
-coming as well.
-
-Each line starting with Package: <name> describes a set of new
-prerequisites pulled in by a package. Lines starting with VPackage <vname>
-describe provides entries and their mappings to required names. The last
-line describes the overall results of the resolver, with three fields:
-a list of packages that should be installed, a list of unresolved
-requirements, and a type for these packages.
+``foo.example.com`` and ``bcfg2-server`` with a client name and list
+of packages, respectively.
+
+Note that resolving a partial package list (as above) may result in
+more unknown entries than you'd have otherwise; some of the package
+drivers (Yum in particular) consider the full package list when
+resolving multiple providers, and will not be able to properly resolve
+some dependencies without a full package list.
+
+You can also view the sources applicable to a client::
+
+    $ bcfg2-info packagesources foo.example.com
+    ...
+    Name: centos-6-x86_64-updates
+      Type: yum
+      URL: http://mirror.example.com/centos-6-x86_64-updates
+      GPG Key(s): http://mirror.example.com/centos-6-x86_64-updates/RPM-GPG-KEY-CentOS-6
+    
+    Name: centos-6-x86_64-os
+      Type: yum
+      URL: http://mirror.example.com/centos-6-x86_64-os
+      GPG Key(s): http://mirror.example.com/centos-6-x86_64-os/RPM-GPG-KEY-CentOS-6
 
 Using bcfg2-server
 ------------------
diff --git a/src/lib/Server/Plugins/Packages/__init__.py b/src/lib/Server/Plugins/Packages/__init__.py
index 29f4cacd9..ae8dc54db 100644
--- a/src/lib/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Server/Plugins/Packages/__init__.py
@@ -75,16 +75,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
                   'type': 'file',
                   'perms': '0644'}
 
-        collection = Collection.factory(metadata, self.sources, self.data,
-                                        debug=self.debug_flag)
+        collection = self._get_collection(metadata)
         entry.text = collection.get_config()
         for (key, value) in list(attrib.items()):
             entry.attrib.__setitem__(key, value)
 
     def HandleEntry(self, entry, metadata):
         if entry.tag == 'Package':
-            collection = Collection.factory(metadata, self.sources, self.data,
-                                            debug=self.debug_flag)
+            collection = self._get_collection(metadata)
             entry.set('version', 'auto')
             entry.set('type', collection.ptype)
         elif entry.tag == 'Path':
@@ -98,9 +96,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         if entry.tag == 'Package':
             if self.config.getboolean("global", "magic_groups",
                                       default=True) == True:
-                collection = Collection.factory(metadata, self.sources,
-                                                self.data,
-                                                debug=self.debug_flag)
+                collection = self._get_collection(metadata)
                 if collection.magic_groups_match():
                     return True
             else:
@@ -121,8 +117,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         metadata - client metadata instance
         structures - a list of structure-stage entry combinations
         '''
-        collection = Collection.factory(metadata, self.sources, self.data,
-                                        debug=self.debug_flag)
+        collection = self._get_collection(metadata)
         indep = lxml.etree.Element('Independent')
         self._build_packages(metadata, indep, structures,
                              collection=collection)
@@ -138,8 +133,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             return
 
         if collection is None:
-            collection = Collection.factory(metadata, self.sources, self.data,
-                                            debug=self.debug_flag)
+            collection = self._get_collection(metadata)
         # initial is the set of packages that are explicitly specified
         # in the configuration
         initial = set()
@@ -256,7 +250,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             if kfile not in keyfiles:
                 os.unlink(kfile)
 
+    def _get_collection(self, metadata):
+        return Collection.factory(metadata, self.sources, self.data,
+                                  debug=self.debug_flag)
+
     def get_additional_data(self, metadata):
-        collection = Collection.factory(metadata, self.sources, self.data,
-                                        debug=self.debug_flag)
+        collection = self._get_collection(metadata)
         return dict(sources=collection.get_additional_data())
diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info
index 31b9ba0c6..808761d7e 100755
--- a/src/sbin/bcfg2-info
+++ b/src/sbin/bcfg2-info
@@ -45,6 +45,8 @@ event_debug - Display filesystem events as they are processed
 groups - List groups
 help - Print this list of available commands
 mappings <type*> <name*> - Print generator mappings for optional type and name
+packageresolve <hostname> <package> [<package>...] - Resolve the specified set of packages
+packagesources - Show package sources
 profile <command> <args> - Profile a single bcfg2-info command
 quit - Exit the bcfg2-info command line
 showentries <hostname> <type> - Show abstract configuration entries for a given host
@@ -454,7 +456,7 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core):
         try:
             meta = self.build_metadata(args)
         except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError:
-            print("Unable to find metadata for host %s" % client)
+            print("Unable to find metadata for host %s" % args)
             return
         structures = self.GetStructures(meta)
         for clist in [struct.findall('Path') for struct in structures]:
@@ -473,6 +475,54 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core):
                         continue
                     print(cand[0].name)
 
+    def do_packageresolve(self, args):
+        arglist = args.split(" ")
+        if len(arglist) < 2:
+            print("Usage: packageresolve <hostname> <package> [<package>...]")
+            return
+
+        hostname = arglist[0]
+        initial = arglist[1:]
+        metadata = self.build_metadata(hostname)
+        self.plugins['Packages'].toggle_debug()
+        collection = self.plugins['Packages']._get_collection(metadata)
+        packages, unknown = collection.complete(initial)
+        newpkgs = list(packages.difference(initial))
+        print("%d initial packages" % len(initial))
+        print("    %s" % "\n    ".join(initial))
+        print("%d new packages added" % len(newpkgs))
+        if newpkgs:
+            print("    %s" % "\n    ".join(newpkgs))
+        print("%d unknown packages" % len(unknown))
+        if unknown:
+            print("    %s" % "\n    ".join(unknown))
+
+    def do_packagesources(self, args):
+        try:
+            metadata = self.build_metadata(args)
+        except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError:
+            print("Unable to build metadata for host %s" % args)
+            return
+        collection = self.plugins['Packages']._get_collection(metadata)
+        for source in collection.sources:
+            # get_urls() loads url_map as a side-effect
+            source.get_urls()
+            for url_map in source.url_map:
+                if url_map['arch'] in metadata.groups:
+                    reponame = source.get_repo_name(url_map)
+                    print("Name: %s" % reponame)
+                    print("  Type: %s" % source.ptype)
+                    print("  URL: %s" % url_map['url'])
+                    if source.gpgkeys:
+                        print("  GPG Key(s): %s" % ", ".join(source.gpgkeys))
+                    else:
+                        print("  GPG Key(s): None")
+                    if len(source.blacklist):
+                        print("  Blacklist: %s" % ", ".join(source.blacklist))
+                    if len(source.whitelist):
+                        print("  Whitelist: %s" % ", ".join(source.whitelist))
+                    print("")
+        
     def do_profile(self, arg):
         """."""
         if not have_profile:
diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper
index 4d3ada741..78a5a479b 100755
--- a/src/sbin/bcfg2-yum-helper
+++ b/src/sbin/bcfg2-yum-helper
@@ -80,13 +80,12 @@ class DepSolver(object):
             matches = self.yumbase.pkgSack.returnNewestByName(name=package)
         except yum.Errors.PackageSackError:
             if not silent:
-                self.logger.warning("bcfg2-yum-helper: Package '%s' not found" %
+                self.logger.warning("Package '%s' not found" %
                                     self.get_package_name(package))
             matches = []
         except yum.Errors.RepoError:
             err = sys.exc_info()[1]
-            self.logger.error("bcfg2-yum-helper: Temporary failure loading "
-                              "metadata for '%s': %s" %
+            self.logger.error("Temporary failure loading metadata for %s: %s" %
                               (self.get_package_name(package), err))
             matches = []
 
@@ -105,7 +104,7 @@ class DepSolver(object):
             deps.difference_update([dep for dep in deps
                                         if pkg.checkPrco('provides', dep)])
         else:
-            self.logger.error("bcfg2-yum-helper: No package available: %s" %
+            self.logger.error("No package available: %s" %
                               self.get_package_name(package))
         return deps
 
@@ -118,15 +117,14 @@ class DepSolver(object):
                 self.yumbase.whatProvides(*required).returnNewestByNameArch()
         except yum.Errors.NoMoreMirrorsRepoError:
             err = sys.exc_info()[1]
-            self.logger.error("bcfg2-yum-helper: Temporary failure loading "
-                              "metadata for '%s': %s" %
+            self.logger.error("Temporary failure loading metadata for %s: %s" %
                               (self.get_package_name(required), err))
             return []
 
         if prov and not all:
             prov = self._filter_provides(required, prov)
         elif not prov and not silent:
-            self.logger.error("bcfg2-yum-helper: No package provides %s" %
+            self.logger.error("No package provides %s" %
                               self.get_package_name(required))
         return prov
 
@@ -138,12 +136,11 @@ class DepSolver(object):
             if self.yumbase.comps.has_group(group):
                 group = self.yumbase.comps.return_group(group)
             else:
-                self.logger.warning("bcfg2-yum-helper: %s is not a valid group"
-                                    % group)
+                self.logger.warning("%s is not a valid group" % group)
                 return []
         except yum.Errors.GroupsError:
             err = sys.exc_info()[1]
-            self.logger.warning("bcfg2-yum-helper: %s" % err)
+            self.logger.warning(err)
             return []
         
         if ptype == "default":
@@ -226,16 +223,14 @@ class DepSolver(object):
                 
                 if not self.is_package(package):
                     # try this package out as a requirement
-                    self.logger.debug("bcfg2-yum-helper: Adding requirement %s"
-                                      % package)
+                    self.logger.debug("Adding requirement %s" % package)
                     requires.add((package, None, (None, None, None)))
                     continue
 
                 packages.add(package)
                 reqs = set(self.get_deps(package)).difference(satisfied)
                 if reqs:
-                    self.logger.debug("bcfg2-yum-helper: Adding requirements "
-                                      "for %s: %s" %
+                    self.logger.debug("Adding requirements for %s: %s" %
                                       (package,
                                        ",".join([self.get_package_name(r)
                                                  for r in reqs])))
@@ -253,8 +248,8 @@ class DepSolver(object):
                     reqs_satisfied.add(req)
                     continue
                     
-                self.logger.debug("bcfg2-yum-helper: Handling requirement '%s'"
-                                  % self.get_package_name(req))
+                self.logger.debug("Handling requirement '%s'" %
+                                  self.get_package_name(req))
                 providers = list(set(self.get_provides(req)))
                 if len(providers) > 1:
                     # hopefully one of the providing packages is already
@@ -268,14 +263,21 @@ class DepSolver(object):
                         if len(best) == 1:
                             providers = best
                         elif not final_pass:
-                            # found no "best" package, so defer
+                            self.logger.debug("%s has multiple providers: %s" %
+                                              (self.get_package_name(req),
+                                               providers))
+                            self.logger.debug("No provider is obviously the "
+                                              "best; deferring")
                             providers = None
-                        # else: found no "best" package, but it's the
-                        # final pass, so include them all
+                        else:
+                            # found no "best" package, but it's the
+                            # final pass, so include them all
+                            self.logger.debug("Found multiple providers for %s,"
+                                              "including all" %
+                                              self.get_package_name(req))
                 
                 if providers:
-                    self.logger.debug("bcfg2-yum-helper: Requirement '%s' "
-                                      "satisfied by %s" %
+                    self.logger.debug("Requirement '%s' satisfied by %s" %
                                       (self.get_package_name(req),
                                        ",".join([self.get_package_name(p)
                                                  for p in providers])))
@@ -289,6 +291,8 @@ class DepSolver(object):
                     reqs_satisfied.add(req)
                 elif providers is not None:
                     # nothing provided this requirement at all
+                    self.logger.debug("Nothing provides %s" %
+                                      self.get_package_name(req))
                     unknown.add(req)
                     reqs_satisfied.add(req)
                 # else, defer
@@ -315,7 +319,7 @@ class DepSolver(object):
             # many files were deleted.  so useful.  thanks, yum.
             msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0]
             if not msg.startswith("0 "):
-                self.logger.info("bcfg2-yum-helper: %s" % msg)
+                self.logger.info(msg)
 
 
 def main():
@@ -327,12 +331,11 @@ def main():
     try:
         cmd = args[0]
     except IndexError:
-        logger.error("bcfg2-yum-helper: No command given")
+        logger.error("No command given")
         return 1
 
     if not os.path.exists(options.config):
-        logger.error("bcfg2-yum-helper: Config file %s not found" %
-                     options.config)
+        logger.error("Config file %s not found" % options.config)
         return 1
 
     depsolver = DepSolver(options.config, options.verbose)