Merge branch 'maint'

Conflicts:
	doc/appendix/guides/centos.txt
	doc/server/plugins/grouping/metadata.txt
	setup.py
	src/lib/Bcfg2/Client/Frame.py
	src/lib/Bcfg2/Client/Proxy.py
	src/lib/Bcfg2/Server/Lint/Genshi.py
	src/lib/Bcfg2/Server/Lint/Validate.py
	src/lib/Bcfg2/Server/Plugins/Bundler.py
	src/lib/Bcfg2/Server/Plugins/SSHbase.py
	src/sbin/bcfg2-lint

10 files changed, 106 insertions, 61 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index d8290d844..d114b0873 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -126,10 +126,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
 
 
 class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
-    """ Perform various bundle checks """
+    """ Perform various :ref:`Bundler
+    <server-plugins-structures-bundler-index>` checks. """
 
     def Run(self):
-        """ run plugin """
         self.missing_bundles()
         for bundle in self.core.plugins['Bundler'].entries.values():
             if self.HandlesFile(bundle.name):
@@ -143,7 +143,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
                 "genshi-extension-bundle": "error"}
 
     def missing_bundles(self):
-        """ find bundles listed in Metadata but not implemented in Bundler """
+        """ Find bundles listed in Metadata but not implemented in
+        Bundler. """
         if self.files is None:
             # when given a list of files on stdin, this check is
             # useless, so skip it
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
index 9042a979e..3e5508160 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -125,7 +125,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin,
 
 
 class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin):
-    """ bcfg2-lint plugin for GroupPatterns """
+    """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns
+    <server-plugins-grouping-grouppatterns>` patterns for validity.
+    This is simply done by trying to create a
+    :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for
+    each pattern, and catching exceptions and presenting them as
+    ``bcfg2-lint`` errors."""
 
     def Run(self):
         cfg = self.core.plugins['GroupPatterns'].config
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 49e36f72b..2bc82caa9 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                         self.debug_log("Client %s set as nonexistent group %s"
                                        % (client, group))
 
-    def set_profile(self, client, profile, addresspair):
+    def set_profile(self, client, profile,  # pylint: disable=W0221
+                    addresspair, require_public=True):
         """Set group parameter for provided client."""
         self.logger.info("Asserting client %s profile to %s" % (client,
                                                                 profile))
@@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
             self.logger.error(msg)
             raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
         group = self.groups[profile]
-        if not group.is_public:
+        if require_public and not group.is_public:
             msg = "Cannot set client %s to private group %s" % (client,
                                                                 profile)
             self.logger.error(msg)
@@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 pgroup = self.default
 
             if pgroup:
-                self.set_profile(client, pgroup, (None, None))
+                self.set_profile(client, pgroup, (None, None),
+                                 require_public=False)
                 profile = _add_group(pgroup)
             else:
                 msg = "Cannot add new client %s; no default group set" % client
@@ -1477,7 +1479,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
 
 
 class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
-    """ bcfg2-lint plugin for Metadata """
+    """ ``bcfg2-lint`` plugin for :ref:`Metadata
+    <server-plugins-grouping-metadata>`.  This checks for several things:
+
+    * ``<Client>`` tags nested inside other ``<Client>`` tags;
+    * Deprecated options (like ``location="floating"``);
+    * Profiles that don't exist, or that aren't profile groups;
+    * Groups or clients that are defined multiple times;
+    * Multiple default groups or a default group that isn't a profile
+      group.
+    """
 
     def Run(self):
         self.nested_clients()
@@ -1500,8 +1511,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
                 "default-is-not-profile": "error"}
 
     def deprecated_options(self):
-        """ check for the location='floating' option, which has been
-        deprecated in favor of floating='true' """
+        """ Check for the ``location='floating'`` option, which has
+        been deprecated in favor of ``floating='true'``. """
         if not hasattr(self.metadata, "clients_xml"):
             # using metadata database
             return
@@ -1519,8 +1530,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
                                (loc, floating, self.RenderXML(el)))
 
     def nested_clients(self):
-        """ check for a Client tag inside a Client tag, which doesn't
-        make any sense """
+        """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag,
+        which is either redundant or will never match. """
         groupdata = self.metadata.groups_xml.xdata
         for el in groupdata.xpath("//Client//Client"):
             self.LintError("nested-client-tags",
@@ -1528,8 +1539,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
                            (el.get("name"), self.RenderXML(el)))
 
     def bogus_profiles(self):
-        """ check for clients that have profiles that are either not
-        flagged as public groups in groups.xml, or don't exist """
+        """ Check for clients that have profiles that are either not
+        flagged as profile groups in ``groups.xml``, or don't exist. """
         if not hasattr(self.metadata, "clients_xml"):
             # using metadata database
             return
@@ -1547,20 +1558,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
                                (profile, client.get("name"), profile,
                                 self.RenderXML(client)))
 
-    def duplicate_groups(self):
-        """ check for groups that are defined twice.  We count a group
-        tag as a definition if it a) has profile or public set; or b)
-        has any children. """
-        self.duplicate_entries(
-            self.metadata.groups_xml.xdata.xpath("//Groups/Group") +
-            self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"),
-            "group",
-            include=lambda g: (g.get("profile") or
-                               g.get("public") or
-                               g.getchildren()))
-
     def duplicate_default_groups(self):
-        """ check for multiple default groups """
+        """ Check for multiple default groups. """
         defaults = []
         for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \
                 self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"):
@@ -1572,7 +1571,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
                            "\n".join(defaults))
 
     def duplicate_clients(self):
-        """ check for clients that are defined twice. """
+        """ Check for clients that are defined more than once. """
         if not hasattr(self.metadata, "clients_xml"):
             # using metadata database
             return
@@ -1580,17 +1579,34 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
             self.metadata.clients_xml.xdata.xpath("//Client"),
             "client")
 
-    def duplicate_entries(self, allentries, etype, include=None):
-        """ generic duplicate entry finder """
-        if include is None:
-            include = lambda e: True
+    def duplicate_groups(self):
+        """ Check for groups that are defined more than once.  We
+        count a group tag as a definition if it a) has profile or
+        public set; or b) has any children."""
+        allgroups = [
+            g
+            for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") +
+            self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group")
+            if g.get("profile") or g.get("public") or g.getchildren()]
+        self.duplicate_entries(allgroups, "group")
+
+    def duplicate_entries(self, allentries, etype):
+        """ Generic duplicate entry finder.
+
+        :param allentries: A list of all entries to check for
+                           duplicates.
+        :type allentries: list of lxml.etree._Element
+        :param etype: The entry type. This will be used to determine
+                      the error name (``duplicate-<etype>``) and for
+                      display to the end user.
+        :type etype: string
+        """
         entries = dict()
         for el in allentries:
-            if include(el):
-                if el.get("name") in entries:
-                    entries[el.get("name")].append(self.RenderXML(el))
-                else:
-                    entries[el.get("name")] = [self.RenderXML(el)]
+            if el.get("name") in entries:
+                entries[el.get("name")].append(self.RenderXML(el))
+            else:
+                entries[el.get("name")] = [self.RenderXML(el)]
         for ename, els in entries.items():
             if len(els) > 1:
                 self.LintError("duplicate-%s" % etype,
@@ -1598,7 +1614,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
                                (etype.title(), ename, "\n".join(els)))
 
     def default_is_profile(self):
-        """ ensure that the default group is a profile group """
+        """ Ensure that the default group is a profile group. """
         if (self.metadata.default and
             not self.metadata.groups[self.metadata.default].is_profile):
             xdata = \
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 48c580be1..dba56eed2 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -39,6 +39,11 @@ class AptCollection(Collection):
             else:
                 lines.append("deb %s %s %s" % (source.url, source.version,
                                                " ".join(source.components)))
+                if source.debsrc:
+                    lines.append("deb-src %s %s %s" %
+                                 (source.url,
+                                  source.version,
+                                  " ".join(source.components)))
                 lines.append("")
 
         return "\n".join(lines)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 3319dda78..767ac13ac 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -152,6 +152,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable):  # pylint: disable=R0902
         #: this source
         self.whitelist = [item.text for item in xsource.findall('Whitelist')]
 
+        #: Whether or not to include deb-src lines in the generated APT
+        #: configuration
+        self.debsrc = xsource.get('debsrc', 'false') == 'true'
+
         #: A dict of repository options that will be included in the
         #: configuration generated on the server side (if such is
         #: applicable; most backends do not generate any sort of
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 052c362ab..567a16c40 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -9,7 +9,7 @@ import shutil
 import lxml.etree
 import Bcfg2.Logger
 import Bcfg2.Server.Plugin
-from Bcfg2.Compat import ConfigParser, urlopen, HTTPError
+from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError
 from Bcfg2.Server.Plugins.Packages.Collection import Collection, \
     get_collection_class
 from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
@@ -445,7 +445,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
                     try:
                         open(localfile, 'w').write(urlopen(key).read())
                         keys.append(key)
-                    except HTTPError:
+                    except (URLError, HTTPError):
                         err = sys.exc_info()[1]
                         self.logger.error("Packages: Error downloading %s: %s"
                                           % (key, err))
diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
index 7dac907e1..a1dcb575f 100644
--- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
+++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
@@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir):
 
 
 class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin):
-    """ find duplicate Pkgmgr entries with the same priority """
+    """ Find duplicate :ref:`Pkgmgr
+    <server-plugins-generators-pkgmgr>` entries with the same
+    priority. """
+
     def Run(self):
         pset = set()
         for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr',
@@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin):
                     # check if package is already listed with same
                     # priority, type, grp
                     if ptuple in pset:
-                        self.LintError("duplicate-package",
-                                       "Duplicate Package %s, priority:%s, type:%s" %
-                                         (pkg.get('name'), priority, ptype))
+                        self.LintError(
+                            "duplicate-package",
+                            "Duplicate Package %s, priority:%s, type:%s" %
+                            (pkg.get('name'), priority, ptype))
                     else:
                         pset.add(ptuple)
-    
+
     @classmethod
     def Errors(cls):
-        return {"duplicate-packages":"error"}
+        return {"duplicate-packages": "error"}
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 7e4935d74..e97607093 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -65,7 +65,7 @@ class ProbeData(str):  # pylint: disable=E0012,R0924
     .json, and .yaml properties to provide convenient ways to use
     ProbeData objects as XML, JSON, or YAML data """
     def __new__(cls, data):
-        return str.__new__(cls, data.encode('utf-8'))
+        return str.__new__(cls, data)
 
     def __init__(self, data):  # pylint: disable=W0613
         str.__init__(self)
@@ -224,15 +224,9 @@ class Probes(Bcfg2.Server.Plugin.Probing,
                 lxml.etree.SubElement(top, 'Client', name=client,
                                       timestamp=str(int(probedata.timestamp)))
             for probe in sorted(probedata):
-                try:
-                    lxml.etree.SubElement(
-                        ctag, 'Probe', name=probe,
-                        value=str(
-                            self.probedata[client][probe]).decode('utf-8'))
-                except AttributeError:
-                    lxml.etree.SubElement(
-                        ctag, 'Probe', name=probe,
-                        value=str(self.probedata[client][probe]))
+                lxml.etree.SubElement(
+                    ctag, 'Probe', name=probe,
+                    value=self.probedata[client][probe])
             for group in sorted(self.cgroups[client]):
                 lxml.etree.SubElement(ctag, "Group", name=group)
         try:
diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
index 1264fd1cf..84dcf2780 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
@@ -173,8 +173,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
                 for name in names[cmeta.hostname]:
                     newnames.add(name.split('.')[0])
                     try:
-                        newips.add(self.get_ipcache_entry(name)[0])
-                    except PluginExecutionError:
+                        newips.update(self.get_ipcache_entry(name)[0])
+                    except:  # pylint: disable=W0702
                         continue
                 names[cmeta.hostname].update(newnames)
                 names[cmeta.hostname].update(cmeta.addresses)
@@ -290,7 +290,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
         else:
             # need to add entry
             try:
-                ipaddr = socket.gethostbyname(client)
+                ipaddr = set([info[4][0]
+                              for info in socket.getaddrinfo(client, None)])
                 self.ipcache[client] = (ipaddr, client)
                 return (ipaddr, client)
             except socket.gaierror:
diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
index e834759c2..050ba3b3e 100644
--- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
+++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
@@ -96,7 +96,18 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin,
 
 
 class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin):
-    """ find duplicate Pkgmgr entries with the same priority """
+    """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper
+    <server-plugins-connectors-templatehelper>` modules are valid.
+    This can check for:
+
+    * A TemplateHelper module that cannot be imported due to syntax or
+      other compile-time errors;
+    * A TemplateHelper module that does not have an ``__export__``
+      attribute, or whose ``__export__`` is not a list;
+    * Bogus symbols listed in ``__export__``, including symbols that
+      don't exist, that are reserved, or that start with underscores.
+    """
+
     def __init__(self, *args, **kwargs):
         Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
         self.reserved_keywords = dir(HelperModule("foo.py"))
@@ -107,7 +118,11 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin):
                 self.check_helper(helper.name)
 
     def check_helper(self, helper):
-        """ check a helper module for export errors """
+        """ Check a single helper module.
+
+        :param helper: The filename of the helper module
+        :type helper: string
+        """
         module_name = MODULE_RE.search(helper).group(1)
 
         try: