Merge branch 'maint'

Signed-off-by: Sol Jerome <sol.jerome@gmail.com>

Conflicts:
	src/lib/Bcfg2/Client/Tools/__init__.py
	src/lib/Bcfg2/Server/BuiltinCore.py
	src/lib/Bcfg2/Server/Plugins/Metadata.py
	src/lib/Bcfg2/Server/Plugins/NagiosGen.py
	src/lib/Bcfg2/Server/Plugins/Probes.py
	src/lib/Bcfg2/Server/SSLServer.py
	tools/README

1 files changed, 149 insertions, 99 deletions

diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 17352dedc..db104b27e 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -545,15 +545,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         self.raliases = {}
         # mapping of groupname -> MetadataGroup object
         self.groups = {}
-        # mappings of predicate -> MetadataGroup object
+        # mappings of groupname -> [predicates]
         self.group_membership = dict()
         self.negated_groups = dict()
+        # list of group names in document order
+        self.ordered_groups = []
         # mapping of hostname -> version string
         if self._use_db:
             self.versions = ClientVersions(core,  # pylint: disable=E1102
                                            datastore)
         else:
             self.versions = dict()
+
         self.uuid = {}
         self.session_cache = {}
         self.cache = Cache("Metadata")
@@ -561,7 +564,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         self.pdirty = False
         self.password = Bcfg2.Options.setup.password
         self.query = MetadataQuery(core.build_metadata,
-                                   lambda: list(self.clients),
+                                   self.list_clients,
                                    self.get_client_names_by_groups,
                                    self.get_client_names_by_profiles,
                                    self.get_all_group_names,
@@ -851,51 +854,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         if self._use_db:
             self.clients = self.list_clients()
 
+    def _get_condition(self, element):
+        """ Return a predicate that returns True if a client meets
+        the condition specified in the given Group or Client
+        element """
+        negate = element.get('negate', 'false').lower() == 'true'
+        pname = element.get("name")
+        if element.tag == 'Group':
+            return lambda c, g, _: negate != (pname in g)
+        elif element.tag == 'Client':
+            return lambda c, g, _: negate != (pname == c)
+
+    def _get_category_condition(self, grpname):
+        """ get a predicate that returns False if a client is already
+        a member of a group in the given group's category, True
+        otherwise"""
+        return lambda client, _, categories: \
+            bool(self._check_category(client, grpname, categories))
+
+    def _aggregate_conditions(self, conditions):
+        """ aggregate all conditions on a given group declaration
+        into a single predicate """
+        return lambda client, groups, cats: \
+            all(cond(client, groups, cats) for cond in conditions)
+
     def _handle_groups_xml_event(self, _):  # pylint: disable=R0912
         """ re-read groups.xml on any event on it """
         self.groups = {}
 
-        # these three functions must be separate functions in order to
-        # ensure that the scope is right for the closures they return
-        def get_condition(element):
-            """ Return a predicate that returns True if a client meets
-            the condition specified in the given Group or Client
-            element """
-            negate = element.get('negate', 'false').lower() == 'true'
-            pname = element.get("name")
-            if element.tag == 'Group':
-                return lambda c, g, _: negate != (pname in g)
-            elif element.tag == 'Client':
-                return lambda c, g, _: negate != (pname == c)
-
-        def get_category_condition(category, gname):
-            """ get a predicate that returns False if a client is
-            already a member of a group in the given category, True
-            otherwise """
-            def in_cat(client, groups, categories):  # pylint: disable=W0613
-                """ return True if the client is already a member of a
-                group in the category given in the enclosing function,
-                False otherwise """
-                if category in categories:
-                    if (gname not in self.groups or
-                        client not in self.groups[gname].warned):
-                        self.logger.warning("%s: Group %s suppressed by "
-                                            "category %s; %s already a member "
-                                            "of %s" %
-                                            (self.name, gname, category,
-                                             client, categories[category]))
-                        if gname in self.groups:
-                            self.groups[gname].warned.append(client)
-                    return False
-                return True
-            return in_cat
-
-        def aggregate_conditions(conditions):
-            """ aggregate all conditions on a given group declaration
-            into a single predicate """
-            return lambda client, groups, cats: \
-                all(cond(client, groups, cats) for cond in conditions)
-
         # first, we get a list of all of the groups declared in the
         # file.  we do this in two stages because the old way of
         # parsing groups.xml didn't support nested groups; in the old
@@ -921,6 +907,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
 
         self.group_membership = dict()
         self.negated_groups = dict()
+        self.ordered_groups = []
 
         # confusing loop condition; the XPath query asks for all
         # elements under a Group tag under a Groups tag; that is
@@ -931,28 +918,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         # XPath.  We do the same thing for Client tags.
         for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \
                 self.groups_xml.xdata.xpath("//Groups/Client//*"):
-            if ((el.tag != 'Group' and el.tag != 'Client') or
-                el.getchildren()):
+            if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren():
                 continue
 
             conditions = []
             for parent in el.iterancestors():
-                cond = get_condition(parent)
+                cond = self._get_condition(parent)
                 if cond:
                     conditions.append(cond)
 
             gname = el.get("name")
             if el.get("negate", "false").lower() == "true":
-                self.negated_groups[aggregate_conditions(conditions)] = \
-                    self.groups[gname]
+                self.negated_groups.setdefault(gname, [])
+                self.negated_groups[gname].append(
+                    self._aggregate_conditions(conditions))
             else:
                 if self.groups[gname].category:
-                    conditions.append(
-                        get_category_condition(self.groups[gname].category,
-                                               gname))
+                    conditions.append(self._get_category_condition(gname))
 
-                self.group_membership[aggregate_conditions(conditions)] = \
-                    self.groups[gname]
+                if gname not in self.ordered_groups:
+                    self.ordered_groups.append(gname)
+                self.group_membership.setdefault(gname, [])
+                self.group_membership[gname].append(
+                    self._aggregate_conditions(conditions))
         self.states['groups.xml'] = True
 
     def HandleEvent(self, event):
@@ -962,6 +950,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 # clear the entire cache when we get an event for any
                 # metadata file
                 self.cache.expire()
+
+                # clear out the list of category suppressions that
+                # have been warned about, since this may change when
+                # clients.xml or groups.xml changes.
+                for group in self.groups.values():
+                    group.warned = []
                 event_handler(event)
 
         if False not in list(self.states.values()) and self.debug_flag:
@@ -1100,22 +1094,77 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
             categories = dict()
         while numgroups != len(groups):
             numgroups = len(groups)
-            for predicate, group in self.group_membership.items():
-                if group.name in groups:
+            newgroups = set()
+            removegroups = set()
+            for grpname in self.ordered_groups:
+                if grpname in groups:
                     continue
-                if predicate(client, groups, categories):
-                    groups.add(group.name)
-                    if group.category:
-                        categories[group.category] = group.name
-            for predicate, group in self.negated_groups.items():
-                if group.name not in groups:
+                if any(p(client, groups, categories)
+                       for p in self.group_membership[grpname]):
+                    newgroups.add(grpname)
+                    if (grpname in self.groups and
+                        self.groups[grpname].category):
+                        categories[self.groups[grpname].category] = grpname
+            groups.update(newgroups)
+            for grpname, predicates in self.negated_groups.items():
+                if grpname not in groups:
                     continue
-                if predicate(client, groups, categories):
-                    groups.remove(group.name)
-                    if group.category:
-                        del categories[group.category]
+                if any(p(client, groups, categories) for p in predicates):
+                    removegroups.add(grpname)
+                    if (grpname in self.groups and
+                        self.groups[grpname].category):
+                        del categories[self.groups[grpname].category]
+            groups.difference_update(removegroups)
         return (groups, categories)
 
+    def _check_category(self, client, grpname, categories):
+        """ Determine if the given client is already a member of a
+        group in the same category as the named group.
+
+        The return value is one of three possibilities:
+
+        * If the client is already a member of a group in the same
+          category, then False is returned (i.e., the category check
+          failed);
+        * If the group is not in any categories, then True is returned;
+        * If the group is not a member of a group in the category,
+          then the name of the category is returned.  This makes it
+          easy to add the category to the ClientMetadata object (or
+          other category list).
+
+        If a pure boolean value is required, you can do
+        ``bool(self._check_category(...))``.
+        """
+        if grpname not in self.groups:
+            return True
+        category = self.groups[grpname].category
+        if not category:
+            return True
+        if category in categories:
+            if client not in self.groups[grpname].warned:
+                self.logger.warning("%s: Group %s suppressed by category %s; "
+                                    "%s already a member of %s" %
+                                    (self.name, grpname, category,
+                                     client, categories[category]))
+                self.groups[grpname].warned.append(client)
+            return False
+        return category
+
+    def _check_and_add_category(self, client, grpname, categories):
+        """ If the client is not a member of a group in the same
+        category as the named group, then the category is added to
+        ``categories``.
+        :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used
+        to determine if the category can be added.
+
+        If the category check failed, returns False; otherwise,
+        returns True. """
+        rv = self._check_category(client, grpname, categories)
+        if rv and rv is not True:
+            categories[rv] = grpname
+            return True
+        return rv
+
     def get_initial_metadata(self, client):  # pylint: disable=R0914,R0912
         """Return the metadata for a given client."""
         if False in list(self.states.values()):
@@ -1137,30 +1186,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
             Handles setting categories and category suppression.
             Returns the new profile for the client (which might be
             unchanged). """
-            groups.add(grpname)
             if grpname in self.groups:
-                group = self.groups[grpname]
-                category = group.category
-                if category:
-                    if category in categories:
-                        self.logger.warning("%s: Group %s suppressed by "
-                                            "category %s; %s already a member "
-                                            "of %s" %
-                                            (self.name, grpname, category,
-                                             client, categories[category]))
-                        return
-                    categories[category] = grpname
-                if not profile and group.is_profile:
+                if not self._check_and_add_category(client, grpname,
+                                                    categories):
+                    return profile
+                groups.add(grpname)
+                if not profile and self.groups[grpname].is_profile:
                     return grpname
                 else:
                     return profile
+            else:
+                groups.add(grpname)
+                return profile
 
         if client not in self.clients:
             pgroup = None
             if client in self.clientgroups:
                 pgroup = self.clientgroups[client][0]
+                self.debug_log("%s: Adding new client with profile %s" %
+                               (self.name, pgroup))
             elif self.default:
                 pgroup = self.default
+                self.debug_log("%s: Adding new client with default profile %s"
+                               % (self.name, pgroup))
 
             if pgroup:
                 self.set_profile(client, pgroup, (None, None),
@@ -1177,6 +1225,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
                 self.groups[cgroup] = MetadataGroup(cgroup)
             profile = _add_group(cgroup)
 
+        # we do this before setting the default because there may be
+        # groups set in <Client> tags in groups.xml that we want to
+        # set
         groups, categories = self._merge_groups(client, groups,
                                                 categories=categories)
 
@@ -1225,8 +1276,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
         """ return a list of all group names """
         all_groups = set()
         all_groups.update(self.groups.keys())
-        all_groups.update([g.name for g in self.group_membership.values()])
-        all_groups.update([g.name for g in self.negated_groups.values()])
+        all_groups.update(self.group_membership.keys())
+        all_groups.update(self.negated_groups.keys())
         for grp in self.clientgroups.values():
             all_groups.update(grp)
         return all_groups
@@ -1239,7 +1290,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
     def get_client_names_by_profiles(self, profiles):
         """ return a list of names of clients in the given profile groups """
         rv = []
-        for client in list(self.clients):
+        for client in self.list_clients():
             mdata = self.core.build_metadata(client)
             if mdata.profile in profiles:
                 rv.append(client)
@@ -1247,34 +1298,33 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
 
     def get_client_names_by_groups(self, groups):
         """ return a list of names of clients in the given groups """
-        mdata = [self.core.build_metadata(client) for client in self.clients]
-        return [md.hostname for md in mdata if md.groups.issuperset(groups)]
+        rv = []
+        for client in self.list_clients():
+            mdata = self.core.build_metadata(client)
+            if mdata.groups.issuperset(groups):
+                rv.append(client)
+        return rv
 
     def get_client_names_by_bundles(self, bundles):
         """ given a list of bundles, return a list of names of clients
         that use those bundles """
-        mdata = [self.core.build_metadata(client) for client in self.clients]
-        return [md.hostname for md in mdata if md.bundles.issuperset(bundles)]
+        rv = []
+        for client in self.list_clients():
+            mdata = self.core.build_metadata(client)
+            if mdata.bundles.issuperset(bundles):
+                rv.append(client)
+        return rv
 
     def merge_additional_groups(self, imd, groups):
         for group in groups:
             if group in imd.groups:
                 continue
-            if group in self.groups and self.groups[group].category:
-                category = self.groups[group].category
-                if self.groups[group].category in imd.categories:
-                    self.logger.warning("%s: Group %s suppressed by category "
-                                        "%s; %s already a member of %s" %
-                                        (self.name, group, category,
-                                         imd.hostname,
-                                         imd.categories[category]))
-                    continue
-                imd.categories[category] = group
+            if not self._check_and_add_category(imd.hostname, group,
+                                                imd.categories):
+                continue
             imd.groups.add(group)
 
-        self._merge_groups(imd.hostname, imd.groups,
-                           categories=imd.categories)
-
+        self._merge_groups(imd.hostname, imd.groups, categories=imd.categories)
         for group in imd.groups:
             if group in self.groups:
                 imd.bundles.update(self.groups[group].bundles)
@@ -1439,7 +1489,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
 
         instances = {}
         rv = []
-        for client in list(self.clients):
+        for client in list(self.list_clients()):
             if not include_client(client):
                 continue
             if client in self.clientgroups: