Metadata and other improvements:

* Added support for Client tag in groups.xml
* Added support for nested Group tags in groups.xml
* Added support for negated groups in groups.xml
* Added DatabaseBacked plugin mixin to easily allow plugins to connect
  to a database specified in global database settings in bcfg2.conf
* Added DBMetadata plugin that uses relational DB to store client
  records instead of writing to clients.xml

13 files changed, 694 insertions, 365 deletions

diff --git a/src/lib/Bcfg2/Server/Admin/Bundle.py b/src/lib/Bcfg2/Server/Admin/Bundle.py
index 89c099602..ab07e29b3 100644
--- a/src/lib/Bcfg2/Server/Admin/Bundle.py
+++ b/src/lib/Bcfg2/Server/Admin/Bundle.py
@@ -8,12 +8,11 @@ from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError
 
 
 class Bundle(Bcfg2.Server.Admin.MetadataCore):
-    __shorthelp__ = "Create or delete bundle entries"
-    # TODO: add/del functions
+    __shorthelp__ = "List and view bundle entries"
     __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin bundle list-xml"
                                     "\nbcfg2-admin bundle list-genshi"
                                     "\nbcfg2-admin bundle show\n")
-    __usage__ = ("bcfg2-admin bundle [options] [add|del] [group]")
+    __usage__ = ("bcfg2-admin bundle [options] [list-xml|list-genshi|show]")
 
     def __call__(self, args):
         Bcfg2.Server.Admin.MetadataCore.__call__(self, args)
@@ -28,18 +27,6 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore):
         if len(args) == 0:
             self.errExit("No argument specified.\n"
                          "Please see bcfg2-admin bundle help for usage.")
-#        if args[0] == 'add':
-#            try:
-#                self.metadata.add_bundle(args[1])
-#            except MetadataConsistencyError:
-#                print("Error in adding bundle.")
-#                raise SystemExit(1)
-#        elif args[0] in ['delete', 'remove', 'del', 'rm']:
-#            try:
-#                self.metadata.remove_bundle(args[1])
-#            except MetadataConsistencyError:
-#                print("Error in deleting bundle.")
-#                raise SystemExit(1)
         # Lists all available xml bundles
         elif args[0] in ['list-xml', 'ls-xml']:
             bundle_name = []
diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py
index 734e9573d..34dfd7550 100644
--- a/src/lib/Bcfg2/Server/Admin/Client.py
+++ b/src/lib/Bcfg2/Server/Admin/Client.py
@@ -4,50 +4,23 @@ from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError
 
 
 class Client(Bcfg2.Server.Admin.MetadataCore):
-    __shorthelp__ = "Create, delete, or modify client entries"
+    __shorthelp__ = "Create, delete, or list client entries"
     __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin client add <client> "
-                                    "attr1=val1 attr2=val2"
-                                    "\nbcfg2-admin client update <client> "
-                                    "attr1=val1 attr2=val2"
                                     "\nbcfg2-admin client list"
                                     "\nbcfg2-admin client del <client>\n")
-    __usage__ = ("bcfg2-admin client [options] [add|del|update|list] [attr=val]")
+    __usage__ = ("bcfg2-admin client [options] [add|del|list] [attr=val]")
 
     def __call__(self, args):
         Bcfg2.Server.Admin.MetadataCore.__call__(self, args)
         if len(args) == 0:
             self.errExit("No argument specified.\n"
-                         "Please see bcfg2-admin client help for usage.")
+                         "Usage: %s" % self.usage)
         if args[0] == 'add':
-            attr_d = {}
-            for i in args[2:]:
-                attr, val = i.split('=', 1)
-                if attr not in ['profile', 'uuid', 'password',
-                                'location', 'secure', 'address',
-                                'auth']:
-                    print("Attribute %s unknown" % attr)
-                    raise SystemExit(1)
-                attr_d[attr] = val
             try:
-                self.metadata.add_client(args[1], attr_d)
+                self.metadata.add_client(args[1])
             except MetadataConsistencyError:
                 print("Error in adding client")
                 raise SystemExit(1)
-        elif args[0] in ['update', 'up']:
-            attr_d = {}
-            for i in args[2:]:
-                attr, val = i.split('=', 1)
-                if attr not in ['profile', 'uuid', 'password',
-                                'location', 'secure', 'address',
-                                'auth']:
-                    print("Attribute %s unknown" % attr)
-                    raise SystemExit(1)
-                attr_d[attr] = val
-            try:
-                self.metadata.update_client(args[1], attr_d)
-            except MetadataConsistencyError:
-                print("Error in updating client")
-                raise SystemExit(1)
         elif args[0] in ['delete', 'remove', 'del', 'rm']:
             try:
                 self.metadata.remove_client(args[1])
@@ -55,7 +28,9 @@ class Client(Bcfg2.Server.Admin.MetadataCore):
                 print("Error in deleting client")
                 raise SystemExit(1)
         elif args[0] in ['list', 'ls']:
-            tree = lxml.etree.parse(self.metadata.data + "/clients.xml")
-            tree.xinclude()
-            for node in tree.findall("//Client"):
-                print(node.attrib["name"])
+            for client in self.metadata.list_clients():
+                print(client.hostname)
+        else:
+            print("No command specified")
+            raise SystemExit(1)
+
diff --git a/src/lib/Bcfg2/Server/Admin/Group.py b/src/lib/Bcfg2/Server/Admin/Group.py
deleted file mode 100644
index 16a773d6f..000000000
--- a/src/lib/Bcfg2/Server/Admin/Group.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import lxml.etree
-import Bcfg2.Server.Admin
-from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError
-
-
-class Group(Bcfg2.Server.Admin.MetadataCore):
-    __shorthelp__ = "Create, delete, or modify group entries"
-    __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin group add <group> "
-                                    "attr1=val1 attr2=val2"
-                                    "\nbcfg2-admin group update <group> "
-                                    "attr1=val1 attr2=val2"
-                                    "\nbcfg2-admin group list"
-                                    "\nbcfg2-admin group del <group>\n")
-    __usage__ = ("bcfg2-admin group [options] [add|del|update|list] [attr=val]")
-
-    def __call__(self, args):
-        Bcfg2.Server.Admin.MetadataCore.__call__(self, args)
-        if len(args) == 0:
-            self.errExit("No argument specified.\n"
-                         "Please see bcfg2-admin group help for usage.")
-        if args[0] == 'add':
-            attr_d = {}
-            for i in args[2:]:
-                attr, val = i.split('=', 1)
-                if attr not in ['profile', 'public', 'default',
-                               'name', 'auth', 'toolset', 'category',
-                               'comment']:
-                    print("Attribute %s unknown" % attr)
-                    raise SystemExit(1)
-                attr_d[attr] = val
-            try:
-                self.metadata.add_group(args[1], attr_d)
-            except MetadataConsistencyError:
-                print("Error in adding group")
-                raise SystemExit(1)
-        elif args[0] in ['update', 'up']:
-            attr_d = {}
-            for i in args[2:]:
-                attr, val = i.split('=', 1)
-                if attr not in ['profile', 'public', 'default',
-                                'name', 'auth', 'toolset', 'category',
-                                'comment']:
-                    print("Attribute %s unknown" % attr)
-                    raise SystemExit(1)
-                attr_d[attr] = val
-            try:
-                self.metadata.update_group(args[1], attr_d)
-            except MetadataConsistencyError:
-                print("Error in updating group")
-                raise SystemExit(1)
-        elif args[0] in ['delete', 'remove', 'del', 'rm']:
-            try:
-                self.metadata.remove_group(args[1])
-            except MetadataConsistencyError:
-                print("Error in deleting group")
-                raise SystemExit(1)
-        elif args[0] in ['list', 'ls']:
-            tree = lxml.etree.parse(self.metadata.data + "/groups.xml")
-            for node in tree.findall("//Group"):
-                print(node.attrib["name"])
-        else:
-            print("No command specified")
-            raise SystemExit(1)
diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py
index 8d0c2a4a9..30603bddc 100644
--- a/src/lib/Bcfg2/Server/Admin/Init.py
+++ b/src/lib/Bcfg2/Server/Admin/Init.py
@@ -308,9 +308,8 @@ class Init(Bcfg2.Server.Admin.Mode):
         for plugin in self.plugins:
             if plugin == 'Metadata':
                 Bcfg2.Server.Plugins.Metadata.Metadata.init_repo(self.repopath,
-                                                                 groups,
-                                                                 self.os_sel,
-                                                                 clients)
+                                                                 groups_xml=groups % self.os_sel,
+                                                                 clients_xml=clients)
             else:
                 try:
                     module = __import__("Bcfg2.Server.Plugins.%s" % plugin, '',
diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py
new file mode 100644
index 000000000..73dc5b8b2
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py
@@ -0,0 +1,33 @@
+import Bcfg2.settings
+import Bcfg2.Options
+import Bcfg2.Server.Admin
+from django.core.management import setup_environ
+
+class Syncdb(Bcfg2.Server.Admin.Mode):
+    __shorthelp__ = ("Sync the Django ORM with the configured database")
+    __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin syncdb"
+    __usage__ = "bcfg2-admin syncdb"
+    options = {'configfile': Bcfg2.Options.CFILE,
+               'repo': Bcfg2.Options.SERVER_REPOSITORY}
+
+    def __call__(self, args):
+        Bcfg2.Server.Admin.Mode.__call__(self, args)
+
+        # Parse options
+        self.opts = Bcfg2.Options.OptionParser(self.options)
+        self.opts.parse(args)
+
+        # we have to set up the django environment before we import
+        # the syncdb command, but we have to wait to set up the
+        # environment until we've read the config, which has to wait
+        # until we've parsed options.  it's a windy, twisting road.
+        Bcfg2.settings.read_config(cfile=self.opts['configfile'],
+                                   repo=self.opts['repo'])
+        setup_environ(Bcfg2.settings)
+        import Bcfg2.Server.models
+        Bcfg2.Server.models.load_models(cfile=self.opts['configfile'])
+
+        from django.core.management.commands import syncdb
+
+        cmd = syncdb.Command()
+        cmd.handle_noargs(interactive=False)
diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py
index 0c9158351..3a7ba45cf 100644
--- a/src/lib/Bcfg2/Server/Admin/__init__.py
+++ b/src/lib/Bcfg2/Server/Admin/__init__.py
@@ -11,6 +11,7 @@ __all__ = [
         'Query',
         'Reports',
         'Snapshots',
+        'Syncdb',
         'Tidy',
         'Viz',
         'Xcmd'
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 1ee01585c..20eee2d7f 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -1,5 +1,6 @@
 """Bcfg2.Server.Core provides the runtime support for Bcfg2 modules."""
 
+import os
 import atexit
 import logging
 import select
@@ -9,6 +10,11 @@ import time
 import inspect
 import lxml.etree
 from traceback import format_exc
+
+# this must be set before we import the Metadata plugin
+os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings'
+
+import Bcfg2.settings
 import Bcfg2.Server
 import Bcfg2.Logger
 import Bcfg2.Server.FileMonitor
@@ -95,6 +101,10 @@ class BaseCore(object):
         # Create an event to signal worker threads to shutdown
         self.terminate = threading.Event()
 
+        # generate Django ORM settings.  this must be done _before_ we
+        # load plugins
+        Bcfg2.settings.read_config(cfile=self.cfile, repo=self.datastore)
+
         if '' in setup['plugins']:
             setup['plugins'].remove('')
 
@@ -195,8 +205,7 @@ class BaseCore(object):
         try:
             self.plugins[plugin] = plug(self, self.datastore)
         except PluginInitError:
-            self.logger.error("Failed to instantiate plugin %s" % plugin,
-                              exc_info=1)
+            logger.error("Failed to instantiate plugin %s" % plugin, exc_info=1)
         except:
             self.logger.error("Unexpected instantiation failure for plugin %s" %
                               plugin, exc_info=1)
@@ -526,8 +535,6 @@ class BaseCore(object):
     def RecvProbeData(self, address, probedata):
         """Receive probe data from clients."""
         client, metadata = self.resolve_client(address)
-        # clear dynamic groups
-        self.metadata.cgroups[metadata.hostname] = []
         try:
             xpdata = lxml.etree.XML(probedata.encode('utf-8'),
                                     parser=Bcfg2.Server.XMLParser)
diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py
index 6b4276444..51d1b1cdb 100644
--- a/src/lib/Bcfg2/Server/Plugin.py
+++ b/src/lib/Bcfg2/Server/Plugin.py
@@ -102,6 +102,16 @@ class Debuggable(object):
             self.logger.error(message)
 
 
+class DatabaseBacked(object):
+    def __init__(self):
+        pass
+
+
+class PluginDatabaseModel(object):
+    class Meta:
+        app_label = "Server"
+
+
 class Plugin(Debuggable):
     """This is the base class for all Bcfg2 Server plugins.
     Several attributes must be defined in the subclass:
@@ -139,8 +149,7 @@ class Plugin(Debuggable):
 
     @classmethod
     def init_repo(cls, repo):
-        path = "%s/%s" % (repo, cls.name)
-        os.makedirs(path)
+        os.makedirs(os.path.join(repo, cls.name))
 
     def shutdown(self):
         self.running = False
@@ -169,7 +178,7 @@ class Structure(object):
 
 class Metadata(object):
     """Signal metadata capabilities for this plugin"""
-    def add_client(self, client_name, attribs):
+    def add_client(self, client_name):
         """Add client."""
         pass
 
@@ -181,6 +190,9 @@ class Metadata(object):
         """Create viz str for viz admin mode."""
         pass
 
+    def _handle_default_event(self, event):
+        pass
+
     def get_initial_metadata(self, client_name):
         raise PluginExecutionError
 
@@ -650,7 +662,7 @@ class XMLFileBacked(FileBacked):
 
     def add_monitor(self, fpath, fname):
         self.extras.append(fname)
-        if self.fam:
+        if self.fam and self.should_monitor:
             self.fam.AddMonitor(fpath, self)
 
     def __iter__(self):
@@ -666,22 +678,13 @@ class StructFile(XMLFileBacked):
 
     def _include_element(self, item, metadata):
         """ determine if an XML element matches the metadata """
+        negate = item.get('negate', 'false').lower() == 'true'
         if item.tag == 'Group':
-            if ((item.get('negate', 'false').lower() == 'true' and
-                 item.get('name') not in metadata.groups) or
-                (item.get('negate', 'false').lower() == 'false' and
-                 item.get('name') in metadata.groups)):
-                return True
-            else:
-                return False
+            return ((negate and item.get('name') not in metadata.groups) or
+                    (not negate and item.get('name') in metadata.groups))
         elif item.tag == 'Client':
-            if ((item.get('negate', 'false').lower() == 'true' and
-                 item.get('name') != metadata.hostname) or
-                (item.get('negate', 'false').lower() == 'false' and
-                 item.get('name') == metadata.hostname)):
-                return True
-            else:
-                return False
+            return ((negate and item.get('name') != metadata.hostname) or
+                    (not negate and item.get('name') == metadata.hostname))
         elif isinstance(item, lxml.etree._Comment):
             return False
         else:
diff --git a/src/lib/Bcfg2/Server/Plugins/DBMetadata.py b/src/lib/Bcfg2/Server/Plugins/DBMetadata.py
new file mode 100644
index 000000000..16a6e0dcc
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/DBMetadata.py
@@ -0,0 +1,128 @@
+import os
+import sys
+from UserDict import DictMixin
+from django.db import models
+import Bcfg2.Server.Lint
+import Bcfg2.Server.Plugin
+from Bcfg2.Server.Plugins.Metadata import *
+
+class MetadataClientModel(models.Model,
+                          Bcfg2.Server.Plugin.PluginDatabaseModel):
+    hostname = models.CharField(max_length=255, primary_key=True)
+    version = models.CharField(max_length=31, null=True)
+
+
+class ClientVersions(DictMixin):
+    def __getitem__(self, key):
+        try:
+            return MetadataClientModel.objects.get(hostname=key).version
+        except MetadataClientModel.DoesNotExist:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        client = MetadataClientModel.objects.get_or_create(hostname=key)[0]
+        client.version = value
+        client.save()
+
+    def keys(self):
+        return [c.hostname for c in MetadataClientModel.objects.all()]
+
+    def __contains__(self, key):
+        try:
+            client = MetadataClientModel.objects.get(hostname=key)
+            return True
+        except MetadataClientModel.DoesNotExist:
+            return False
+
+
+class DBMetadata(Metadata, Bcfg2.Server.Plugin.DatabaseBacked):
+    __files__ = ["groups.xml"]
+    experimental = True
+    conflicts = ['Metadata']
+
+    def __init__(self, core, datastore, watch_clients=True):
+        Metadata.__init__(self, core, datastore, watch_clients=watch_clients)
+        Bcfg2.Server.Plugin.DatabaseBacked.__init__(self)
+        if os.path.exists(os.path.join(self.data, "clients.xml")):
+            self.logger.warning("DBMetadata: clients.xml found, parsing in "
+                                "compatibility mode")
+            self._handle_file("clients.xml")
+        self.versions = ClientVersions()
+
+    def add_group(self, group_name, attribs):
+        msg = "DBMetadata does not support adding groups"
+        self.logger.error(msg)
+        raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def add_bundle(self, bundle_name):
+        msg = "DBMetadata does not support adding bundles"
+        self.logger.error(msg)
+        raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def add_client(self, client_name):
+        """Add client to clients database."""
+        client = MetadataClientModel(hostname=client_name)
+        client.save()
+        self.clients = self.list_clients()
+        return client
+
+    def update_group(self, group_name, attribs):
+        msg = "DBMetadata does not support updating groups"
+        self.logger.error(msg)
+        raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def update_bundle(self, bundle_name):
+        msg = "DBMetadata does not support updating bundles"
+        self.logger.error(msg)
+        raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def update_client(self, client_name, attribs):
+        msg = "DBMetadata does not support updating clients"
+        self.logger.error(msg)
+        raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def list_clients(self):
+        """ List all clients in client database """
+        return set([c.hostname for c in MetadataClientModel.objects.all()])
+
+    def remove_group(self, group_name, attribs):
+        msg = "DBMetadata does not support removing groups"
+        self.logger.error(msg)
+        raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def remove_bundle(self, bundle_name):
+        msg = "DBMetadata does not support removing bundles"
+        self.logger.error(msg)
+        raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def remove_client(self, client_name):
+        """Remove a client"""
+        try:
+            client = MetadataClientModel.objects.get(hostname=client_name)
+        except MetadataClientModel.DoesNotExist:
+            msg = "Client %s does not exist" % client_name
+            self.logger.warning(msg)
+            raise MetadataConsistencyError(msg)
+        client.delete()
+        self.clients = self.list_clients()
+
+    def _set_profile(self, client, profile, addresspair):
+        if client not in self.clients:
+            # adding a new client
+            self.add_client(client)
+            if client not in self.clientgroups:
+                self.clientgroups[client] = [profile]
+        else:
+            msg = "DBMetadata does not support asserting client profiles"
+            self.logger.error(msg)
+            raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
+
+    def _handle_clients_xml_event(self, event):
+        # clients.xml is parsed and the options specified in it are
+        # understood, but it does _not_ assert client existence.
+        Metadata._handle_clients_xml_event(self, event)
+        self.clients = self.list_clients()
+
+
+class DBMetadataLint(MetadataLint):
+    pass
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 4f6e82128..447a7cd05 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -2,6 +2,7 @@
 This file stores persistent metadata for the Bcfg2 Configuration Repository.
 """
 
+import re
 import copy
 import fcntl
 import lxml.etree
@@ -10,8 +11,9 @@ import socket
 import sys
 import time
 import Bcfg2.Server
-import Bcfg2.Server.FileMonitor
+import Bcfg2.Server.Lint
 import Bcfg2.Server.Plugin
+import Bcfg2.Server.FileMonitor
 from Bcfg2.version import Bcfg2VersionInfo
 
 def locked(fd):
@@ -38,10 +40,10 @@ class MetadataRuntimeError(Exception):
 class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked):
     """Handles xml config files and all XInclude statements"""
     def __init__(self, metadata, watch_clients, basefile):
-        # we tell XMLFileBacked _not_ to add a monitor for this
-        # file, because the main Metadata plugin has already added
-        # one.  then we immediately set should_monitor to the proper
-        # value, so that XIinclude'd files get properly watched
+        # we tell XMLFileBacked _not_ to add a monitor for this file,
+        # because the main Metadata plugin has already added one.
+        # then we immediately set should_monitor to the proper value,
+        # so that XInclude'd files get properly watched
         fpath = os.path.join(metadata.data, basefile)
         Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath,
                                                    fam=metadata.core.fam,
@@ -210,7 +212,8 @@ class ClientMetadata(object):
 
 
 class MetadataQuery(object):
-    def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category):
+    def __init__(self, by_name, get_clients, by_groups, by_profiles,
+                 all_groups, all_groups_in_category):
         # resolver is set later
         self.by_name = by_name
         self.names_by_groups = by_groups
@@ -229,6 +232,36 @@ class MetadataQuery(object):
         return [self.by_name(name) for name in self.all_clients()]
 
 
+class MetadataGroup(tuple):
+    def __new__(cls, name, bundles=None, category=None,
+                 is_profile=False, is_public=False, is_private=False):
+        if bundles is None:
+            bundles = set()
+        return tuple.__new__(cls, (bundles, category))
+
+    def __init__(self, name, bundles=None, category=None,
+                 is_profile=False, is_public=False, is_private=False):
+        if bundles is None:
+            bundles = set()
+        tuple.__init__(self)
+        self.name = name
+        self.bundles = bundles
+        self.category = category
+        self.is_profile = is_profile
+        self.is_public = is_public
+        self.is_private = is_private
+
+    def __str__(self):
+        return repr(self)
+
+    def __repr__(self):
+        return "%s %s (bundles=%s, category=%s)" % \
+            (self.__class__.__name__, self.name, self.bundles,
+             self.category)
+
+    def __hash__(self):
+        return hash(self.name)
+
 class Metadata(Bcfg2.Server.Plugin.Plugin,
                Bcfg2.Server.Plugin.Metadata,
                Bcfg2.Server.Plugin.Statistics):
@@ -236,69 +269,80 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
     __author__ = 'bcfg-dev@mcs.anl.gov'
     name = "Metadata"
     sort_order = 500
+    __files__ = ["groups.xml", "clients.xml"]
 
     def __init__(self, core, datastore, watch_clients=True):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Metadata.__init__(self)
         Bcfg2.Server.Plugin.Statistics.__init__(self)
+        self.watch_clients = watch_clients
         self.states = dict()
-        if watch_clients:
-            for fname in ["groups.xml", "clients.xml"]:
-                self.states[fname] = False
-                try:
-                    core.fam.AddMonitor(os.path.join(self.data, fname), self)
-                except:
-                    err = sys.exc_info()[1]
-                    msg = "Unable to add file monitor for %s: %s" % (fname, err)
-                    print(msg)
-                    raise Bcfg2.Server.Plugin.PluginInitError(msg)
-
-        self.clients_xml = XMLMetadataConfig(self, watch_clients, 'clients.xml')
-        self.groups_xml = XMLMetadataConfig(self, watch_clients, 'groups.xml')
-        self.addresses = {}
+        self.extra = dict()
+        self.handlers = []
+        for fname in self.__files__:
+            self._handle_file(fname)
+
+        # mapping of clientname -> authtype
         self.auth = dict()
-        self.clients = {}
-        self.aliases = {}
-        self.groups = {}
-        self.cgroups = {}
-        self.versions = {}
-        self.public = []
-        self.private = []
-        self.profiles = []
-        self.categories = {}
-        self.bad_clients = {}
-        self.uuid = {}
+        # list of clients required to have non-global password
         self.secure = []
+        # list of floating clients
         self.floating = []
+        # mapping of clientname -> password
         self.passwords = {}
+        self.addresses = {}
+        self.raddresses = {}
+        # mapping of clientname -> [groups]
+        self.clientgroups = {}
+        # list of clients
+        self.clients = []
+        self.aliases = {}
+        self.raliases = {}
+        # mapping of groupname -> MetadataGroup object
+        self.groups = {}
+        # mappings of predicate -> MetadataGroup object
+        self.group_membership = dict()
+        self.negated_groups = dict()
+        # mapping of hostname -> version string
+        self.versions = dict()
+        self.uuid = {}
         self.session_cache = {}
         self.default = None
         self.pdirty = False
-        self.extra = {'groups.xml': [],
-                      'clients.xml': []}
         self.password = core.password
         self.query = MetadataQuery(core.build_metadata,
-                                   lambda: list(self.clients.keys()),
+                                   lambda: list(self.clients),
                                    self.get_client_names_by_groups,
                                    self.get_client_names_by_profiles,
                                    self.get_all_group_names,
                                    self.get_all_groups_in_category)
 
     @classmethod
-    def init_repo(cls, repo, groups, os_selection, clients):
-        path = os.path.join(repo, cls.name)
-        os.makedirs(path)
-        open(os.path.join(repo, "Metadata", "groups.xml"),
-             "w").write(groups % os_selection)
-        open(os.path.join(repo, "Metadata", "clients.xml"),
-             "w").write(clients % socket.getfqdn())
-
-    def get_groups(self):
-        '''return groups xml tree'''
-        groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"),
-                                       parser=Bcfg2.Server.XMLParser)
-        root = groups_tree.getroot()
-        return root
+    def init_repo(cls, repo, **kwargs):
+        # must use super here; inheritance works funny with class methods
+        super(Metadata, cls).init_repo(repo)
+
+        for fname in cls.__files__:
+            aname = re.sub(r'[^A-z0-9_]', '_', fname)
+            if aname in kwargs:
+                open(os.path.join(repo, cls.name, fname),
+                     "w").write(kwargs[aname])
+
+    def _handle_file(self, fname):
+        if self.watch_clients:
+            try:
+                self.core.fam.AddMonitor(os.path.join(self.data, fname), self)
+            except:
+                err = sys.exc_info()[1]
+                msg = "Unable to add file monitor for %s: %s" % (fname, err)
+                self.logger.error(msg)
+                raise Bcfg2.Server.Plugin.PluginInitError(msg)
+            self.states[fname] = False
+        aname = re.sub(r'[^A-z0-9_]', '_', fname)
+        xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname)
+        setattr(self, aname, xmlcfg)
+        self.handlers.append(xmlcfg.HandleEvent)
+        self.extra[fname] = []
 
     def _search_xdata(self, tag, name, tree, alias=False):
         for node in tree.findall("//%s" % tag):
@@ -325,9 +369,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
     def _add_xdata(self, config, tag, name, attribs=None, alias=False):
         node = self._search_xdata(tag, name, config.xdata, alias=alias)
         if node != None:
-            msg = "%s \"%s\" already exists" % (tag, name)
-            self.logger.error(msg)
-            raise MetadataConsistencyError(msg)
+            self.logger.error("%s \"%s\" already exists" % (tag, name))
+            raise MetadataConsistencyError
         element = lxml.etree.SubElement(config.base_xdata.getroot(),
                                         tag, name=name)
         if attribs:
@@ -352,15 +395,14 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
     def _update_xdata(self, config, tag, name, attribs, alias=False):
         node = self._search_xdata(tag, name, config.xdata, alias=alias)
         if node == None:
-            msg = "%s \"%s\" does not exist" % (tag, name)
-            self.logger.error(msg)
-            raise MetadataConsistencyError(msg)
+            self.logger.error("%s \"%s\" does not exist" % (tag, name))
+            raise MetadataConsistencyError
         xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
                                           (tag, node.get('name')))
         if not xdict:
-            msg = "Unexpected error finding %s \"%s\"" % (tag, name)
-            self.logger.error(msg)
-            raise MetadataConsistencyError(msg)
+            self.logger.error("Unexpected error finding %s \"%s\"" %
+                              (tag, name))
+            raise MetadataConsistencyError
         for key, val in list(attribs.items()):
             xdict['xquery'][0].set(key, val)
         config.write_xml(xdict['filename'], xdict['xmltree'])
@@ -377,17 +419,16 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
     def _remove_xdata(self, config, tag, name, alias=False):
         node = self._search_xdata(tag, name, config.xdata)
         if node == None:
-            msg = "%s \"%s\" does not exist" % (tag, name)
-            self.logger.error(msg)
-            raise MetadataConsistencyError(msg)
+            self.logger.error("%s \"%s\" does not exist" % (tag, name))
+            raise MetadataConsistencyError
         xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
                                           (tag, node.get('name')))
         if not xdict:
-            msg = "Unexpected error finding %s \"%s\"" % (tag, name)
-            self.logger.error(msg)
-            raise MetadataConsistencyError(msg)
+            self.logger.error("Unexpected error finding %s \"%s\"" %
+                              (tag, name))
+            raise MetadataConsistencyError
         xdict['xquery'][0].getparent().remove(xdict['xquery'][0])
-        self.groups_xml.write_xml(xdict['filename'], xdict['xmltree'])
+        config.write_xml(xdict['filename'], xdict['xmltree'])
 
     def remove_group(self, group_name):
         """Remove a group."""
@@ -397,12 +438,16 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
         """Remove a bundle."""
         return self._remove_xdata(self.groups_xml, "Bundle", bundle_name)
 
+    def remove_client(self, client_name):
+        """Remove a bundle."""
+        return self._remove_xdata(self.clients_xml, "Client", client_name)
+
     def _handle_clients_xml_event(self, event):
         xdata = self.clients_xml.xdata
-        self.clients = {}
+        self.clients = []
+        self.clientgroups = {}
         self.aliases = {}
         self.raliases = {}
-        self.bad_clients = {}
         self.secure = []
         self.floating = []
         self.addresses = {}
@@ -423,9 +468,10 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                                                            'cert+password')
             if 'uuid' in client.attrib:
                 self.uuid[client.get('uuid')] = clname
-            if client.get('secure', 'false') == 'true':
+            if client.get('secure', 'false').lower() == 'true':
                 self.secure.append(clname)
-            if client.get('location', 'fixed') == 'floating':
+            if (client.get('location', 'fixed') == 'floating' or
+                client.get('floating', 'false').lower() == 'true'):
                 self.floating.append(clname)
             if 'password' in client.attrib:
                 self.passwords[clname] = client.get('password')
@@ -445,106 +491,157 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                 if clname not in self.raddresses:
                     self.raddresses[clname] = set()
                 self.raddresses[clname].add(alias.get('address'))
-            self.clients.update({clname: client.get('profile')})
+            self.clients.append(clname)
+            try:
+                self.clientgroups[clname].append(client.get('profile'))
+            except KeyError:
+                self.clientgroups[clname] = [client.get('profile')]
         self.states['clients.xml'] = True
 
     def _handle_groups_xml_event(self, event):
-        xdata = self.groups_xml.xdata
-        self.public = []
-        self.private = []
-        self.profiles = []
         self.groups = {}
-        grouptmp = {}
-        self.categories = {}
-        groupseen = list()
-        for group in xdata.xpath('//Groups/Group'):
-            if group.get('name') not in groupseen:
-                groupseen.append(group.get('name'))
+
+        # get_condition and aggregate_conditions must be separate
+        # functions in order to ensure that the scope is right for the
+        # closures they return
+        def get_condition(element):
+            negate = element.get('negate', 'false').lower() == 'true'
+            pname = element.get("name")
+            if element.tag == 'Group':
+                return lambda c, g, _: negate != (pname in g)
+            elif element.tag == 'Client':
+                return lambda c, g, _: negate != (pname == c)
+
+        def aggregate_conditions(conditions):
+            return lambda client, groups, cats: \
+                all(cond(client, groups, cats) for cond in conditions)
+
+        # first, we get a list of all of the groups declared in the
+        # file.  we do this in two stages because the old way of
+        # parsing groups.xml didn't support nested groups; in the old
+        # way, only Group tags under a Groups tag counted as
+        # declarative.  so we parse those first, and then parse the
+        # other Group tags if they haven't already been declared.
+        # this lets you set options on a group (e.g., public="false")
+        # at the top level and then just use the name elsewhere, which
+        # is the original behavior
+        for grp in self.groups_xml.xdata.xpath("//Groups/Group") + \
+                self.groups_xml.xdata.xpath("//Groups/Group//Group"):
+            if grp.get("name") in self.groups:
+                continue
+            self.groups[grp.get("name")] = \
+                MetadataGroup(grp.get("name"),
+                              bundles=[b.get("name")
+                                       for b in grp.findall("Bundle")],
+                              category=grp.get("category"),
+                              is_profile=grp.get("profile", "false") == "true",
+                              is_public=grp.get("public", "false") == "true",
+                              is_private=grp.get("public", "true") == "false")
+            if grp.get('default', 'false') == 'true':
+                self.default = grp.get('name')
+
+        self.group_membership = dict()
+        self.negated_groups = dict()
+        self.options = dict()
+        # confusing loop condition; the XPath query asks for all
+        # elements under a Group tag under a Groups tag; that is
+        # infinitely recursive, so "all" elements really means _all_
+        # elements.  We then manually filter out non-Group elements
+        # since there doesn't seem to be a way to get Group elements
+        # of arbitrary depth with particular ultimate ancestors in
+        # XPath.  We do the same thing for Client tags.
+        for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \
+                self.groups_xml.xdata.xpath("//Groups/Client//*"):
+            if ((el.tag != 'Group' and el.tag != 'Client') or
+                el.getchildren()):
+                continue
+
+            conditions = []
+            for parent in el.iterancestors():
+                cond = get_condition(parent)
+                if cond:
+                    conditions.append(cond)
+
+            gname = el.get("name")
+            if el.get("negate", "false").lower() == "true":
+                self.negated_groups[aggregate_conditions(conditions)] = \
+                    self.groups[gname]
             else:
-                self.logger.error("Metadata: Group %s defined multiply" %
-                                  group.get('name'))
-            grouptmp[group.get('name')] = \
-                ([item.get('name') for item in group.findall('./Bundle')],
-                 [item.get('name') for item in group.findall('./Group')])
-            grouptmp[group.get('name')][1].append(group.get('name'))
-            if group.get('default', 'false') == 'true':
-                self.default = group.get('name')
-            if group.get('profile', 'false') == 'true':
-                self.profiles.append(group.get('name'))
-            if group.get('public', 'false') == 'true':
-                self.public.append(group.get('name'))
-            elif group.get('public', 'true') == 'false':
-                self.private.append(group.get('name'))
-            if 'category' in group.attrib:
-                self.categories[group.get('name')] = group.get('category')
-
-        for group in grouptmp:
-            # self.groups[group] => (bundles, groups, categories)
-            self.groups[group] = (set(), set(), {})
-            tocheck = [group]
-            group_cat = self.groups[group][2]
-            while tocheck:
-                now = tocheck.pop()
-                self.groups[group][1].add(now)
-                if now in grouptmp:
-                    (bundles, groups) = grouptmp[now]
-                    for ggg in groups:
-                        if ggg in self.groups[group][1]:
-                            continue
-                        if (ggg not in self.categories or \
-                            self.categories[ggg] not in self.groups[group][2]):
-                            self.groups[group][1].add(ggg)
-                            tocheck.append(ggg)
-                            if ggg in self.categories:
-                                group_cat[self.categories[ggg]] = ggg
-                        elif ggg in self.categories:
-                            self.logger.info("Group %s: %s cat-suppressed %s" % \
-                                             (group,
-                                              group_cat[self.categories[ggg]],
-                                              ggg))
-                    [self.groups[group][0].add(bund) for bund in bundles]
+                if self.groups[gname].category and gname in self.groups:
+                    category = self.groups[gname].category
+
+                    def in_cat(client, groups, categories):
+                        if category in categories:
+                            self.logger.warning("%s: Group %s suppressed by "
+                                                "category %s; %s already a "
+                                                "member of %s" %
+                                                (self.name, gname, category,
+                                                 client, categories[category]))
+                            return False
+                        return True
+                    conditions.append(in_cat)
+
+                self.group_membership[aggregate_conditions(conditions)] = \
+                    self.groups[gname]
         self.states['groups.xml'] = True
 
     def HandleEvent(self, event):
         """Handle update events for data files."""
-        if self.clients_xml.HandleEvent(event):
-            self._handle_clients_xml_event(event)
-        elif self.groups_xml.HandleEvent(event):
-            self._handle_groups_xml_event(event)
-
-        if False not in list(self.states.values()):
-            # check that all client groups are real and complete
-            real = list(self.groups.keys())
-            for client in list(self.clients.keys()):
-                if self.clients[client] not in self.profiles:
-                    self.logger.error("Client %s set as nonexistent or "
-                                      "incomplete group %s" %
-                                      (client, self.clients[client]))
-                    self.logger.error("Removing client mapping for %s" % client)
-                    self.bad_clients[client] = self.clients[client]
-                    del self.clients[client]
-            for bclient in list(self.bad_clients.keys()):
-                if self.bad_clients[bclient] in self.profiles:
-                    self.logger.info("Restored profile mapping for client %s" %
-                                     bclient)
-                    self.clients[bclient] = self.bad_clients[bclient]
-                    del self.bad_clients[bclient]
-
-    def set_profile(self, client, profile, addresspair):
+        for hdlr in self.handlers:
+            aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(event.filename))
+            if hdlr(event):
+                try:
+                    proc = getattr(self, "_handle_%s_event" % aname)
+                except AttributeError:
+                    proc = self._handle_default_event
+                proc(event)
+
+        if False not in list(self.states.values()) and self.debug_flag:
+            # check that all groups are real and complete. this is
+            # just logged at a debug level because many groups might
+            # be probed, and we don't want to warn about them.
+            for client, groups in list(self.clientgroups.items()):
+                for group in groups:
+                    if group not in self.groups:
+                        self.debug_log("Client %s set as nonexistent group %s" %
+                                       (client, group))
+            for gname, ginfo in list(self.groups.items()):
+                for group in ginfo.groups:
+                    if group not in self.groups:
+                        self.debug_log("Group %s set as nonexistent group %s" %
+                                       (gname, group))
+
+
+    def set_profile(self, client, profile, addresspair, force=False):
         """Set group parameter for provided client."""
-        self.logger.info("Asserting client %s profile to %s" % (client,
-                                                                profile))
+        self.logger.info("Asserting client %s profile to %s" %
+                         (client, profile))
         if False in list(self.states.values()):
-            raise MetadataRuntimeError("Metadata has not been read yet")
-        if profile not in self.public:
-            msg = "Failed to set client %s to private group %s" % (client,
-                                                                   profile)
+            raise MetadataRuntimeError
+        if not force and profile not in self.groups:
+            msg = "Profile group %s does not exist" % profile
+            self.logger.error(msg)
+            raise MetadataConsistencyError(msg)
+        group = self.groups[profile]
+        if not force and not group.is_public:
+            msg = "Cannot set client %s to private group %s" % (client, profile)
             self.logger.error(msg)
             raise MetadataConsistencyError(msg)
+        self._set_profile(client, profile, addresspair)
+
+    def _set_profile(self, client, profile, addresspair):
         if client in self.clients:
-            self.logger.info("Changing %s group from %s to %s" %
-                             (client, self.clients[client], profile))
+            profiles = [g for g in self.clientgroups[client]
+                        if g in self.groups and self.groups[g].is_profile]
+            self.logger.info("Changing %s profile from %s to %s" %
+                             (client, profiles, profile))
             self.update_client(client, dict(profile=profile))
+            if client in self.clientgroups:
+                for p in profiles:
+                    self.clientgroups[client].remove(p)
+                self.clientgroups[client].append(profile)
+            else:
+                self.clientgroups[client] = [profile]
         else:
             self.logger.info("Creating new client: %s, profile %s" %
                              (client, profile))
@@ -555,7 +652,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                                      address=addresspair[0]))
             else:
                 self.add_client(client, dict(profile=profile))
-        self.clients[client] = profile
+            self.clients.append(client)
+            self.clientgroups[client] = [profile]
         self.clients_xml.write()
 
     def set_version(self, client, version):
@@ -614,6 +712,31 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
             self.logger.warning(warning)
             raise MetadataConsistencyError(warning)
 
+    def _merge_groups(self, client, groups, categories=None):
+        """ set group membership based on the contents of groups.xml
+        and initial group membership of this client. Returns a tuple
+        of (allgroups, categories)"""
+        numgroups = -1 # force one initial pass
+        if categories is None:
+            categories = dict()
+        while numgroups != len(groups):
+            numgroups = len(groups)
+            for predicate, group in self.group_membership.items():
+                if group.name in groups:
+                    continue
+                if predicate(client, groups, categories):
+                    groups.add(group.name)
+                    if group.category:
+                        categories[group.category] = group.name
+            for predicate, group in self.negated_groups.items():
+                if group.name not in groups:
+                    continue
+                if predicate(client, groups, categories):
+                    groups.remove(group.name)
+                    if group.category:
+                        del categories[group.category]
+        return (groups, categories)
+
     def get_initial_metadata(self, client):
         """Return the metadata for a given client."""
         if False in list(self.states.values()):
@@ -621,25 +744,66 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
         client = client.lower()
         if client in self.aliases:
             client = self.aliases[client]
-        if client in self.clients:
-            profile = self.clients[client]
-            (bundles, groups, categories) = self.groups[profile]
-        else:
-            if self.default == None:
-                msg = "Cannot set group for client %s; no default group set" % \
-                    client
+        
+        groups = set()
+        categories = dict()
+        profile = None
+
+        if client not in self.clients:
+            pgroup = None
+            if client in self.clientgroups:
+                pgroup = self.clientgroups[client][0]
+            elif self.default:
+                pgroup = self.default
+
+            if pgroup:
+                self.set_profile(client, pgroup, (None, None), force=True)
+                groups.add(pgroup)
+                category = self.groups[pgroup].category
+                if category:
+                    categories[category] = pgroup
+                if (pgroup in self.groups and self.groups[pgroup].is_profile):
+                    profile = pgroup
+            else:
+                msg = "Cannot add new client %s; no default group set" % client
                 self.logger.error(msg)
                 raise MetadataConsistencyError(msg)
-            self.set_profile(client, self.default, (None, None))
-            profile = self.default
-            [bundles, groups, categories] = self.groups[self.default]
+
+        if client in self.clientgroups:
+            for cgroup in self.clientgroups[client]:
+                if cgroup in groups:
+                    continue
+                if cgroup not in self.groups:
+                    self.groups[cgroup] = MetadataGroup(cgroup)
+                category = self.groups[cgroup].category
+                if category and category in categories:
+                    self.logger.warning("%s: Group %s suppressed by "
+                                        "category %s; %s already a member "
+                                        "of %s" %
+                                        (self.name, cgroup, category,
+                                         client, categories[category]))
+                    continue
+                if category:
+                    categories[category] = cgroup
+                groups.add(cgroup)
+                # favor client groups for setting profile
+                if not profile and self.groups[cgroup].is_profile:
+                    profile = cgroup
+
+        groups, categories = self._merge_groups(client, groups,
+                                                categories=categories)
+
+        bundles = set()
+        for group in groups:
+            try:
+                bundles.update(self.groups[group].bundles)
+            except KeyError:
+                self.logger.warning("%s: %s is a member of undefined group %s" %
+                                    (self.name, client, group))
+
         aliases = self.raliases.get(client, set())
         addresses = self.raddresses.get(client, set())
         version = self.versions.get(client, None)
-        newgroups = set(groups)
-        newbundles = set(bundles)
-        newcategories = {}
-        newcategories.update(categories)
         if client in self.passwords:
             password = self.passwords[client]
         else:
@@ -650,36 +814,41 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
             uuid = uuids[0]
         else:
             uuid = None
-        for group in self.cgroups.get(client, []):
-            if group in self.groups:
-                nbundles, ngroups, ncategories = self.groups[group]
-            else:
-                nbundles, ngroups, ncategories = ([], [group], {})
-            [newbundles.add(b) for b in nbundles if b not in newbundles]
-            [newgroups.add(g) for g in ngroups if g not in newgroups]
-            newcategories.update(ncategories)
-        return ClientMetadata(client, profile, newgroups, newbundles, aliases,
-                              addresses, newcategories, uuid, password, version,
+        if not profile:
+            # one last ditch attempt at setting the profile
+            profiles = [g for g in groups
+                        if g in self.groups and self.groups[g].is_profile]
+            if len(profiles) >= 1:
+                profile = profiles[0]
+
+        return ClientMetadata(client, profile, groups, bundles, aliases,
+                              addresses, categories, uuid, password, version,
                               self.query)
 
     def get_all_group_names(self):
         all_groups = set()
-        [all_groups.update(g[1]) for g in list(self.groups.values())]
+        all_groups.update(self.groups.keys())
+        all_groups.update([g.name for g in self.group_membership.values()])
+        all_groups.update([g.name for g in self.negated_groups.values()])
+        for grp in self.clientgroups.values():
+            all_groups.update(grp)
         return all_groups
 
     def get_all_groups_in_category(self, category):
-        all_groups = set()
-        [all_groups.add(g) for g in self.categories \
-                if self.categories[g] == category]
-        return all_groups
+        return set([g.name for g in self.groups.values()
+                    if g.category == category])
 
     def get_client_names_by_profiles(self, profiles):
-        return [client for client, profile in list(self.clients.items()) \
-                if profile in profiles]
+        rv = []
+        for client in list(self.clients):
+            mdata = self.get_initial_metadata(client)
+            if mdata.profile in profiles:
+                rv.append(client)
+        return rv
 
     def get_client_names_by_groups(self, groups):
         mdata = [self.core.build_metadata(client)
-                 for client in list(self.clients.keys())]
+                 for client in list(self.clients)]
         return [md.hostname for md in mdata if md.groups.issuperset(groups)]
 
     def get_client_names_by_bundles(self, bundles):
@@ -689,27 +858,26 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
 
     def merge_additional_groups(self, imd, groups):
         for group in groups:
-            if (group in self.categories and
-                self.categories[group] in imd.categories):
+            if group in imd.groups or group not in self.groups:
                 continue
-            newbundles, newgroups, _ = self.groups.get(group,
-                                                       (list(),
-                                                        [group],
-                                                        dict()))
-            for newbundle in newbundles:
-                if newbundle not in imd.bundles:
-                    imd.bundles.add(newbundle)
-            for newgroup in newgroups:
-                if newgroup not in imd.groups:
-                    if (newgroup in self.categories and
-                        self.categories[newgroup] in imd.categories):
-                        continue
-                    if newgroup in self.private:
-                        self.logger.error("Refusing to add dynamic membership "
-                                          "in private group %s for client %s" %
-                                          (newgroup, imd.hostname))
-                        continue
-                    imd.groups.add(newgroup)
+            category = self.groups[group].category
+            if category:
+                if self.groups[group].category in imd.categories:
+                    self.logger.warning("%s: Group %s suppressed by category "
+                                        "%s; %s already a member of %s" %
+                                        (self.name, group, category,
+                                         imd.hostname,
+                                         imd.categories[category]))
+                    continue
+                imd.categories[group] = category
+            imd.groups.add(group)
+
+        self._merge_groups(imd.hostname, imd.groups,
+                           categories=imd.categories)
+
+        for group in imd.groups:
+            if group in self.groups:
+                imd.bundles.update(self.groups[group].bundles)
 
     def merge_additional_data(self, imd, source, data):
         if not hasattr(imd, source):
@@ -728,8 +896,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                                (client, address))
                 return True
             else:
-                self.logger.error("Got request for non-float client %s from %s" %
-                                  (client, address))
+                self.logger.error("Got request for non-float client %s from %s"
+                                  % (client, address))
                 return False
         resolved = self.resolve_client(addresspair)
         if resolved.lower() == client.lower():
@@ -853,20 +1021,26 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
             del categories[None]
         if hosts:
             instances = {}
-            clients = self.clients
-            for client, profile in list(clients.items()):
+            for client in list(self.clients):
                 if include_client(client):
                     continue
-                if profile in instances:
-                    instances[profile].append(client)
+                if client in self.clientgroups:
+                    groups = self.clientgroups[client]
+                elif self.default:
+                    groups = [self.default]
                 else:
-                    instances[profile] = [client]
-            for profile, clist in list(instances.items()):
+                    continue
+                for group in groups:
+                    try:
+                        instances[group].append(client)
+                    except KeyError:
+                        instances[group] = [client]
+            for group, clist in list(instances.items()):
                 clist.sort()
                 viz_str.append('"%s-instances" [ label="%s", shape="record" ];' %
-                               (profile, '|'.join(clist)))
+                               (group, '|'.join(clist)))
                 viz_str.append('"%s-instances" -> "group-%s";' %
-                               (profile, profile))
+                               (group, group))
         if bundles:
             bundles = []
             [bundles.append(bund.get('name')) \
@@ -907,3 +1081,35 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                 viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' %
                                (category, category, categories[category]))
         return "\n".join("\t" + s for s in viz_str)
+
+
+class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
+    def Run(self):
+        self.nested_clients()
+        self.deprecated_options()
+
+    @classmethod
+    def Errors(cls):
+        return {"nested-client-tags": "warning",
+                "deprecated-clients-options": "warning"}
+
+    def deprecated_options(self):
+        groupdata = self.metadata.clients_xml.xdata
+        for el in groupdata.xpath("//Client"):
+            loc = el.get("location")
+            if loc:
+                if loc == "floating":
+                    floating = True
+                else:
+                    floating = False
+                self.LintError("deprecated-clients-options",
+                               "The location='%s' option is deprecated.  "
+                               "Please use floating='%s' instead: %s" %
+                               (loc, floating, self.RenderXML(el)))        
+
+    def nested_clients(self):
+        groupdata = self.metadata.groups_xml.xdata
+        for el in groupdata.xpath("//Client//Client"):
+            self.LintError("nested-client-tags",
+                           "Client %s nested within Client tag: %s" %
+                           (el.get("name"), self.RenderXML(el)))
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index ac78ea0fc..9cea9da48 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -375,15 +375,7 @@ def factory(metadata, sources, basepath, debug=False):
                        ",".join([s.__name__ for s in sclasses]))
         cclass = Collection
     elif len(sclasses) == 0:
-        # you'd think this should be a warning, but it happens all the
-        # freaking time if you have a) machines in your clients.xml
-        # that do not have the proper groups set up yet (e.g., if you
-        # have multiple Bcfg2 servers and Packages-relevant groups set
-        # by probes); and b) templates that query all or multiple
-        # machines (e.g., with metadata.query.all_clients())
-        if debug:
-            logger.error("Packages: No sources found for %s" %
-                         metadata.hostname)
+        logger.error("Packages: No sources found for %s" % metadata.hostname)
         cclass = Collection
     else:
         cclass = get_collection_class(sclasses.pop().__name__.replace("Source",
@@ -398,4 +390,3 @@ def factory(metadata, sources, basepath, debug=False):
     clients[metadata.hostname] = ckey
     collections[ckey] = collection
     return collection
-
diff --git a/src/lib/Bcfg2/Server/Reports/settings.py b/src/lib/Bcfg2/Server/Reports/settings.py
index b27348aee..26138cddb 100644
--- a/src/lib/Bcfg2/Server/Reports/settings.py
+++ b/src/lib/Bcfg2/Server/Reports/settings.py
@@ -43,7 +43,7 @@ try:
     db_engine = c.get('statistics', 'database_engine')
 except ConfigParser.NoSectionError:
     e = sys.exc_info()[1]
-    raise ImportError("Failed to determine database engine: %s" % e)
+    raise ImportError("Failed to determine database engine for reports: %s" % e)
 db_name = ''
 if c.has_option('statistics', 'database_name'):
     db_name = c.get('statistics', 'database_name')
diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py
new file mode 100644
index 000000000..ba9ea761c
--- /dev/null
+++ b/src/lib/Bcfg2/Server/models.py
@@ -0,0 +1,62 @@
+import sys
+import logging
+import Bcfg2.Options
+import Bcfg2.Server.Plugins
+from django.db import models
+from Bcfg2.Bcfg2Py3k import ConfigParser
+
+logger = logging.getLogger('Bcfg2.Server.models')
+
+MODELS = []
+
+def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True):
+    global MODELS
+
+    if plugins is None:
+        # we want to provide a different default plugin list --
+        # namely, _all_ plugins, so that the database is guaranteed to
+        # work, even if /etc/bcfg2.conf isn't set up properly
+        plugin_opt = Bcfg2.Options.SERVER_PLUGINS
+        plugin_opt.default = Bcfg2.Server.Plugins.__all__
+
+        setup = Bcfg2.Options.OptionParser(dict(plugins=plugin_opt,
+                                                configfile=Bcfg2.Options.CFILE),
+                                           quiet=quiet)
+        setup.parse([Bcfg2.Options.CFILE.cmd, cfile])
+        plugins = setup['plugins']
+
+    if MODELS:
+        # load_models() has been called once, so first unload all of
+        # the models; otherwise we might call load_models() with no
+        # arguments, end up with _all_ models loaded, and then in a
+        # subsequent call only load a subset of models
+        for model in MODELS:
+            delattr(sys.modules[__name__], model)
+        MODELS = []
+
+    for plugin in plugins:
+        try:
+            mod = getattr(__import__("Bcfg2.Server.Plugins.%s" %
+                                     plugin).Server.Plugins, plugin)
+        except ImportError:
+            try:
+                mod = __import__(plugin)
+            except:
+                if plugins != Bcfg2.Server.Plugins.__all__:
+                    # only produce errors if the default plugin list
+                    # was not used -- i.e., if the config file was set
+                    # up.  don't produce errors when trying to load
+                    # all plugins, IOW
+                    err = sys.exc_info()[1]
+                    logger.error("Failed to load plugin %s: %s" % (plugin, err))
+                    continue
+        for sym in dir(mod):
+            obj = getattr(mod, sym)
+            if hasattr(obj, "__bases__") and models.Model in obj.__bases__:
+                print("Adding %s to models" % sym)
+                setattr(sys.modules[__name__], sym, obj)
+                MODELS.append(sym)
+
+# basic invocation to ensure that a default set of models is loaded,
+# and thus that this module will always work.
+load_models(quiet=True)