diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/Plugins/Metadata.py')
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 300 |
1 files changed, 169 insertions, 131 deletions
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 12ece1f19..db104b27e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -16,10 +16,10 @@ import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked +from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo - # pylint: disable=C0103 ClientVersions = None MetadataClientModel = None @@ -89,7 +89,7 @@ def load_django_models(): def keys(self): """ Get keys for the mapping """ - return [c.hostname for c in MetadataClientModel.objects.all()] + return list(iter(self)) def __contains__(self, key): try: @@ -102,17 +102,12 @@ def load_django_models(): class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" - def __init__(self, metadata, watch_clients, basefile): - # we tell XMLFileBacked _not_ to add a monitor for this file, - # because the main Metadata plugin has already added one. - # then we immediately set should_monitor to the proper value, - # so that XInclude'd files get properly watched + def __init__(self, metadata, basefile): fpath = os.path.join(metadata.data, basefile) toptag = os.path.splitext(basefile)[0].title() Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, should_monitor=False, create=toptag) - self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile self.data = None @@ -257,8 +252,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): def add_monitor(self, fpath): self.extras.append(fpath) - if self.should_monitor: - self.fam.AddMonitor(fpath, self.metadata) + self.fam.AddMonitor(fpath, self.metadata) def HandleEvent(self, event=None): """Handle fam events""" @@ -500,7 +494,6 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" @@ -518,12 +511,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, help='Default client authentication method')] options_parsed_hook = staticmethod(load_django_models) - def __init__(self, core, datastore, watch_clients=True): + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) - self.watch_clients = watch_clients self.states = dict() self.extra = dict() self.handlers = dict() @@ -554,22 +545,26 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} - # mappings of predicate -> MetadataGroup object + # mappings of groupname -> [predicates] self.group_membership = dict() self.negated_groups = dict() + # list of group names in document order + self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions(core, # pylint: disable=E1102 datastore) else: self.versions = dict() + self.uuid = {} self.session_cache = {} + self.cache = Cache("Metadata") self.default = None self.pdirty = False self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients), + self.list_clients, self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -595,17 +590,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_file(self, fname): """ set up the necessary magic for handling a metadata file (clients.xml or groups.xml, e.g.) """ - if self.watch_clients: - try: - Bcfg2.Server.FileMonitor.get_fam().AddMonitor( - os.path.join(self.data, fname), self) - except: - err = sys.exc_info()[1] - msg = "Unable to add file monitor for %s: %s" % (fname, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - self.states[fname] = False - xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) + try: + Bcfg2.Server.FileMonitor.get_fam().AddMonitor( + os.path.join(self.data, fname), self) + except: + err = sys.exc_info()[1] + msg = "Unable to add file monitor for %s: %s" % (fname, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + self.states[fname] = False + xmlcfg = XMLMetadataConfig(self, fname) aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(fname)) self.handlers[xmlcfg.HandleEvent] = getattr(self, "_handle_%s_event" % aname) @@ -860,51 +854,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() + def _get_condition(self, element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ + negate = element.get('negate', 'false').lower() == 'true' + pname = element.get("name") + if element.tag == 'Group': + return lambda c, g, _: negate != (pname in g) + elif element.tag == 'Client': + return lambda c, g, _: negate != (pname == c) + + def _get_category_condition(self, grpname): + """ get a predicate that returns False if a client is already + a member of a group in the given group's category, True + otherwise""" + return lambda client, _, categories: \ + bool(self._check_category(client, grpname, categories)) + + def _aggregate_conditions(self, conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ + return lambda client, groups, cats: \ + all(cond(client, groups, cats) for cond in conditions) + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ self.groups = {} - # these three functions must be separate functions in order to - # ensure that the scope is right for the closures they return - def get_condition(element): - """ Return a predicate that returns True if a client meets - the condition specified in the given Group or Client - element """ - negate = element.get('negate', 'false').lower() == 'true' - pname = element.get("name") - if element.tag == 'Group': - return lambda c, g, _: negate != (pname in g) - elif element.tag == 'Client': - return lambda c, g, _: negate != (pname == c) - - def get_category_condition(category, gname): - """ get a predicate that returns False if a client is - already a member of a group in the given category, True - otherwise """ - def in_cat(client, groups, categories): # pylint: disable=W0613 - """ return True if the client is already a member of a - group in the category given in the enclosing function, - False otherwise """ - if category in categories: - if (gname not in self.groups or - client not in self.groups[gname].warned): - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, gname, category, - client, categories[category])) - if gname in self.groups: - self.groups[gname].warned.append(client) - return False - return True - return in_cat - - def aggregate_conditions(conditions): - """ aggregate all conditions on a given group declaration - into a single predicate """ - return lambda client, groups, cats: \ - all(cond(client, groups, cats) for cond in conditions) - # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old @@ -930,6 +907,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() + self.ordered_groups = [] # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is @@ -940,40 +918,44 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): - if ((el.tag != 'Group' and el.tag != 'Client') or - el.getchildren()): + if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren(): continue conditions = [] for parent in el.iterancestors(): - cond = get_condition(parent) + cond = self._get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[aggregate_conditions(conditions)] = \ - self.groups[gname] + self.negated_groups.setdefault(gname, []) + self.negated_groups[gname].append( + self._aggregate_conditions(conditions)) else: if self.groups[gname].category: - conditions.append( - get_category_condition(self.groups[gname].category, - gname)) + conditions.append(self._get_category_condition(gname)) - self.group_membership[aggregate_conditions(conditions)] = \ - self.groups[gname] + if gname not in self.ordered_groups: + self.ordered_groups.append(gname) + self.group_membership.setdefault(gname, []) + self.group_membership[gname].append( + self._aggregate_conditions(conditions)) self.states['groups.xml'] = True - def expire_cache(self, key=None): - self.core.metadata_cache.expire(key) - def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.expire_cache() + self.cache.expire() + + # clear out the list of category suppressions that + # have been warned about, since this may change when + # clients.xml or groups.xml changes. + for group in self.groups.values(): + group.warned = [] event_handler(event) if False not in list(self.states.values()) and self.debug_flag: @@ -1112,30 +1094,85 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() while numgroups != len(groups): numgroups = len(groups) - for predicate, group in self.group_membership.items(): - if group.name in groups: + newgroups = set() + removegroups = set() + for grpname in self.ordered_groups: + if grpname in groups: continue - if predicate(client, groups, categories): - groups.add(group.name) - if group.category: - categories[group.category] = group.name - for predicate, group in self.negated_groups.items(): - if group.name not in groups: + if any(p(client, groups, categories) + for p in self.group_membership[grpname]): + newgroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + categories[self.groups[grpname].category] = grpname + groups.update(newgroups) + for grpname, predicates in self.negated_groups.items(): + if grpname not in groups: continue - if predicate(client, groups, categories): - groups.remove(group.name) - if group.category: - del categories[group.category] + if any(p(client, groups, categories) for p in predicates): + removegroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + del categories[self.groups[grpname].category] + groups.difference_update(removegroups) return (groups, categories) + def _check_category(self, client, grpname, categories): + """ Determine if the given client is already a member of a + group in the same category as the named group. + + The return value is one of three possibilities: + + * If the client is already a member of a group in the same + category, then False is returned (i.e., the category check + failed); + * If the group is not in any categories, then True is returned; + * If the group is not a member of a group in the category, + then the name of the category is returned. This makes it + easy to add the category to the ClientMetadata object (or + other category list). + + If a pure boolean value is required, you can do + ``bool(self._check_category(...))``. + """ + if grpname not in self.groups: + return True + category = self.groups[grpname].category + if not category: + return True + if category in categories: + if client not in self.groups[grpname].warned: + self.logger.warning("%s: Group %s suppressed by category %s; " + "%s already a member of %s" % + (self.name, grpname, category, + client, categories[category])) + self.groups[grpname].warned.append(client) + return False + return category + + def _check_and_add_category(self, client, grpname, categories): + """ If the client is not a member of a group in the same + category as the named group, then the category is added to + ``categories``. + :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used + to determine if the category can be added. + + If the category check failed, returns False; otherwise, + returns True. """ + rv = self._check_category(client, grpname, categories) + if rv and rv is not True: + categories[rv] = grpname + return True + return rv + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") client = client.lower() - if client in self.core.metadata_cache: - return self.core.metadata_cache[client] + if client in self.cache: + return self.cache[client] if client in self.aliases: client = self.aliases[client] @@ -1149,30 +1186,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Handles setting categories and category suppression. Returns the new profile for the client (which might be unchanged). """ - groups.add(grpname) if grpname in self.groups: - group = self.groups[grpname] - category = group.category - if category: - if category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, grpname, category, - client, categories[category])) - return - categories[category] = grpname - if not profile and group.is_profile: + if not self._check_and_add_category(client, grpname, + categories): + return profile + groups.add(grpname) + if not profile and self.groups[grpname].is_profile: return grpname else: return profile + else: + groups.add(grpname) + return profile if client not in self.clients: pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] + self.debug_log("%s: Adding new client with profile %s" % + (self.name, pgroup)) elif self.default: pgroup = self.default + self.debug_log("%s: Adding new client with default profile %s" + % (self.name, pgroup)) if pgroup: self.set_profile(client, pgroup, (None, None), @@ -1189,6 +1225,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[cgroup] = MetadataGroup(cgroup) profile = _add_group(cgroup) + # we do this before setting the default because there may be + # groups set in <Client> tags in groups.xml that we want to + # set groups, categories = self._merge_groups(client, groups, categories=categories) @@ -1230,15 +1269,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, addresses, categories, uuid, password, version, self.query) if self.core.metadata_cache_mode == 'initial': - self.core.metadata_cache[client] = rv + self.cache[client] = rv return rv def get_all_group_names(self): """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) - all_groups.update([g.name for g in self.group_membership.values()]) - all_groups.update([g.name for g in self.negated_groups.values()]) + all_groups.update(self.group_membership.keys()) + all_groups.update(self.negated_groups.keys()) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups @@ -1251,7 +1290,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.clients): + for client in self.list_clients(): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1259,34 +1298,33 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.groups.issuperset(groups): + rv.append(client) + return rv def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.bundles.issuperset(bundles): + rv.append(client) + return rv def merge_additional_groups(self, imd, groups): for group in groups: if group in imd.groups: continue - if group in self.groups and self.groups[group].category: - category = self.groups[group].category - if self.groups[group].category in imd.categories: - self.logger.warning("%s: Group %s suppressed by category " - "%s; %s already a member of %s" % - (self.name, group, category, - imd.hostname, - imd.categories[category])) - continue - imd.categories[category] = group + if not self._check_and_add_category(imd.hostname, group, + imd.categories): + continue imd.groups.add(group) - self._merge_groups(imd.hostname, imd.groups, - categories=imd.categories) - + self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) @@ -1451,7 +1489,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] - for client in list(self.clients): + for client in list(self.list_clients()): if not include_client(client): continue if client in self.clientgroups: |