""" This file stores persistent metadata for the Bcfg2 Configuration Repository. """ import re import os import sys import time import copy import fcntl import socket import logging import lxml.etree import Bcfg2.Server import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Compat import MutableMapping, all from Bcfg2.version import Bcfg2VersionInfo try: from django.db import models has_django = True except ImportError: has_django = False logger = logging.getLogger(__name__) def locked(fd): """Aquire a lock on a file""" try: fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: return True return False if has_django: class MetadataClientModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): hostname = models.CharField(max_length=255, primary_key=True) version = models.CharField(max_length=31, null=True) class ClientVersions(MutableMapping): def __getitem__(self, key): try: return MetadataClientModel.objects.get(hostname=key).version except MetadataClientModel.DoesNotExist: raise KeyError(key) def __setitem__(self, key, value): client = MetadataClientModel.objects.get_or_create(hostname=key)[0] client.version = value client.save() def __delitem__(self, key): # UserDict didn't require __delitem__, but MutableMapping # does. we don't want deleting a client version record to # delete the client, so we just set the version to None, # which is kinda like deleting it, but not really. try: client = MetadataClientModel.objects.get(hostname=key) except MetadataClientModel.DoesNotExist: raise KeyError(key) client.version = None client.save() def __len__(self): return MetadataClientModel.objects.count() def __iter__(self): for client in MetadataClientModel.objects.all(): yield client.hostname def keys(self): return [c.hostname for c in MetadataClientModel.objects.all()] def __contains__(self, key): try: client = MetadataClientModel.objects.get(hostname=key) return True except MetadataClientModel.DoesNotExist: return False class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" def __init__(self, metadata, watch_clients, basefile): # we tell XMLFileBacked _not_ to add a monitor for this file, # because the main Metadata plugin has already added one. # then we immediately set should_monitor to the proper value, # so that XInclude'd files get properly watched fpath = os.path.join(metadata.data, basefile) Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, fam=metadata.core.fam, should_monitor=False) self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile self.data = None self.basedata = None self.basedir = metadata.data self.logger = metadata.logger self.pseudo_monitor = isinstance(metadata.core.fam, Bcfg2.Server.FileMonitor.Pseudo) @property def xdata(self): if not self.data: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) return self.data @property def base_xdata(self): if not self.basedata: raise Bcfg2.Server.Plugin.MetadataRuntimeError("%s has no data" % self.basefile) return self.basedata def load_xml(self): """Load changes from XML""" try: xdata = lxml.etree.parse(os.path.join(self.basedir, self.basefile), parser=Bcfg2.Server.XMLParser) except lxml.etree.XMLSyntaxError: self.logger.error('Failed to parse %s' % self.basefile) return self.extras = [] self.basedata = copy.copy(xdata) self._follow_xincludes(xdata=xdata) if self.extras: try: xdata.xinclude() except lxml.etree.XIncludeError: self.logger.error("Failed to process XInclude for file %s" % self.basefile) self.data = xdata def write(self): """Write changes to xml back to disk.""" self.write_xml(os.path.join(self.basedir, self.basefile), self.basedata) def write_xml(self, fname, xmltree): """Write changes to xml back to disk.""" tmpfile = "%s.new" % fname try: datafile = open(tmpfile, 'w') except IOError: msg = "Failed to write %s: %s" % (tmpfile, sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) # prep data dataroot = xmltree.getroot() newcontents = lxml.etree.tostring(dataroot, xml_declaration=False, pretty_print=True).decode('UTF-8') fd = datafile.fileno() while locked(fd) == True: pass try: datafile.write(newcontents) except: fcntl.lockf(fd, fcntl.LOCK_UN) msg = "Metadata: Failed to write new xml data to %s: %s" % \ (tmpfile, sys.exc_info()[1]) self.logger.error(msg, exc_info=1) os.unlink(tmpfile) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) datafile.close() # check if clients.xml is a symlink if os.path.islink(fname): fname = os.readlink(fname) try: os.rename(tmpfile, fname) except: msg = "Metadata: Failed to rename %s: %s" % (tmpfile, sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" if self.pseudo_monitor: # Reload xml if we don't have a real monitor self.load_xml() cli = self.basedata.xpath(xpath) if len(cli) > 0: return {'filename': os.path.join(self.basedir, self.basefile), 'xmltree': self.basedata, 'xquery': cli} else: """Try to find the data in included files""" for included in self.extras: try: xdata = lxml.etree.parse(included, parser=Bcfg2.Server.XMLParser) cli = xdata.xpath(xpath) if len(cli) > 0: return {'filename': included, 'xmltree': xdata, 'xquery': cli} except lxml.etree.XMLSyntaxError: self.logger.error('Failed to parse %s' % included) return {} def add_monitor(self, fpath): self.extras.append(fpath) if self.fam and self.should_monitor: self.fam.AddMonitor(fpath, self.metadata) def HandleEvent(self, event): """Handle fam events""" filename = os.path.basename(event.filename) if event.filename in self.extras: if event.code2str() == 'exists': return False elif filename != self.basefile: return False if event.code2str() == 'endExist': return False self.load_xml() return True class ClientMetadata(object): """This object contains client metadata.""" def __init__(self, client, profile, groups, bundles, aliases, addresses, categories, uuid, password, version, query): self.hostname = client self.profile = profile self.bundles = bundles self.aliases = aliases self.addresses = addresses self.groups = groups self.categories = categories self.uuid = uuid self.password = password self.connectors = [] self.version = version try: self.version_info = Bcfg2VersionInfo(version) except: self.version_info = None self.query = query def inGroup(self, group): """Test to see if client is a member of group.""" return group in self.groups def group_in_category(self, category): for grp in self.query.all_groups_in_category(category): if grp in self.groups: return grp return '' class MetadataQuery(object): def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): # resolver is set later self.by_name = by_name self.names_by_groups = self._warn_string(by_groups) self.names_by_profiles = self._warn_string(by_profiles) self.all_clients = get_clients self.all_groups = all_groups self.all_groups_in_category = all_groups_in_category def _warn_string(self, func): # it's a common mistake to call by_groups, etc., in templates with # a single string argument instead of a list. that doesn't cause # errors because strings are iterables. this decorator warns # about that usage. def inner(arg): if isinstance(arg, str): logger.warning("%s: %s takes a list as argument, not a string" % (self.__class__.__name__, func.__name__)) return func(arg) return inner def by_groups(self, groups): # don't need to decorate this with _warn_string because # names_by_groups is decorated return [self.by_name(name) for name in self.names_by_groups(groups)] def by_profiles(self, profiles): # don't need to decorate this with _warn_string because # names_by_profiles is decorated return [self.by_name(name) for name in self.names_by_profiles(profiles)] def all(self): return [self.by_name(name) for name in self.all_clients()] class MetadataGroup(tuple): def __new__(cls, name, bundles=None, category=None, is_profile=False, is_public=False, is_private=False): if bundles is None: bundles = set() return tuple.__new__(cls, (bundles, category)) def __init__(self, name, bundles=None, category=None, is_profile=False, is_public=False, is_private=False): if bundles is None: bundles = set() tuple.__init__(self) self.name = name self.bundles = bundles self.category = category self.is_profile = is_profile self.is_public = is_public self.is_private = is_private # record which clients we've warned about category suppression self.warned = [] def __str__(self): return repr(self) def __repr__(self): return "%s %s (bundles=%s, category=%s)" % \ (self.__class__.__name__, self.name, self.bundles, self.category) def __hash__(self): return hash(self.name) class Metadata(Bcfg2.Server.Plugin.Metadata, Bcfg2.Server.Plugin.Statistics, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" __author__ = 'bcfg-dev@mcs.anl.gov' name = "Metadata" sort_order = 500 def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) Bcfg2.Server.Plugin.Statistics.__init__(self, core, datastore) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) self.watch_clients = watch_clients self.states = dict() self.extra = dict() self.handlers = [] self._handle_file("groups.xml") if (self._use_db and os.path.exists(os.path.join(self.data, "clients.xml"))): self.logger.warning("Metadata: database enabled but clients.xml" "found, parsing in compatibility mode") self._handle_file("clients.xml") elif not self._use_db: self._handle_file("clients.xml") # mapping of clientname -> authtype self.auth = dict() # list of clients required to have non-global password self.secure = [] # list of floating clients self.floating = [] # mapping of clientname -> password self.passwords = {} self.addresses = {} self.raddresses = {} # mapping of clientname -> [groups] self.clientgroups = {} # list of clients self.clients = [] self.aliases = {} self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} # mappings of predicate -> MetadataGroup object self.group_membership = dict() self.negated_groups = dict() # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions() else: self.versions = dict() self.uuid = {} self.session_cache = {} self.default = None self.pdirty = False self.password = core.password self.query = MetadataQuery(core.build_metadata, lambda: list(self.clients), self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, self.get_all_groups_in_category) @classmethod def init_repo(cls, repo, **kwargs): # must use super here; inheritance works funny with class methods super(Metadata, cls).init_repo(repo) for fname in ["clients.xml", "groups.xml"]: aname = re.sub(r'[^A-z0-9_]', '_', fname) if aname in kwargs: open(os.path.join(repo, cls.name, fname), "w").write(kwargs[aname]) def _handle_file(self, fname): if self.watch_clients: try: self.core.fam.AddMonitor(os.path.join(self.data, fname), self) except: err = sys.exc_info()[1] msg = "Unable to add file monitor for %s: %s" % (fname, err) self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) self.states[fname] = False aname = re.sub(r'[^A-z0-9_]', '_', fname) xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) setattr(self, aname, xmlcfg) self.handlers.append(xmlcfg.HandleEvent) self.extra[fname] = [] def _search_xdata(self, tag, name, tree, alias=False): for node in tree.findall("//%s" % tag): if node.get("name") == name: return node elif alias: for child in node: if (child.tag == "Alias" and child.attrib["name"] == name): return node return None def search_group(self, group_name, tree): """Find a group.""" return self._search_xdata("Group", group_name, tree) def search_bundle(self, bundle_name, tree): """Find a bundle.""" return self._search_xdata("Bundle", bundle_name, tree) def search_client(self, client_name, tree): return self._search_xdata("Client", client_name, tree, alias=True) def _add_xdata(self, config, tag, name, attribs=None, alias=False): node = self._search_xdata(tag, name, config.xdata, alias=alias) if node != None: self.logger.error("%s \"%s\" already exists" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError element = lxml.etree.SubElement(config.base_xdata.getroot(), tag, name=name) if attribs: for key, val in list(attribs.items()): element.set(key, val) config.write() def add_group(self, group_name, attribs): """Add group to groups.xml.""" if self._use_db: msg = "Metadata does not support adding groups with use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._add_xdata(self.groups_xml, "Group", group_name, attribs=attribs) def add_bundle(self, bundle_name): """Add bundle to groups.xml.""" if self._use_db: msg = "Metadata does not support adding bundles with use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._add_xdata(self.groups_xml, "Bundle", bundle_name) def add_client(self, client_name, attribs=None): """Add client to clients.xml.""" if attribs is None: attribs = dict() if self._use_db: client = MetadataClientModel(hostname=client_name) client.save() self.clients = self.list_clients() return client else: return self._add_xdata(self.clients_xml, "Client", client_name, attribs=attribs, alias=True) def _update_xdata(self, config, tag, name, attribs, alias=False): node = self._search_xdata(tag, name, config.xdata, alias=alias) if node == None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: self.logger.error("Unexpected error finding %s \"%s\"" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) def update_group(self, group_name, attribs): """Update a groups attributes.""" if self._use_db: msg = "Metadata does not support updating groups with use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._update_xdata(self.groups_xml, "Group", group_name, attribs) def update_client(self, client_name, attribs): """Update a clients attributes.""" if self._use_db: msg = "Metadata does not support updating clients with use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._update_xdata(self.clients_xml, "Client", client_name, attribs, alias=True) def list_clients(self): """ List all clients in client database """ if self._use_db: return set([c.hostname for c in MetadataClientModel.objects.all()]) else: return self.clients def _remove_xdata(self, config, tag, name, alias=False): node = self._search_xdata(tag, name, config.xdata) if node == None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: self.logger.error("Unexpected error finding %s \"%s\"" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError xdict['xquery'][0].getparent().remove(xdict['xquery'][0]) config.write_xml(xdict['filename'], xdict['xmltree']) def remove_group(self, group_name): """Remove a group.""" if self._use_db: msg = "Metadata does not support removing groups with use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._remove_xdata(self.groups_xml, "Group", group_name) def remove_bundle(self, bundle_name): """Remove a bundle.""" if self._use_db: msg = "Metadata does not support removing bundles with use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) def remove_client(self, client_name): """Remove a bundle.""" if self._use_db: try: client = MetadataClientModel.objects.get(hostname=client_name) except MetadataClientModel.DoesNotExist: msg = "Client %s does not exist" % client_name self.logger.warning(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) client.delete() self.clients = self.list_clients() else: return self._remove_xdata(self.clients_xml, "Client", client_name) def _handle_clients_xml_event(self, event): xdata = self.clients_xml.xdata self.clients = [] self.clientgroups = {} self.aliases = {} self.raliases = {} self.secure = [] self.floating = [] self.addresses = {} self.raddresses = {} for client in xdata.findall('.//Client'): clname = client.get('name').lower() if 'address' in client.attrib: caddr = client.get('address') if caddr in self.addresses: self.addresses[caddr].append(clname) else: self.addresses[caddr] = [clname] if clname not in self.raddresses: self.raddresses[clname] = set() self.raddresses[clname].add(caddr) if 'auth' in client.attrib: self.auth[client.get('name')] = client.get('auth', 'cert+password') if 'uuid' in client.attrib: self.uuid[client.get('uuid')] = clname if client.get('secure', 'false').lower() == 'true': self.secure.append(clname) if (client.get('location', 'fixed') == 'floating' or client.get('floating', 'false').lower() == 'true'): self.floating.append(clname) if 'password' in client.attrib: self.passwords[clname] = client.get('password') if 'version' in client.attrib: self.versions[clname] = client.get('version') self.raliases[clname] = set() for alias in client.findall('Alias'): self.aliases.update({alias.get('name'): clname}) self.raliases[clname].add(alias.get('name')) if 'address' not in alias.attrib: continue if alias.get('address') in self.addresses: self.addresses[alias.get('address')].append(clname) else: self.addresses[alias.get('address')] = [clname] if clname not in self.raddresses: self.raddresses[clname] = set() self.raddresses[clname].add(alias.get('address')) self.clients.append(clname) try: self.clientgroups[clname].append(client.get('profile')) except KeyError: self.clientgroups[clname] = [client.get('profile')] self.states['clients.xml'] = True if self._use_db: self.clients = self.list_clients() def _handle_groups_xml_event(self, event): self.groups = {} # these three functions must be separate functions in order to # ensure that the scope is right for the closures they return def get_condition(element): negate = element.get('negate', 'false').lower() == 'true' pname = element.get("name") if element.tag == 'Group': return lambda c, g, _: negate != (pname in g) elif element.tag == 'Client': return lambda c, g, _: negate != (pname == c) def get_category_condition(category, gname): def in_cat(client, groups, categories): if category in categories: if (gname not in self.groups or client not in self.groups[gname].warned): self.logger.warning("%s: Group %s suppressed by " "category %s; %s already a member " "of %s" % (self.name, gname, category, client, categories[category])) if gname in self.groups: self.groups[gname].warned.append(client) return False return True return in_cat def aggregate_conditions(conditions): return lambda client, groups, cats: \ all(cond(client, groups, cats) for cond in conditions) # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old # way, only Group tags under a Groups tag counted as # declarative. so we parse those first, and then parse the # other Group tags if they haven't already been declared. # this lets you set options on a group (e.g., public="false") # at the top level and then just use the name elsewhere, which # is the original behavior for grp in self.groups_xml.xdata.xpath("//Groups/Group") + \ self.groups_xml.xdata.xpath("//Groups/Group//Group"): if grp.get("name") in self.groups: continue self.groups[grp.get("name")] = \ MetadataGroup(grp.get("name"), bundles=[b.get("name") for b in grp.findall("Bundle")], category=grp.get("category"), is_profile=grp.get("profile", "false") == "true", is_public=grp.get("public", "false") == "true", is_private=grp.get("public", "true") == "false") if grp.get('default', 'false') == 'true': self.default = grp.get('name') self.group_membership = dict() self.negated_groups = dict() self.options = dict() # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is # infinitely recursive, so "all" elements really means _all_ # elements. We then manually filter out non-Group elements # since there doesn't seem to be a way to get Group elements # of arbitrary depth with particular ultimate ancestors in # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): if ((el.tag != 'Group' and el.tag != 'Client') or el.getchildren()): continue conditions = [] for parent in el.iterancestors(): cond = get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": self.negated_groups[aggregate_conditions(conditions)] = \ self.groups[gname] else: if self.groups[gname].category: conditions.append( get_category_condition(self.groups[gname].category, gname)) self.group_membership[aggregate_conditions(conditions)] = \ self.groups[gname] self.states['groups.xml'] = True def HandleEvent(self, event): """Handle update events for data files.""" for hdlr in self.handlers: aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(event.filename)) if hdlr(event): # clear the entire cache when we get an event for any # metadata file self.core.metadata_cache.expire() try: proc = getattr(self, "_handle_%s_event" % aname) except AttributeError: proc = self._handle_default_event proc(event) if False not in list(self.states.values()) and self.debug_flag: # check that all groups are real and complete. this is # just logged at a debug level because many groups might # be probed, and we don't want to warn about them. for client, groups in list(self.clientgroups.items()): for group in groups: if group not in self.groups: self.debug_log("Client %s set as nonexistent group %s" % (client, group)) for gname, ginfo in list(self.groups.items()): for group in ginfo.groups: if group not in self.groups: self.debug_log("Group %s set as nonexistent group %s" % (gname, group)) def set_profile(self, client, profile, addresspair, force=False): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet") if not force and profile not in self.groups: msg = "Profile group %s does not exist" % profile self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) group = self.groups[profile] if not force and not group.is_public: msg = "Cannot set client %s to private group %s" % (client, profile) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) if client in self.clients: if self._use_db: msg = "DBMetadata does not support asserting client profiles" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) profiles = [g for g in self.clientgroups[client] if g in self.groups and self.groups[g].is_profile] self.logger.info("Changing %s profile from %s to %s" % (client, profiles, profile)) self.update_client(client, dict(profile=profile)) if client in self.clientgroups: for p in profiles: self.clientgroups[client].remove(p) self.clientgroups[client].append(profile) else: self.clientgroups[client] = [profile] else: self.logger.info("Creating new client: %s, profile %s" % (client, profile)) if self._use_db: self.add_client(client) else: if addresspair in self.session_cache: # we are working with a uuid'd client self.add_client(self.session_cache[addresspair][1], dict(uuid=client, profile=profile, address=addresspair[0])) else: self.add_client(client, dict(profile=profile)) self.clients.append(client) self.clientgroups[client] = [profile] if not self._use_db: self.clients_xml.write() def set_version(self, client, version): """Set group parameter for provided client.""" if client in self.clients: if client not in self.versions or version != self.versions[client]: self.logger.info("Setting client %s version to %s" % (client, version)) self.update_client(client, dict(version=version)) self.versions[client] = version self.clients_xml.write() else: msg = "Cannot set version on non-existent client %s" % client self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) def resolve_client(self, addresspair, cleanup_cache=False): """Lookup address locally or in DNS to get a hostname.""" if addresspair in self.session_cache: # client _was_ cached, so there can be some expired # entries. we need to clean them up to avoid potentially # infinite memory swell cache_ttl = 90 if cleanup_cache: # remove entries for this client's IP address with # _any_ port numbers - perhaps a priority queue could # be faster? curtime = time.time() for addrpair in list(self.session_cache.keys()): if addresspair[0] == addrpair[0]: (stamp, _) = self.session_cache[addrpair] if curtime - stamp > cache_ttl: del self.session_cache[addrpair] # return the cached data try: (stamp, uuid) = self.session_cache[addresspair] if time.time() - stamp < cache_ttl: return self.session_cache[addresspair][1] except KeyError: # we cleaned all cached data for this client in cleanup_cache pass address = addresspair[0] if address in self.addresses: if len(self.addresses[address]) != 1: err = "Address %s has multiple reverse assignments; a uuid must be used" % address self.logger.error(err) raise Bcfg2.Server.Plugin.MetadataConsistencyError(err) return self.addresses[address][0] try: cname = socket.gethostbyaddr(address)[0].lower() if cname in self.aliases: return self.aliases[cname] return cname except socket.herror: warning = "address resolution error for %s" % address self.logger.warning(warning) raise Bcfg2.Server.Plugin.MetadataConsistencyError(warning) def _merge_groups(self, client, groups, categories=None): """ set group membership based on the contents of groups.xml and initial group membership of this client. Returns a tuple of (allgroups, categories)""" numgroups = -1 # force one initial pass if categories is None: categories = dict() while numgroups != len(groups): numgroups = len(groups) for predicate, group in self.group_membership.items(): if group.name in groups: continue if predicate(client, groups, categories): groups.add(group.name) if group.category: categories[group.category] = group.name for predicate, group in self.negated_groups.items(): if group.name not in groups: continue if predicate(client, groups, categories): groups.remove(group.name) if group.category: del categories[group.category] return (groups, categories) def get_initial_metadata(self, client): """Return the metadata for a given client.""" if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not been read yet") client = client.lower() if client in self.core.metadata_cache: return self.core.metadata_cache[client] if client in self.aliases: client = self.aliases[client] groups = set() categories = dict() profile = None if client not in self.clients: pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] elif self.default: pgroup = self.default if pgroup: self.set_profile(client, pgroup, (None, None), force=True) groups.add(pgroup) category = self.groups[pgroup].category if category: categories[category] = pgroup if (pgroup in self.groups and self.groups[pgroup].is_profile): profile = pgroup else: msg = "Cannot add new client %s; no default group set" % client self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) if client in self.clientgroups: for cgroup in self.clientgroups[client]: if cgroup in groups: continue if cgroup not in self.groups: self.groups[cgroup] = MetadataGroup(cgroup) category = self.groups[cgroup].category if category and category in categories: self.logger.warning("%s: Group %s suppressed by " "category %s; %s already a member " "of %s" % (self.name, cgroup, category, client, categories[category])) continue if category: categories[category] = cgroup groups.add(cgroup) # favor client groups for setting profile if not profile and self.groups[cgroup].is_profile: profile = cgroup groups, categories = self._merge_groups(client, groups, categories=categories) bundles = set() for group in groups: try: bundles.update(self.groups[group].bundles) except KeyError: self.logger.warning("%s: %s is a member of undefined group %s" % (self.name, client, group)) aliases = self.raliases.get(client, set()) addresses = self.raddresses.get(client, set()) version = self.versions.get(client, None) if client in self.passwords: password = self.passwords[client] else: password = None uuids = [item for item, value in list(self.uuid.items()) if value == client] if uuids: uuid = uuids[0] else: uuid = None if not profile: # one last ditch attempt at setting the profile profiles = [g for g in groups if g in self.groups and self.groups[g].is_profile] if len(profiles) >= 1: profile = profiles[0] rv = ClientMetadata(client, profile, groups, bundles, aliases, addresses, categories, uuid, password, version, self.query) if self.core.metadata_cache_mode == 'initial': self.core.metadata_cache[client] = rv return rv def get_all_group_names(self): all_groups = set() all_groups.update(self.groups.keys()) all_groups.update([g.name for g in self.group_membership.values()]) all_groups.update([g.name for g in self.negated_groups.values()]) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups def get_all_groups_in_category(self, category): return set([g.name for g in self.groups.values() if g.category == category]) def get_client_names_by_profiles(self, profiles): rv = [] for client in list(self.clients): mdata = self.get_initial_metadata(client) if mdata.profile in profiles: rv.append(client) return rv def get_client_names_by_groups(self, groups): mdata = [self.core.build_metadata(client) for client in list(self.clients)] return [md.hostname for md in mdata if md.groups.issuperset(groups)] def get_client_names_by_bundles(self, bundles): mdata = [self.core.build_metadata(client) for client in list(self.clients.keys())] return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] def merge_additional_groups(self, imd, groups): for group in groups: if group in imd.groups: continue if group in self.groups and self.groups[group].category: category = self.groups[group].category if self.groups[group].category in imd.categories: self.logger.warning("%s: Group %s suppressed by category " "%s; %s already a member of %s" % (self.name, group, category, imd.hostname, imd.categories[category])) continue imd.categories[group] = category imd.groups.add(group) self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) def merge_additional_data(self, imd, source, data): if not hasattr(imd, source): setattr(imd, source, data) imd.connectors.append(source) def validate_client_address(self, client, addresspair): """Check address against client.""" address = addresspair[0] if client in self.floating: self.debug_log("Client %s is floating" % client) return True if address in self.addresses: if client in self.addresses[address]: self.debug_log("Client %s matches address %s" % (client, address)) return True else: self.logger.error("Got request for non-float client %s from %s" % (client, address)) return False resolved = self.resolve_client(addresspair) if resolved.lower() == client.lower(): return True else: self.logger.error("Got request for %s from incorrect address %s" % (client, address)) self.logger.error("Resolved to %s" % resolved) return False def AuthenticateConnection(self, cert, user, password, address): """This function checks auth creds.""" if not isinstance(user, str): user = user.decode('utf-8') if cert: id_method = 'cert' certinfo = dict([x[0] for x in cert['subject']]) # look at cert.cN client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) auth_type = self.auth.get(client, 'cert+password') elif user == 'root': id_method = 'address' try: client = self.resolve_client(address) except Bcfg2.Server.Plugin.MetadataConsistencyError: err = sys.exc_info()[1] self.logger.error("Client %s failed to resolve: %s" % (address[0], err)) return False else: id_method = 'uuid' # user maps to client if user not in self.uuid: client = user self.uuid[user] = user else: client = self.uuid[user] # we have the client name self.debug_log("Authenticating client %s" % client) # next we validate the address if id_method == 'uuid': addr_is_valid = True else: addr_is_valid = self.validate_client_address(client, address) if not addr_is_valid: return False if id_method == 'cert' and auth_type != 'cert+password': # remember the cert-derived client name for this connection if client in self.floating: self.session_cache[address] = (time.time(), client) # we are done if cert+password not required return True if client not in self.passwords: if client in self.secure: self.logger.error("Client %s in secure mode but has no password" % address[0]) return False if password != self.password: self.logger.error("Client %s used incorrect global password" % address[0]) return False if client not in self.secure: if client in self.passwords: plist = [self.password, self.passwords[client]] else: plist = [self.password] if password not in plist: self.logger.error("Client %s failed to use either allowed " "password" % address[0]) return False else: # client in secure mode and has a client password if password != self.passwords[client]: self.logger.error("Client %s failed to use client password in " "secure mode" % address[0]) return False # populate the session cache if user != 'root': self.session_cache[address] = (time.time(), client) return True def process_statistics(self, meta, _): """Hook into statistics interface to toggle clients in bootstrap mode.""" client = meta.hostname if client in self.auth and self.auth[client] == 'bootstrap': self.update_client(client, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): """Admin mode viz support.""" if only_client: clientmeta = self.core.build_metadata(only_client) def include_client(client): return not only_client or client != only_client def include_bundle(bundle): return not only_client or bundle in clientmeta.bundles def include_group(group): return not only_client or group in clientmeta.groups groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"), parser=Bcfg2.Server.XMLParser) try: groups_tree.xinclude() except lxml.etree.XIncludeError: self.logger.error("Failed to process XInclude for file %s: %s" % (dest, sys.exc_info()[1])) groups = groups_tree.getroot() categories = {'default': 'grey83'} viz_str = [] egroups = groups.findall("Group") + groups.findall('.//Groups/Group') for group in egroups: if not group.get('category') in categories: categories[group.get('category')] = colors.pop() group.set('color', categories[group.get('category')]) if None in categories: del categories[None] if hosts: instances = {} for client in list(self.clients): if include_client(client): continue if client in self.clientgroups: groups = self.clientgroups[client] elif self.default: groups = [self.default] else: continue for group in groups: try: instances[group].append(client) except KeyError: instances[group] = [client] for group, clist in list(instances.items()): clist.sort() viz_str.append('"%s-instances" [ label="%s", shape="record" ];' % (group, '|'.join(clist))) viz_str.append('"%s-instances" -> "group-%s";' % (group, group)) if bundles: bundles = [] [bundles.append(bund.get('name')) \ for bund in groups.findall('.//Bundle') \ if bund.get('name') not in bundles \ and include_bundle(bund.get('name'))] bundles.sort() for bundle in bundles: viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' % (bundle, bundle)) gseen = [] for group in egroups: if group.get('profile', 'false') == 'true': style = "filled, bold" else: style = "filled" gseen.append(group.get('name')) if include_group(group.get('name')): viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' % (group.get('name'), group.get('name'), style, group.get('color'))) if bundles: for bundle in group.findall('Bundle'): viz_str.append('"group-%s" -> "bundle-%s";' % (group.get('name'), bundle.get('name'))) gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];' for group in egroups: for parent in group.findall('Group'): if parent.get('name') not in gseen and include_group(parent.get('name')): viz_str.append(gfmt % (parent.get('name'), parent.get('name'))) gseen.append(parent.get("name")) if include_group(group.get('name')): viz_str.append('"group-%s" -> "group-%s";' % (group.get('name'), parent.get('name'))) if key: for category in categories: viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' % (category, category, categories[category])) return "\n".join("\t" + s for s in viz_str) class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): def Run(self): self.nested_clients() self.deprecated_options() @classmethod def Errors(cls): return {"nested-client-tags": "warning", "deprecated-clients-options": "warning"} def deprecated_options(self): clientdata = self.metadata.clients_xml.xdata for el in groupdata.xpath("//Client"): loc = el.get("location") if loc: if loc == "floating": floating = True else: floating = False self.LintError("deprecated-clients-options", "The location='%s' option is deprecated. " "Please use floating='%s' instead: %s" % (loc, floating, self.RenderXML(el))) def nested_clients(self): groupdata = self.metadata.groups_xml.xdata for el in groupdata.xpath("//Client//Client"): self.LintError("nested-client-tags", "Client %s nested within Client tag: %s" % (el.get("name"), self.RenderXML(el)))