diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/POSIX/File.py | 17 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/POSIX/base.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/__init__.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Reporting/Collector.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/BuiltinCore.py | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 248 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/NagiosGen.py | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Probes.py | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/SSLServer.py | 32 |
9 files changed, 191 insertions, 129 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 482320e0d..c12188e1c 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -54,6 +54,10 @@ class POSIXFile(POSIXTool): def verify(self, entry, modlist): ondisk = self._exists(entry) tempdata, is_binary = self._get_data(entry) + if isinstance(tempdata, str) and str != unicode: + tempdatasize = len(tempdata) + else: + tempdatasize = len(tempdata.encode(self.setup['encoding'])) different = False content = None @@ -62,7 +66,7 @@ class POSIXFile(POSIXTool): # they're clearly different different = True content = "" - elif len(tempdata) != ondisk[stat.ST_SIZE]: + elif tempdatasize != ondisk[stat.ST_SIZE]: # next, see if the size of the target file is different # from the size of the desired content different = True @@ -73,6 +77,9 @@ class POSIXFile(POSIXTool): # for everything else try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), + encoding=self.setup['encoding']).read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) @@ -90,7 +97,7 @@ class POSIXFile(POSIXTool): def _write_tmpfile(self, entry): """ Write the file data to a temp file """ - filedata, _ = self._get_data(entry) + filedata = self._get_data(entry)[0] # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions # protections on that directory, and also to avoid issues with @@ -106,7 +113,11 @@ class POSIXFile(POSIXTool): (os.path.dirname(entry.get('name')), err)) return False try: - os.fdopen(newfd, 'w').write(filedata) + if isinstance(filedata, str) and str != unicode: + os.fdopen(newfd, 'w').write(filedata) + else: + os.fdopen(newfd, 'wb').write( + filedata.encode(self.setup['encoding'])) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index c9164cb88..bd2f8f87e 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -525,7 +525,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if entry.get("secontext") == "__default__": try: wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + selinux.matchpathcon( + path, ondisk[stat.ST_MODE])[1].split(":")[2] except OSError: errors.append("%s has no default SELinux context" % entry.get("name")) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index aab2459f2..4a808aa60 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -590,14 +590,15 @@ class SvcTool(Tool): if not self.handlesEntry(entry): continue + estatus = entry.get('status') restart = entry.get("restart", "true").lower() - if (restart == "false" or + if (restart == "false" or estatus == 'ignore' or (restart == "interactive" and not Bcfg2.Options.setup.interactive)): continue success = False - if entry.get('status') == 'on': + if estatus == 'on': if Bcfg2.Options.setup.service_mode == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index ccd175dc3..a93f1b0ae 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -78,7 +78,7 @@ class ReportingCollector(object): """Startup the processing and go!""" self.terminate = threading.Event() atexit.register(self.shutdown) - self.context = daemon.DaemonContext() + self.context = daemon.DaemonContext(detach_process=True) if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 179a6aa9f..0023e9313 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -113,7 +113,6 @@ class BuiltinCore(NetworkCore): keyfile=Bcfg2.Options.setup.key, certfile=Bcfg2.Options.setup.cert, register=False, - timeout=1, ca=Bcfg2.Options.setup.ca) except: # pylint: disable=W0702 err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 17352dedc..db104b27e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -545,15 +545,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} - # mappings of predicate -> MetadataGroup object + # mappings of groupname -> [predicates] self.group_membership = dict() self.negated_groups = dict() + # list of group names in document order + self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions(core, # pylint: disable=E1102 datastore) else: self.versions = dict() + self.uuid = {} self.session_cache = {} self.cache = Cache("Metadata") @@ -561,7 +564,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.pdirty = False self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients), + self.list_clients, self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -851,51 +854,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() + def _get_condition(self, element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ + negate = element.get('negate', 'false').lower() == 'true' + pname = element.get("name") + if element.tag == 'Group': + return lambda c, g, _: negate != (pname in g) + elif element.tag == 'Client': + return lambda c, g, _: negate != (pname == c) + + def _get_category_condition(self, grpname): + """ get a predicate that returns False if a client is already + a member of a group in the given group's category, True + otherwise""" + return lambda client, _, categories: \ + bool(self._check_category(client, grpname, categories)) + + def _aggregate_conditions(self, conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ + return lambda client, groups, cats: \ + all(cond(client, groups, cats) for cond in conditions) + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ self.groups = {} - # these three functions must be separate functions in order to - # ensure that the scope is right for the closures they return - def get_condition(element): - """ Return a predicate that returns True if a client meets - the condition specified in the given Group or Client - element """ - negate = element.get('negate', 'false').lower() == 'true' - pname = element.get("name") - if element.tag == 'Group': - return lambda c, g, _: negate != (pname in g) - elif element.tag == 'Client': - return lambda c, g, _: negate != (pname == c) - - def get_category_condition(category, gname): - """ get a predicate that returns False if a client is - already a member of a group in the given category, True - otherwise """ - def in_cat(client, groups, categories): # pylint: disable=W0613 - """ return True if the client is already a member of a - group in the category given in the enclosing function, - False otherwise """ - if category in categories: - if (gname not in self.groups or - client not in self.groups[gname].warned): - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, gname, category, - client, categories[category])) - if gname in self.groups: - self.groups[gname].warned.append(client) - return False - return True - return in_cat - - def aggregate_conditions(conditions): - """ aggregate all conditions on a given group declaration - into a single predicate """ - return lambda client, groups, cats: \ - all(cond(client, groups, cats) for cond in conditions) - # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old @@ -921,6 +907,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() + self.ordered_groups = [] # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is @@ -931,28 +918,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): - if ((el.tag != 'Group' and el.tag != 'Client') or - el.getchildren()): + if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren(): continue conditions = [] for parent in el.iterancestors(): - cond = get_condition(parent) + cond = self._get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[aggregate_conditions(conditions)] = \ - self.groups[gname] + self.negated_groups.setdefault(gname, []) + self.negated_groups[gname].append( + self._aggregate_conditions(conditions)) else: if self.groups[gname].category: - conditions.append( - get_category_condition(self.groups[gname].category, - gname)) + conditions.append(self._get_category_condition(gname)) - self.group_membership[aggregate_conditions(conditions)] = \ - self.groups[gname] + if gname not in self.ordered_groups: + self.ordered_groups.append(gname) + self.group_membership.setdefault(gname, []) + self.group_membership[gname].append( + self._aggregate_conditions(conditions)) self.states['groups.xml'] = True def HandleEvent(self, event): @@ -962,6 +950,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # clear the entire cache when we get an event for any # metadata file self.cache.expire() + + # clear out the list of category suppressions that + # have been warned about, since this may change when + # clients.xml or groups.xml changes. + for group in self.groups.values(): + group.warned = [] event_handler(event) if False not in list(self.states.values()) and self.debug_flag: @@ -1100,22 +1094,77 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() while numgroups != len(groups): numgroups = len(groups) - for predicate, group in self.group_membership.items(): - if group.name in groups: + newgroups = set() + removegroups = set() + for grpname in self.ordered_groups: + if grpname in groups: continue - if predicate(client, groups, categories): - groups.add(group.name) - if group.category: - categories[group.category] = group.name - for predicate, group in self.negated_groups.items(): - if group.name not in groups: + if any(p(client, groups, categories) + for p in self.group_membership[grpname]): + newgroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + categories[self.groups[grpname].category] = grpname + groups.update(newgroups) + for grpname, predicates in self.negated_groups.items(): + if grpname not in groups: continue - if predicate(client, groups, categories): - groups.remove(group.name) - if group.category: - del categories[group.category] + if any(p(client, groups, categories) for p in predicates): + removegroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + del categories[self.groups[grpname].category] + groups.difference_update(removegroups) return (groups, categories) + def _check_category(self, client, grpname, categories): + """ Determine if the given client is already a member of a + group in the same category as the named group. + + The return value is one of three possibilities: + + * If the client is already a member of a group in the same + category, then False is returned (i.e., the category check + failed); + * If the group is not in any categories, then True is returned; + * If the group is not a member of a group in the category, + then the name of the category is returned. This makes it + easy to add the category to the ClientMetadata object (or + other category list). + + If a pure boolean value is required, you can do + ``bool(self._check_category(...))``. + """ + if grpname not in self.groups: + return True + category = self.groups[grpname].category + if not category: + return True + if category in categories: + if client not in self.groups[grpname].warned: + self.logger.warning("%s: Group %s suppressed by category %s; " + "%s already a member of %s" % + (self.name, grpname, category, + client, categories[category])) + self.groups[grpname].warned.append(client) + return False + return category + + def _check_and_add_category(self, client, grpname, categories): + """ If the client is not a member of a group in the same + category as the named group, then the category is added to + ``categories``. + :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used + to determine if the category can be added. + + If the category check failed, returns False; otherwise, + returns True. """ + rv = self._check_category(client, grpname, categories) + if rv and rv is not True: + categories[rv] = grpname + return True + return rv + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): @@ -1137,30 +1186,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Handles setting categories and category suppression. Returns the new profile for the client (which might be unchanged). """ - groups.add(grpname) if grpname in self.groups: - group = self.groups[grpname] - category = group.category - if category: - if category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, grpname, category, - client, categories[category])) - return - categories[category] = grpname - if not profile and group.is_profile: + if not self._check_and_add_category(client, grpname, + categories): + return profile + groups.add(grpname) + if not profile and self.groups[grpname].is_profile: return grpname else: return profile + else: + groups.add(grpname) + return profile if client not in self.clients: pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] + self.debug_log("%s: Adding new client with profile %s" % + (self.name, pgroup)) elif self.default: pgroup = self.default + self.debug_log("%s: Adding new client with default profile %s" + % (self.name, pgroup)) if pgroup: self.set_profile(client, pgroup, (None, None), @@ -1177,6 +1225,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[cgroup] = MetadataGroup(cgroup) profile = _add_group(cgroup) + # we do this before setting the default because there may be + # groups set in <Client> tags in groups.xml that we want to + # set groups, categories = self._merge_groups(client, groups, categories=categories) @@ -1225,8 +1276,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) - all_groups.update([g.name for g in self.group_membership.values()]) - all_groups.update([g.name for g in self.negated_groups.values()]) + all_groups.update(self.group_membership.keys()) + all_groups.update(self.negated_groups.keys()) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups @@ -1239,7 +1290,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.clients): + for client in self.list_clients(): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1247,34 +1298,33 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.groups.issuperset(groups): + rv.append(client) + return rv def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.bundles.issuperset(bundles): + rv.append(client) + return rv def merge_additional_groups(self, imd, groups): for group in groups: if group in imd.groups: continue - if group in self.groups and self.groups[group].category: - category = self.groups[group].category - if self.groups[group].category in imd.categories: - self.logger.warning("%s: Group %s suppressed by category " - "%s; %s already a member of %s" % - (self.name, group, category, - imd.hostname, - imd.categories[category])) - continue - imd.categories[category] = group + if not self._check_and_add_category(imd.hostname, group, + imd.categories): + continue imd.groups.add(group) - self._merge_groups(imd.hostname, imd.groups, - categories=imd.categories) - + self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) @@ -1439,7 +1489,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] - for client in list(self.clients): + for client in list(self.list_clients()): if not include_client(client): continue if client in self.clientgroups: diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index c807d51f4..a27664215 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -21,9 +21,9 @@ class NagiosGen(Plugin, Generator): self.config = \ StructFile(os.path.join(self.data, 'config.xml'), should_monitor=True, create=self.name) - self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/conf.d/bcfg2.cfg': self.createserverconfig}} + self.Entries = { + 'Path': {'/etc/nagiosgen.status': self.createhostconfig, + '/etc/nagios/conf.d/bcfg2.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index ef339a34b..010a69dd1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -10,6 +10,7 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Server.Cache import Bcfg2.Server.Plugin +from Bcfg2.Compat import unicode # pylint: disable=W0622 import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics @@ -285,7 +286,10 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + if isinstance(data, unicode): + return str.__new__(cls, data.encode('utf-8')) + else: + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 646124fcc..215fd0e2b 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -5,7 +5,6 @@ better. """ import os import sys import socket -import select import signal import logging import ssl @@ -237,22 +236,23 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): return False return True - ### need to override do_POST here def do_POST(self): try: max_chunk_size = 10 * 1024 * 1024 size_remaining = int(self.headers["content-length"]) L = [] while size_remaining: - try: - select.select([self.rfile.fileno()], [], [], 3) - except select.error: - self.logger.error("Got select timeout") - raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size).decode('utf-8')) + chunk = self.rfile.read(chunk_size) + if not chunk: + break + L.append(chunk) size_remaining -= len(L[-1]) data = ''.join(L) + data = self.decode_request_content(data) + if data is None: + return # response has been sent + response = self.server._marshaled_dispatch(self.client_address, data) if sys.hexversion >= 0x03000000: @@ -265,6 +265,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): (self.client_address, sys.exc_info()[1])) try: self.send_response(500) + self.send_header("Content-length", "0") self.end_headers() except: (etype, msg) = sys.exc_info()[:2] @@ -321,14 +322,11 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def finish(self): # shut down the connection - if not self.wfile.closed: - try: - self.wfile.flush() - self.wfile.close() - except socket.error: - err = sys.exc_info()[1] - self.logger.warning("Error closing connection: %s" % err) - self.rfile.close() + try: + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.finish(self) + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, @@ -446,8 +444,6 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, self.handle_request() except socket.timeout: pass - except select.error: - pass except: self.logger.error("Got unexpected error in handle_request", exc_info=1) |