From 29a95907c36cd3e791bdb281948324c719a50f77 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 10 Aug 2013 10:03:07 -0500 Subject: Revert "Probes: Fix failing nosetests" This reverts commit 4f745cc2731f7035f02566ba8bc1a0e9ae1b1a71. This breaks handling of unicode on python 2. --- src/lib/Bcfg2/Server/Plugins/Probes.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index b9f93052a..35ae9d479 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -64,7 +64,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + return str.__new__(cls, data.encode('utf-8')) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -225,9 +225,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=self.probedata[client][probe]) + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str( + self.probedata[client][probe]).decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: -- cgit v1.2.3-1-g7c22 From 3b6b21c41101f1bb8be5cae096953b3907a38838 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 10 Aug 2013 10:04:03 -0500 Subject: Probes: Fix unicode probe handling Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Probes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 35ae9d479..ad25de194 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -9,6 +9,7 @@ import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin +from Bcfg2.Compat import unicode try: from django.db import models @@ -64,7 +65,10 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data.encode('utf-8')) + if isinstance(data, unicode): + return str.__new__(cls, data.encode('utf-8')) + else: + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) -- cgit v1.2.3-1-g7c22 From b425fdec5edc55ce44d6f7f0e9dad89df14e4814 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 10 Aug 2013 10:17:05 -0500 Subject: Probes: Disable builtin redefinition check Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Probes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index ad25de194..fd6fd3bd1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -9,7 +9,7 @@ import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin -from Bcfg2.Compat import unicode +from Bcfg2.Compat import unicode # pylint: disable=W0622 try: from django.db import models -- cgit v1.2.3-1-g7c22 From 30dd41484c7ecff5e047279a8d8ada3672d7dc55 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 10 Aug 2013 10:21:55 -0500 Subject: POSIX: Fix unicode file verification Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Tools/POSIX/File.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 9f47fb53a..1fdb95b32 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -53,6 +53,10 @@ class POSIXFile(POSIXTool): def verify(self, entry, modlist): ondisk = self._exists(entry) tempdata, is_binary = self._get_data(entry) + if isinstance(tempdata, str) and str != unicode: + tempdatasize = len(tempdata) + else: + tempdatasize = len(tempdata.encode(self.setup['encoding'])) different = False content = None @@ -61,7 +65,7 @@ class POSIXFile(POSIXTool): # they're clearly different different = True content = "" - elif len(tempdata) != ondisk[stat.ST_SIZE]: + elif tempdatasize != ondisk[stat.ST_SIZE]: # next, see if the size of the target file is different # from the size of the desired content different = True @@ -72,6 +76,9 @@ class POSIXFile(POSIXTool): # for everything else try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), + encoding=self.setup['encoding']).read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) @@ -89,7 +96,7 @@ class POSIXFile(POSIXTool): def _write_tmpfile(self, entry): """ Write the file data to a temp file """ - filedata, _ = self._get_data(entry) + filedata = self._get_data(entry)[0] # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions # protections on that directory, and also to avoid issues with @@ -105,7 +112,10 @@ class POSIXFile(POSIXTool): (os.path.dirname(entry.get('name')), err)) return False try: - os.fdopen(newfd, 'w').write(filedata) + if isinstance(filedata, str) and str != unicode: + os.fdopen(newfd, 'w').write(filedata) + else: + os.fdopen(newfd, 'wb').write(filedata.encode(self.setup['encoding'])) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " -- cgit v1.2.3-1-g7c22 From 8b1fa7e4868cd75fd541ca4a2295923f185d953d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 10 Aug 2013 10:35:55 -0500 Subject: POSIX: Fix pylint failure Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Tools/POSIX/File.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 1fdb95b32..b1bde1057 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -115,7 +115,8 @@ class POSIXFile(POSIXTool): if isinstance(filedata, str) and str != unicode: os.fdopen(newfd, 'w').write(filedata) else: - os.fdopen(newfd, 'wb').write(filedata.encode(self.setup['encoding'])) + os.fdopen(newfd, 'wb').write( + filedata.encode(self.setup['encoding'])) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " -- cgit v1.2.3-1-g7c22 From 924457a917eba6b04c57dbc1fa003dbcb60daca5 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 12 Aug 2013 10:56:05 -0500 Subject: Client: Don't modify running services if ignored Previously, when a bundle's contents were updated, all services listed in the bundle were stopped if the status attribute was set to 'ignore'. Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Tools/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 11fe55bd6..703b8ff57 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -594,13 +594,14 @@ class SvcTool(Tool): if not self.handlesEntry(entry): continue + estatus = entry.get('status') restart = entry.get("restart", "true").lower() - if (restart == "false" or + if (restart == "false" or estatus == 'ignore' or (restart == "interactive" and not self.setup['interactive'])): continue success = False - if entry.get('status') == 'on': + if estatus == 'on': if self.setup['servicemode'] == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: -- cgit v1.2.3-1-g7c22 From 8bfcf90be04fbad5abec66467ffcdd8c48299129 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 12 Aug 2013 13:02:06 -0400 Subject: Metadata: log now client creation with profile --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index bb0fad011..dd8805ab0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1143,8 +1143,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] + self.debug_log("%s: Adding new client with profile %s" % + (self.name, pgroup)) elif self.default: pgroup = self.default + self.debug_log("%s: Adding new client with default profile %s" + % (self.name, pgroup)) if pgroup: self.set_profile(client, pgroup, (None, None), -- cgit v1.2.3-1-g7c22 From cf8860931cb1f39c1606b73b3f2df9e9f4a2ebe1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 14 Aug 2013 13:06:05 -0400 Subject: tools: added git_commit.py script --- tools/README | 4 +- tools/git_commit.py | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100755 tools/git_commit.py diff --git a/tools/README b/tools/README index 9e7f667e3..8fe2c1a28 100644 --- a/tools/README +++ b/tools/README @@ -61,6 +61,9 @@ export.sh generate-manpages.bash - Generate man pages from the Sphinx source +git_commit.py + - Trigger script to commit local changes back to a git repository + hostbasepush.py - Call the Hostbase.rebuildState XML-RPC method @@ -100,4 +103,3 @@ upgrade yum-listpkgs-xml.py - Produces a list of all packages installed and available in a format suitable for use by Packages or Pkgmgr - diff --git a/tools/git_commit.py b/tools/git_commit.py new file mode 100755 index 000000000..cc4061f25 --- /dev/null +++ b/tools/git_commit.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +""" Trigger script to commit selected changes to a local repository +back to git. To use this script, enable the Trigger plugin, put this +script in /var/lib/bcfg2/Trigger/, and create /etc/bcfg2-commit.conf. + +The config file, /etc/bcfg2-commit.conf, may contain four options in +the [global] section: + +* "config" is the path to the Bcfg2 server config file. (Default: + /etc/bcfg2.conf) +* "commit" is a comma-separated list of globs giving the paths that + should be committed back to the repository. Default is 'SSLCA/*, + SSHbase/*, Cfg/*', which will commit data back for SSLCA, SSHbase, + Cfg, FileProbes, etc., but not, for instance, Probes/probed.xml. + You may wish to add Metadata/clients.xml to the commit list. +* "debug" and "verbose" let you set the log level for git_commit.py + itself. +""" + + +import os +import sys +import git +import logging +import Bcfg2.Logger +import Bcfg2.Options +from Bcfg2.Compat import ConfigParser +from fnmatch import fnmatch + +# config file path +CONFIG = "/etc/bcfg2-commit.conf" + +# config defaults. all config options are in the [global] section +DEFAULTS = dict(config='/etc/bcfg2.conf', + commit="SSLCA/*, SSHbase/*, Cfg/*") + + +def list_changed_files(repo): + return [d for d in repo.index.diff(None) + if (d.a_blob is not None and not d.deleted_file and + not d.renamed and not d.new_file)] + + +def add_to_commit(patterns, path, repo, relpath): + progname = os.path.basename(sys.argv[0]) + logger = logging.getLogger(progname) + for pattern in patterns: + if fnmatch(path, os.path.join(relpath, pattern)): + logger.debug("%s: Adding %s to commit" % (progname, path)) + repo.index.add([path]) + return True + return False + + +def parse_options(): + config = ConfigParser.SafeConfigParser(DEFAULTS) + config.read(CONFIG) + + optinfo = dict( + profile=Bcfg2.Options.CLIENT_PROFILE, + dryrun=Bcfg2.Options.CLIENT_DRYRUN, + groups=Bcfg2.Options.Option("Groups", + default=[], + cmd="-g", + odesc=':', + cook=Bcfg2.Options.colon_split)) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + argv = [Bcfg2.Options.CFILE.cmd, config.get("global", "config")] + argv.extend(sys.argv[1:]) + setup = Bcfg2.Options.OptionParser(optinfo, argv=argv) + setup.parse(argv) + + setup['commit'] = Bcfg2.Options.list_split(config.get("global", + "commit")) + for opt in ['debug', 'verbose']: + try: + setup[opt] = config.getboolean("global", opt) + except ConfigParser.NoOptionError: + pass + + try: + hostname = setup['args'][0] + except IndexError: + print(setup.hm) + raise SystemExit(1) + return (setup, hostname) + + +def setup_logging(setup): + progname = os.path.basename(sys.argv[0]) + log_args = dict(to_syslog=setup['syslog'], to_console=sys.stdout.isatty(), + to_file=setup['logging'], level=logging.WARNING) + if setup['debug']: + log_args['level'] = logging.DEBUG + elif setup['verbose']: + log_args['level'] = logging.INFO + Bcfg2.Logger.setup_logging(progname, **log_args) + return logging.getLogger(progname) + + +def main(): + progname = os.path.basename(sys.argv[0]) + setup, hostname = parse_options() + logger = setup_logging(setup) + if setup['dryrun']: + logger.info("%s: In dry-run mode, changes will not be committed" % + progname) + + if setup['vcs_root']: + gitroot = os.path.realpath(setup['vcs_root']) + else: + gitroot = os.path.realpath(setup['repo']) + logger.info("%s: Using Git repo at %s" % (progname, gitroot)) + try: + repo = git.Repo(gitroot) + except: # pylint: disable=W0702 + logger.error("%s: Error setting up Git repo at %s: %s" % + (progname, gitroot, sys.exc_info()[1])) + return 1 + + # canonicalize the repo path so that git will recognize it as + # being inside the git repo + bcfg2root = os.path.realpath(setup['repo']) + + if not bcfg2root.startswith(gitroot): + logger.error("%s: Bcfg2 repo %s is not inside Git repo %s" % + (progname, bcfg2root, gitroot)) + return 1 + + # relative path to Bcfg2 root from VCS root + if gitroot == bcfg2root: + relpath = '' + else: + relpath = bcfg2root[len(gitroot) + 1:] + + new = 0 + changed = 0 + logger.debug("%s: Untracked files: %s" % (progname, repo.untracked_files)) + for path in repo.untracked_files: + if add_to_commit(setup['commit'], path, repo, relpath): + new += 1 + else: + logger.debug("%s: Not adding %s to commit" % (progname, path)) + logger.debug("%s: Untracked files after building commit: %s" % + (progname, repo.untracked_files)) + + changes = list_changed_files(repo) + logger.info("%s: Changed files: %s" % (progname, + [d.a_blob.path for d in changes])) + for diff in changes: + if add_to_commit(setup['commit'], diff.a_blob.path, repo, relpath): + changed += 1 + else: + logger.debug("%s: Not adding %s to commit" % (progname, + diff.a_blob.path)) + logger.info("%s: Changed files after building commit: %s" % + (progname, [d.a_blob.path for d in list_changed_files(repo)])) + + if new + changed > 0: + logger.debug("%s: Committing %s new files and %s changed files" % + (progname, new, changed)) + if setup['dryrun']: + logger.warning("%s: In dry-run mode, skipping commit and push" % + progname) + else: + output = repo.index.commit("Auto-commit with %s from %s run" % + (progname, hostname)) + if output: + logger.debug("%s: %s" % (progname, output)) + remote = repo.remote() + logger.debug("%s: Pushing to remote %s at %s" % (progname, remote, + remote.url)) + output = remote.push() + if output: + logger.debug("%s: %s" % (progname, output)) + else: + logger.info("%s: No changes to commit" % progname) + +if __name__ == '__main__': + sys.exit(main()) -- cgit v1.2.3-1-g7c22 From 4afe2a1b06823032b588d932e2dc9c0a9c95809e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 14 Aug 2013 15:36:54 -0400 Subject: Metadata: metadata generation improvements Primarily, this fixes a few potential bugs in initial metadata, which may result in inconsistent ordering of groups, which can be a problem when relying on category suppression. This also deduplicates some category suppression code, and moves some nested functions to top-level methods, which are surprisingly faster. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 167 ++++++++++++++++++------------- 1 file changed, 97 insertions(+), 70 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index dd8805ab0..ac31ecffe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -832,51 +832,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() + def _get_condition(self, element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ + negate = element.get('negate', 'false').lower() == 'true' + pname = element.get("name") + if element.tag == 'Group': + return lambda c, g, _: negate != (pname in g) + elif element.tag == 'Client': + return lambda c, g, _: negate != (pname == c) + + def _get_category_condition(self, grpname): + """ get a predicate that returns False if a client is already + a member of a group in the given group's category, True + otherwise""" + return lambda client, _, categories: \ + bool(self._check_category(client, grpname, categories)) + + def _aggregate_conditions(self, conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ + return lambda client, groups, cats: \ + all(cond(client, groups, cats) for cond in conditions) + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ self.groups = {} - # these three functions must be separate functions in order to - # ensure that the scope is right for the closures they return - def get_condition(element): - """ Return a predicate that returns True if a client meets - the condition specified in the given Group or Client - element """ - negate = element.get('negate', 'false').lower() == 'true' - pname = element.get("name") - if element.tag == 'Group': - return lambda c, g, _: negate != (pname in g) - elif element.tag == 'Client': - return lambda c, g, _: negate != (pname == c) - - def get_category_condition(category, gname): - """ get a predicate that returns False if a client is - already a member of a group in the given category, True - otherwise """ - def in_cat(client, groups, categories): # pylint: disable=W0613 - """ return True if the client is already a member of a - group in the category given in the enclosing function, - False otherwise """ - if category in categories: - if (gname not in self.groups or - client not in self.groups[gname].warned): - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, gname, category, - client, categories[category])) - if gname in self.groups: - self.groups[gname].warned.append(client) - return False - return True - return in_cat - - def aggregate_conditions(conditions): - """ aggregate all conditions on a given group declaration - into a single predicate """ - return lambda client, groups, cats: \ - all(cond(client, groups, cats) for cond in conditions) - # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old @@ -918,21 +901,20 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, conditions = [] for parent in el.iterancestors(): - cond = get_condition(parent) + cond = self._get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[aggregate_conditions(conditions)] = \ + self.negated_groups[self._aggregate_conditions(conditions)] = \ self.groups[gname] else: if self.groups[gname].category: - conditions.append( - get_category_condition(self.groups[gname].category, - gname)) + conditions.append(self._get_category_condition(gname)) - self.group_membership[aggregate_conditions(conditions)] = \ + self.group_membership[ + self._aggregate_conditions(conditions)] = \ self.groups[gname] self.states['groups.xml'] = True @@ -946,6 +928,12 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # clear the entire cache when we get an event for any # metadata file self.expire_cache() + + # clear out the list of category suppressions that + # have been warned about, since this may change when + # clients.xml or groups.xml changes. + for group in self.groups.values(): + group.warned = [] event_handler(event) if False not in list(self.states.values()) and self.debug_flag: @@ -1100,6 +1088,54 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, del categories[group.category] return (groups, categories) + def _check_category(self, client, grpname, categories): + """ Determine if the given client is already a member of a + group in the same category as the named group. + + The return value is one of three possibilities: + + * If the client is already a member of a group in the same + category, then False is returned (i.e., the category check + failed); + * If the group is not in any categories, then True is returned; + * If the group is not a member of a group in the category, + then the name of the category is returned. This makes it + easy to add the category to the ClientMetadata object (or + other category list). + + If a pure boolean value is required, you can do + ``bool(self._check_category(...))``. + """ + if grpname not in self.groups: + return True + category = self.groups[grpname].category + if not category: + return True + if category in categories: + if client not in self.groups[grpname].warned: + self.logger.warning("%s: Group %s suppressed by category %s; " + "%s already a member of %s" % + (self.name, grpname, category, + client, categories[category])) + self.groups[grpname].warned.append(client) + return False + return category + + def _check_and_add_category(self, client, grpname, categories): + """ If the client is not a member of a group in the same + category as the named group, then the category is added to + ``categories``. + :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used + to determine if the category can be added. + + If the category check failed, returns False; otherwise, + returns True. """ + rv = self._check_category(client, grpname, categories) + if rv and rv is not True: + categories[rv] = grpname + return True + return rv + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): @@ -1121,23 +1157,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Handles setting categories and category suppression. Returns the new profile for the client (which might be unchanged). """ - groups.add(grpname) if grpname in self.groups: - group = self.groups[grpname] - category = group.category - if category: - if category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, grpname, category, - client, categories[category])) - return - categories[category] = grpname - if not profile and group.is_profile: + if not self._check_and_add_category(client, grpname, + categories): + return profile + groups.add(grpname) + if not profile and self.groups[grpname].is_profile: return grpname else: return profile + else: + groups.add(grpname) + return profile if client not in self.clients: pgroup = None @@ -1165,6 +1196,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[cgroup] = MetadataGroup(cgroup) profile = _add_group(cgroup) + # we do this before setting the default because there may be + # groups set in tags in groups.xml that we want to + # set groups, categories = self._merge_groups(client, groups, categories=categories) @@ -1248,16 +1282,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, for group in groups: if group in imd.groups: continue - if group in self.groups and self.groups[group].category: - category = self.groups[group].category - if self.groups[group].category in imd.categories: - self.logger.warning("%s: Group %s suppressed by category " - "%s; %s already a member of %s" % - (self.name, group, category, - imd.hostname, - imd.categories[category])) - continue - imd.categories[category] = group + if not self._check_and_add_category(imd.hostname, group, + imd.categories): + continue imd.groups.add(group) self._merge_groups(imd.hostname, imd.groups, -- cgit v1.2.3-1-g7c22 From 435d173f493bf8b1c5d71abef94ba9733d22839e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 15 Aug 2013 13:48:26 -0400 Subject: Metadata: search groups breadth-first when building metadata This makes category negation consistent, and also makes it work like it did in Bcfg2 1.2 --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 62 +++++++++++++--------- .../Testlib/TestServer/TestPlugins/TestMetadata.py | 17 +++--- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index ac31ecffe..650429f0d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -530,14 +530,17 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} - # mappings of predicate -> MetadataGroup object + # mappings of groupname -> [predicates] self.group_membership = dict() self.negated_groups = dict() + # list of group names in document order + self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions(core, datastore) else: self.versions = dict() + self.uuid = {} self.session_cache = {} self.default = None @@ -885,6 +888,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() + self.ordered_groups = [] # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is @@ -895,8 +899,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): - if ((el.tag != 'Group' and el.tag != 'Client') or - el.getchildren()): + if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren(): continue conditions = [] @@ -907,15 +910,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[self._aggregate_conditions(conditions)] = \ - self.groups[gname] + self.negated_groups.setdefault(gname, []) + self.negated_groups[gname].append( + self._aggregate_conditions(conditions)) else: if self.groups[gname].category: conditions.append(self._get_category_condition(gname)) - self.group_membership[ - self._aggregate_conditions(conditions)] = \ - self.groups[gname] + if gname not in self.ordered_groups: + self.ordered_groups.append(gname) + self.group_membership.setdefault(gname, []) + self.group_membership[gname].append( + self._aggregate_conditions(conditions)) self.states['groups.xml'] = True def expire_cache(self, key=None): @@ -1072,20 +1078,27 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() while numgroups != len(groups): numgroups = len(groups) - for predicate, group in self.group_membership.items(): - if group.name in groups: + newgroups = set() + removegroups = set() + for grpname in self.ordered_groups: + if grpname in groups: continue - if predicate(client, groups, categories): - groups.add(group.name) - if group.category: - categories[group.category] = group.name - for predicate, group in self.negated_groups.items(): - if group.name not in groups: + if any(p(client, groups, categories) + for p in self.group_membership[grpname]): + newgroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + categories[self.groups[grpname].category] = grpname + groups.update(newgroups) + for grpname, predicates in self.negated_groups.items(): + if grpname not in groups: continue - if predicate(client, groups, categories): - groups.remove(group.name) - if group.category: - del categories[group.category] + if any(p(client, groups, categories) for p in predicates): + removegroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + del categories[self.groups[grpname].category] + groups.difference_update(removegroups) return (groups, categories) def _check_category(self, client, grpname, categories): @@ -1189,6 +1202,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataConsistencyError( "Cannot add new client %s; no default group set" % client) + for cgroup in self.clientgroups.get(client, []): if cgroup in groups: continue @@ -1247,8 +1261,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) - all_groups.update([g.name for g in self.group_membership.values()]) - all_groups.update([g.name for g in self.negated_groups.values()]) + all_groups.update(self.group_membership.keys()) + all_groups.update(self.negated_groups.keys()) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups @@ -1287,9 +1301,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, continue imd.groups.add(group) - self._merge_groups(imd.hostname, imd.groups, - categories=imd.categories) - + self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index 32c893548..e839914d7 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -830,21 +830,18 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): self.assertEqual(metadata.groups['group4'].category, 'category1') self.assertEqual(metadata.default, "group1") - all_groups = [] - negated_groups = [] + all_groups = set() + negated_groups = set() for group in get_groups_test_tree().xpath("//Groups/Client//*") + \ get_groups_test_tree().xpath("//Groups/Group//*"): if group.tag == 'Group' and not group.getchildren(): if group.get("negate", "false").lower() == 'true': - negated_groups.append(group.get("name")) + negated_groups.add(group.get("name")) else: - all_groups.append(group.get("name")) - self.assertItemsEqual([g.name - for g in metadata.group_membership.values()], - all_groups) - self.assertItemsEqual([g.name - for g in metadata.negated_groups.values()], - negated_groups) + all_groups.add(group.get("name")) + self.assertItemsEqual(metadata.ordered_groups, all_groups) + self.assertItemsEqual(metadata.group_membership.keys(), all_groups) + self.assertItemsEqual(metadata.negated_groups.keys(), negated_groups) @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) def test_set_profile(self): -- cgit v1.2.3-1-g7c22 From 97b614b55e43801ef74f49c066788821419e3140 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 15 Aug 2013 14:27:39 -0400 Subject: Metadata: removed stray blank line --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 650429f0d..54440605e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1202,7 +1202,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataConsistencyError( "Cannot add new client %s; no default group set" % client) - for cgroup in self.clientgroups.get(client, []): if cgroup in groups: continue -- cgit v1.2.3-1-g7c22 From 0a04b47b38e185463fb200ee7a11cf44500029e5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 19 Aug 2013 10:24:54 -0400 Subject: doc: updated pros/cons for builtin yum libraries --- doc/server/plugins/generators/packages.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index cdc4f7282..31f3ccf22 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -490,17 +490,18 @@ Benefits to this include: * Much lower memory usage by the ``bcfg2-server`` process. * Much faster ``Packages.Refresh`` behavior. * More accurate dependency resolution. +* Better use of multiple processors/cores. Drawbacks include: -* More disk I/O. In some cases, you may have to raise the open file +* Resolution of package dependencies is slower and more + resource-intensive. At times it can be much slower, particularly + after running ``Packages.Refresh``. +* More disk I/O. This can be alleviated by putting + ``/var/lib/bcfg2/Packages/cache`` on tmpfs, but that offsets the + lower memory usage. In some cases, you may have to raise the open file limit for the user who runs your Bcfg2 server process, particularly if you have a lot of repositories. -* Resolution of package dependencies is slower in some cases, - particularly after running ``Packages.Refresh``. -* If you have a very large number of clients using a very small number - of repositories, using native yum libraries may actually increase - memory usage. Configuring the Yum Helper -------------------------- -- cgit v1.2.3-1-g7c22 From 1c54466ff7d6a2673453042164b8a14cf4fb577d Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 19 Aug 2013 19:30:16 -0400 Subject: Reporting: have bcfg2-report-collector be explicit about detaching Following the same logic as 360ba2e7, we should be explicit about the need to detach the bcfg2-report-collector process. Side note: this bit me because I was starting the bcfg2 server processes via SSH, and the way python-daemon checks for being started by inetd is to see if stdin is a socket. (??) --- src/lib/Bcfg2/Reporting/Collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 3d224432e..4556cda82 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -82,7 +82,7 @@ class ReportingCollector(object): """Startup the processing and go!""" self.terminate = threading.Event() atexit.register(self.shutdown) - self.context = daemon.DaemonContext() + self.context = daemon.DaemonContext(detach_process=True) if self.setup['daemon']: self.logger.debug("Daemonizing") -- cgit v1.2.3-1-g7c22 From 1f0fa73a04a68146cbe3527d5748b82bdb4e5b1e Mon Sep 17 00:00:00 2001 From: Jonathan Billings Date: Wed, 21 Aug 2013 14:36:23 -0400 Subject: SELinux: Update the matchpathcon function to use the file's mode If you don't supply a mode to the selinux.matchpathcon() function, it fails to properly look up the context in some circumstances related to context patterns in the SELinux policy. This change looks up the mode and supplies it to the function. (cherry picked from commit 20a2c9a8fb6c6ecbed259b5deccb01c01bf3304f) --- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index fb5d06e54..1e73d4f11 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -525,7 +525,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if entry.get("secontext") == "__default__": try: wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + selinux.matchpathcon( + path, ondisk[stat.ST_MODE])[1].split(":")[2] except OSError: errors.append("%s has no default SELinux context" % entry.get("name")) -- cgit v1.2.3-1-g7c22 From 76165d797e9dc0ef4741c0faf42c8586543d3558 Mon Sep 17 00:00:00 2001 From: Jason Kincl Date: Tue, 27 Aug 2013 15:15:57 -0400 Subject: Metadata: Fix client list calls use list_clients() which is DB-aware instead of just self.clients --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 54440605e..ece852ad7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -547,7 +547,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.pdirty = False self.password = core.setup['password'] self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients), + self.list_clients, self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -1274,7 +1274,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.clients): + for client in list(self.list_clients()): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1282,13 +1282,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.clients] + mdata = [self.core.build_metadata(client) for client in self.list_clients()] return [md.hostname for md in mdata if md.groups.issuperset(groups)] def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.clients] + mdata = [self.core.build_metadata(client) for client in self.list_clients()] return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] def merge_additional_groups(self, imd, groups): @@ -1462,7 +1462,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] - for client in list(self.clients): + for client in list(self.list_clients()): if not include_client(client): continue if client in self.clientgroups: -- cgit v1.2.3-1-g7c22 From ee3c5b0aa41e82b71b3df93f07dd92392e914264 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 28 Aug 2013 16:40:43 -0400 Subject: SSLServer: Poll for POST data correctly This updates the do_POST() code to cleave more closely to the code in SimpleXMLRPCHandler.do_POST (plus SSL magic, of course). In doing so, it eliminates a select() call whose timeouts were not handled properly, and which thus left stray idle processes around after incomplete XML-RPC calls. --- src/lib/Bcfg2/SSLServer.py | 21 ++++++++++----------- src/lib/Bcfg2/Server/BuiltinCore.py | 1 - 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 316c2f86c..b4fa4cf8d 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -5,7 +5,6 @@ better. """ import os import sys import socket -import select import signal import logging import ssl @@ -183,7 +182,6 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): Adds support for HTTP authentication. """ - logger = logging.getLogger("Bcfg2.SSLServer.XMLRPCRequestHandler") def authenticate(self): @@ -228,22 +226,24 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): return False return True - ### need to override do_POST here def do_POST(self): try: max_chunk_size = 10 * 1024 * 1024 size_remaining = int(self.headers["content-length"]) L = [] while size_remaining: - try: - select.select([self.rfile.fileno()], [], [], 3) - except select.error: - print("got select timeout") - raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size).decode('utf-8')) + chunk = self.rfile.read(chunk_size) + if not chunk: + break + L.append(chunk) size_remaining -= len(L[-1]) data = ''.join(L) + + data = self.decode_request_content(data) + if data is None: + return # response has been sent + response = self.server._marshaled_dispatch(self.client_address, data) if sys.hexversion >= 0x03000000: @@ -251,6 +251,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): except: # pylint: disable=W0702 try: self.send_response(500) + self.send_header("Content-length", "0") self.end_headers() except: (etype, msg) = sys.exc_info()[:2] @@ -431,8 +432,6 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, self.handle_request() except socket.timeout: pass - except select.error: - pass except: self.logger.error("Got unexpected error in handle_request", exc_info=1) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 2dd83289d..93da767c7 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -110,7 +110,6 @@ class Core(BaseCore): keyfile=self.setup['key'], certfile=self.setup['cert'], register=False, - timeout=1, ca=self.setup['ca'], protocol=self.setup['protocol']) except: # pylint: disable=W0702 -- cgit v1.2.3-1-g7c22 From 7c4692049d3c50de30127dae5555b7f228375c57 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 28 Aug 2013 16:51:16 -0400 Subject: POSIX: fixed unit tests for 1f0fa73 --- .../Testlib/TestClient/TestTools/TestPOSIX/Testbase.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index 8e7b58d30..d2f383f42 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -711,12 +711,13 @@ class TestPOSIXTool(TestTool): gather_data_rv[idx] = val ptool._gather_data.return_value = tuple(gather_data_rv) + stat_mode = 17407 mtime = 1344430414 + stat_rv = (stat_mode, Mock(), Mock(), Mock(), Mock(), Mock(), Mock(), + Mock(), mtime, Mock()) + gather_data_rv[0] = stat_rv entry = reset() entry.set("mtime", str(mtime)) - stat_rv = MagicMock() - stat_rv.__getitem__.return_value = mtime - gather_data_rv[0] = stat_rv ptool._gather_data.return_value = tuple(gather_data_rv) self.assertTrue(ptool._verify_metadata(entry)) ptool._gather_data.assert_called_with(entry.get("name")) @@ -788,7 +789,7 @@ class TestPOSIXTool(TestTool): ptool._gather_data.assert_called_with(entry.get("name")) ptool._verify_acls.assert_called_with(entry, path=entry.get("name")) - mock_matchpathcon.assert_called_with(entry.get("name"), 0) + mock_matchpathcon.assert_called_with(entry.get("name"), stat_mode) self.assertEqual(entry.get("current_exists", 'true'), 'true') for attr, idx, val in expected: self.assertEqual(entry.get(attr), val) @@ -803,7 +804,7 @@ class TestPOSIXTool(TestTool): ptool._gather_data.assert_called_with(entry.get("name")) ptool._verify_acls.assert_called_with(entry, path=entry.get("name")) - mock_matchpathcon.assert_called_with(entry.get("name"), 0) + mock_matchpathcon.assert_called_with(entry.get("name"), stat_mode) self.assertEqual(entry.get("current_exists", 'true'), 'true') for attr, idx, val in expected: self.assertEqual(entry.get(attr), val) -- cgit v1.2.3-1-g7c22 From 557446e489f3fdf6a66a6d2494cbda726117c7ac Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 29 Aug 2013 08:04:31 -0400 Subject: Metadata: get_client_names_by_* fixes Fixed long lines. Improved efficiency by avoiding multiple loops over the same data. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index ece852ad7..abda4376e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1274,7 +1274,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.list_clients()): + for client in self.list_clients(): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1282,14 +1282,22 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.list_clients()] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.groups.issuperset(groups): + rv.append(client) + return rv def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.list_clients()] - return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.bundles.issuperset(bundles): + rv.append(client) + return rv def merge_additional_groups(self, imd, groups): for group in groups: -- cgit v1.2.3-1-g7c22 From 47dccf125c42e1b888ef4b7800406d975be1b285 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 29 Aug 2013 08:06:02 -0400 Subject: SSLServer: Removed decode_request_content call decode_request_content() just adds support for gzipped content, which the Bcfg2 client doesn't support. Given that this feature was only added in Python 2.7, it's unlikely we'll be adding support for gzipped POST content any time soon. --- src/lib/Bcfg2/SSLServer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index b4fa4cf8d..2107f8abb 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -239,8 +239,6 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): L.append(chunk) size_remaining -= len(L[-1]) data = ''.join(L) - - data = self.decode_request_content(data) if data is None: return # response has been sent -- cgit v1.2.3-1-g7c22 From 01d7c89326cabe4a45ff99accba395eeed89f9c4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 29 Aug 2013 08:38:45 -0400 Subject: SSLServer: close connections properly --- src/lib/Bcfg2/SSLServer.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 2107f8abb..cee42e618 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -305,14 +305,11 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def finish(self): # shut down the connection - if not self.wfile.closed: - try: - self.wfile.flush() - self.wfile.close() - except socket.error: - err = sys.exc_info()[1] - self.logger.warning("Error closing connection: %s" % err) - self.rfile.close() + try: + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.finish(self) + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, -- cgit v1.2.3-1-g7c22 From 542ededa8297b1889542d97b9bdfb8345dd7550c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 29 Aug 2013 08:40:18 -0400 Subject: Rules: fixed HandlesEntry for XMLSrc replacement This makes HandlesEntry work with the new code that's in place for StructFile to replace XMLSrc. Fixes https://github.com/Bcfg2/bcfg2/commit/2169edc1bba82076db776b75db89b79d6f2f4786#commitcomment-3968162 --- src/lib/Bcfg2/Server/Plugins/Rules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 21862c5db..42d66e9d8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -13,9 +13,10 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): self._regex_cache = dict() def HandlesEntry(self, entry, metadata): - if entry.tag in self.Entries: - return self._matches(entry, metadata, - self.Entries[entry.tag].keys()) + for src in self.entries.values(): + for candidate in src.XMLMatch(metadata).xpath("//%s" % entry.tag): + if self._matches(entry, metadata, candidate): + return True return False def BindEntry(self, entry, metadata): -- cgit v1.2.3-1-g7c22 From 3c38f4fab6a0128f8a0da5155f8bcb382fbd5b8a Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 29 Aug 2013 09:01:25 -0400 Subject: Revert "Rules: fixed HandlesEntry for XMLSrc replacement" This reverts commit 542ededa8297b1889542d97b9bdfb8345dd7550c. Committed to wrong branch. --- src/lib/Bcfg2/Server/Plugins/Rules.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 42d66e9d8..21862c5db 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -13,10 +13,9 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): self._regex_cache = dict() def HandlesEntry(self, entry, metadata): - for src in self.entries.values(): - for candidate in src.XMLMatch(metadata).xpath("//%s" % entry.tag): - if self._matches(entry, metadata, candidate): - return True + if entry.tag in self.Entries: + return self._matches(entry, metadata, + self.Entries[entry.tag].keys()) return False def BindEntry(self, entry, metadata): -- cgit v1.2.3-1-g7c22 From ca6abe45ceeea410cd03dc0ecfd422ce07f352d7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 30 Aug 2013 09:39:21 -0500 Subject: doc: Remove bcfg2-admin query (moved to bcfg2-info) Signed-off-by: Sol Jerome --- doc/server/admin/query.txt | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 doc/server/admin/query.txt diff --git a/doc/server/admin/query.txt b/doc/server/admin/query.txt deleted file mode 100644 index 65851a43d..000000000 --- a/doc/server/admin/query.txt +++ /dev/null @@ -1,15 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-query: - -query -===== - -Query clients. - -The default result format is suitable for consumption by `pdsh`_. -This example queries the server for all clients in the *ubuntu* group:: - - bcfg2-admin query g=ubuntu - -.. _pdsh: http://sourceforge.net/projects/pdsh/ -- cgit v1.2.3-1-g7c22 From 8a36d9dccc6c4aed460763fb20083877c77a34cd Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sun, 1 Sep 2013 10:40:39 -0500 Subject: NagiosGen: Fix pylint failure Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/NagiosGen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index 466665382..011d367be 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -25,8 +25,8 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, core.fam, should_monitor=True, create=self.name) self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + {'/etc/nagiosgen.status': self.createhostconfig, + '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', -- cgit v1.2.3-1-g7c22