diff options
Diffstat (limited to 'src/sbin')
| -rwxr-xr-x | src/sbin/bcfg2-admin | 1 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-crypt | 2 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-info | 48 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-reports | 12 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-server | 1 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-test | 231 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-yum-helper | 4 |
7 files changed, 229 insertions, 70 deletions
diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 3c63cd3f5..3bce7fdab 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -10,6 +10,7 @@ import Bcfg2.Options import Bcfg2.Server.Admin from Bcfg2.Compat import StringIO + def mode_import(modename): """Load Bcfg2.Server.Admin.<mode>.""" modname = modename.capitalize() diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 810406567..f7deba90c 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -258,7 +258,7 @@ class Encryptor(object): (self.pname, pname)) return (passphrase, pname) - def _get_passphrase(self, chunk): # pylint: disable=W0613 + def _get_passphrase(self, chunk): # pylint: disable=W0613 """ get the passphrase for a chunk of a file """ return None diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 287d0a161..ad35bbeeb 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -473,7 +473,7 @@ Bcfg2 client itself.""") ('Password', self.setup['password']), ('Server Metadata Connector', self.setup['mconnect']), ('Filemonitor', self.setup['filemonitor']), - ('Server address', self.setup['location']), + ('Server address', self.setup['location']), ('Path to key', self.setup['key']), ('Path to SSL certificate', self.setup['cert']), ('Path to SSL CA certificate', self.setup['ca']), @@ -640,21 +640,24 @@ Bcfg2 client itself.""") if 'Packages' not in self.plugins: print("Packages plugin not enabled") return + self.plugins['Packages'].toggle_debug() + + indep = lxml.etree.Element("Independent") + structures = [lxml.etree.Element("Bundle", name="packages")] + for arg in arglist[1:]: + lxml.etree.SubElement(structures[0], "Package", name=arg) + hostname = arglist[0] - initial = arglist[1:] metadata = self.build_metadata(hostname) - self.plugins['Packages'].toggle_debug() - collection = self.plugins['Packages'].get_collection(metadata) - packages, unknown = collection.complete(initial) - newpkgs = list(packages.difference(initial)) - print("%d initial packages" % len(initial)) - print(" %s" % "\n ".join(initial)) - print("%d new packages added" % len(newpkgs)) - if newpkgs: - print(" %s" % "\n ".join(newpkgs)) - print("%d unknown packages" % len(unknown)) - if unknown: - print(" %s" % "\n ".join(unknown)) + + # pylint: disable=W0212 + self.plugins['Packages']._build_packages(metadata, indep, structures) + # pylint: enable=W0212 + + print("%d new packages added" % len(indep.getchildren())) + if len(indep.getchildren()): + print(" %s" % "\n ".join(lxml.etree.tostring(p) + for p in indep.getchildren())) def do_packagesources(self, args): """ packagesources <hostname> - Show package sources """ @@ -726,17 +729,19 @@ Bcfg2 client itself.""") pass - def build_usage(): """ build usage message """ cmd_blacklist = ["do_loop", "do_EOF"] usage = dict() for attrname in dir(InfoCore): attr = getattr(InfoCore, attrname) - if (hasattr(attr, "__func__") and - attr.__func__.func_name not in cmd_blacklist and - attr.__func__.func_name.startswith("do_") and - attr.__func__.func_doc): + + # shim for python 2.4, __func__ is im_func + funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None)) + if (funcattr != None and + funcattr.func_name not in cmd_blacklist and + funcattr.func_name.startswith("do_") and + funcattr.func_doc): usage[attr.__name__] = re.sub(r'\s+', ' ', attr.__doc__) return "Commands:\n" + "\n".join(usage[k] for k in sorted(usage.keys())) @@ -748,9 +753,8 @@ def main(): optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, interactive=Bcfg2.Options.INTERACTIVE, interpreter=Bcfg2.Options.INTERPRETER) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.load_option_parser(optinfo) + optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) + setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", "Options:", setup.buildHelpMessage(), diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 9f2ff96c2..2c4a918be 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -10,7 +10,7 @@ from Bcfg2.Compat import ConfigParser try: import Bcfg2.settings except ConfigParser.NoSectionError: - print("Your bcfg2.conf is currently missing the statistics section which " + print("Your bcfg2.conf is currently missing the [database] section which " "is necessary for the reporting interface. Please see bcfg2.conf(5) " "for more details.") sys.exit(1) @@ -121,7 +121,7 @@ def main(): help="Show hosts that haven't run in the last 24 " "hours") parser.add_option_group(allhostmodes) - + # entry modes entrymodes = \ OptionGroup(parser, "Entry Modes", @@ -166,7 +166,7 @@ def main(): (mode.get_opt_string(), opt.get_opt_string())) mode = opt mode_family = parser.get_option_group(opt.get_opt_string()) - + # you can specify more than one of --bad, --extra, --modified, --show, so # consider single-host options separately if not mode_family: @@ -174,7 +174,7 @@ def main(): if getattr(options, opt.dest): mode_family = parser.get_option_group(opt.get_opt_string()) break - + if not mode_family: parser.error("You must specify a mode") @@ -243,7 +243,7 @@ def main(): parser.error("%s require either a list of entries on the " "command line or the --file options" % mode_family.title) - + if options.badentry: result = hosts_by_entry_type(clients, "bad", entries) elif options.modifiedentry: @@ -263,7 +263,7 @@ def main(): # todo batch fetch this. sqlite could break for client in clients: - ents = entry_cls.objects.filter(name=entries[0][1], + ents = entry_cls.objects.filter(name=entries[0][1], interaction=client.current_interaction) if len(ents) == 0: continue diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index 8e96d65f2..33ee327fc 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -11,6 +11,7 @@ from Bcfg2.Server.Core import CoreInitError LOGGER = logging.getLogger('bcfg2-server') + def main(): optinfo = dict() optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 75079ec3f..510bb898b 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -5,29 +5,87 @@ without failures""" import os import sys -import signal import fnmatch import logging import Bcfg2.Logger import Bcfg2.Server.Core +from math import ceil from nose.core import TestProgram from nose.suite import LazySuite from unittest import TestCase +try: + from multiprocessing import Process, Queue, active_children + HAS_MULTIPROC = True +except ImportError: + HAS_MULTIPROC = False + active_children = lambda: [] # pylint: disable=C0103 + + +class CapturingLogger(object): + """ Fake logger that captures logging output so that errors are + only displayed for clients that fail tests """ + def __init__(self, *args, **kwargs): # pylint: disable=W0613 + self.output = [] + + def error(self, msg): + """ discard error messages """ + self.output.append(msg) + + def warning(self, msg): + """ discard error messages """ + self.output.append(msg) + + def info(self, msg): + """ discard error messages """ + self.output.append(msg) + + def debug(self, msg): + """ discard error messages """ + self.output.append(msg) + + def reset_output(self): + """ Reset the captured output """ + self.output = [] + + +class ClientTestFromQueue(TestCase): + """ A test case that tests a value that has been enqueued by a + child test process. ``client`` is the name of the client that has + been tested; ``result`` is the result from the :class:`ClientTest` + test. ``None`` indicates a successful test; a string value + indicates a failed test; and an exception indicates an error while + running the test. """ + __test__ = False # Do not collect + + def __init__(self, client, result): + TestCase.__init__(self) + self.client = client + self.result = result + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ parse the result from this test """ + if isinstance(self.result, Exception): + raise self.result + assert self.result is None, self.result + class ClientTest(TestCase): - """ - A test case representing the build of all of the configuration for + """ A test case representing the build of all of the configuration for a single host. Checks that none of the build config entities has had a failure when it is building. Optionally ignores some config files that we know will cause errors (because they are private - files we don't have access to, for instance) - """ + files we don't have access to, for instance) """ __test__ = False # Do not collect + divider = "-" * 70 - def __init__(self, bcfg2_core, client, ignore=None): + def __init__(self, core, client, ignore=None): TestCase.__init__(self) - self.bcfg2_core = bcfg2_core + self.core = core + self.core.logger = CapturingLogger() self.client = client if ignore is None: self.ignore = dict() @@ -52,13 +110,34 @@ class ClientTest(TestCase): def runTest(self): """ run this individual test """ - config = self.bcfg2_core.BuildConfiguration(self.client) + config = self.core.BuildConfiguration(self.client) + output = self.core.logger.output[:] + if output: + output.append(self.divider) + self.core.logger.reset_output() + # check for empty client configuration assert len(config.findall("Bundle")) > 0, \ - "%s has no content" % self.client + "\n".join(output + ["%s has no content" % self.client]) + + # check for missing bundles + metadata = self.core.build_metadata(self.client) + sbundles = [el.get('name') for el in config.findall("Bundle")] + missing = [b for b in metadata.bundles if b not in sbundles] + assert len(missing) == 0, \ + "\n".join(output + ["Configuration is missing bundle(s): %s" % + ':'.join(missing)]) + + # check for unknown packages + unknown_pkgs = [el.get("name") + for el in config.xpath('//Package[@type="unknown"]') + if not self.ignore_entry(el.tag, el.get("name"))] + assert len(unknown_pkgs) == 0, \ + "Configuration contains unknown packages: %s" % \ + ", ".join(unknown_pkgs) failures = [] - msg = ["Failures:"] + msg = output + ["Failures:"] for failure in config.xpath('//*[@failure]'): if not self.ignore_entry(failure.tag, failure.get('name')): failures.append(failure) @@ -73,23 +152,46 @@ class ClientTest(TestCase): id = __str__ -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" +def get_core(setup): + """ Get a server core, with events handled """ + core = Bcfg2.Server.Core.BaseCore(setup) + core.fam.handle_events_in_interval(0.1) + return core - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - return hdlr +def get_ignore(setup): + """ Given an options dict, get a dict of entry tags and names to + ignore errors from """ + ignore = dict() + for entry in setup['test_ignore']: + tag, name = entry.split(":") + try: + ignore[tag].append(name) + except KeyError: + ignore[tag] = [name] + return ignore -def main(): - optinfo = dict(noseopts=Bcfg2.Options.TEST_NOSEOPTS, - test_ignore=Bcfg2.Options.TEST_IGNORE, - validate=Bcfg2.Options.CFG_VALIDATION) +def run_child(setup, clients, queue): + """ Run tests for the given clients in a child process, returning + results via the given Queue """ + core = get_core(setup) + ignore = get_ignore(setup) + for client in clients: + try: + ClientTest(core, client, ignore).runTest() + queue.put((client, None)) + except AssertionError: + queue.put((client, str(sys.exc_info()[1]))) + except: + queue.put((client, sys.exc_info()[1])) + + core.shutdown() + + +def parse_args(): + """ Parse command line arguments. """ + optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.load_option_parser(optinfo) @@ -109,37 +211,88 @@ def main(): to_syslog=False, to_file=setup['logging'], level=level) + logger = logging.getLogger(sys.argv[0]) if (setup['debug'] or setup['verbose']) and "-v" not in setup['noseopts']: setup['noseopts'].append("-v") - core = Bcfg2.Server.Core.BaseCore() - signal.signal(signal.SIGINT, get_sigint_handler(core)) + if setup['children'] and not HAS_MULTIPROC: + logger.warning("Python multiprocessing library not found, running " + "with no children") + setup['children'] = 0 - ignore = dict() - for entry in setup['test_ignore']: - tag, name = entry.split(":") - try: - ignore[tag].append(name) - except KeyError: - ignore[tag] = [name] + if (setup['children'] and ('--with-xunit' in setup['noseopts'] or + '--xunit-file' in setup['noseopts'])): + logger.warning("Use the --xunit option to bcfg2-test instead of the " + "--with-xunit or --xunit-file options to nosetest") + xunitfile = None + if '--with-xunit' in setup['noseopts']: + setup['noseopts'].remove('--with-xunit') + xunitfile = "nosetests.xml" + if '--xunit-file' in setup['noseopts']: + idx = setup['noseopts'].index('--xunit-file') + try: + setup['noseopts'].pop(idx) # remove --xunit-file + # remove the argument to it + xunitfile = setup['noseopts'].pop(idx) + except IndexError: + pass + if xunitfile and not setup['xunit']: + setup['xunit'] = xunitfile + return setup - core.fam.handle_events_in_interval(0.1) + +def main(): + setup = parse_args() + logger = logging.getLogger(sys.argv[0]) + core = get_core(setup) if setup['args']: clients = setup['args'] else: clients = core.metadata.clients - def run_tests(): - """ Run the test suite """ - for client in clients: - yield ClientTest(core, client, ignore) + ignore = get_ignore(setup) - TestProgram(argv=sys.argv[0:1] + setup['noseopts'], - suite=LazySuite(run_tests)) + if setup['children']: + if setup['children'] > len(clients): + logger.info("Refusing to spawn more children than clients to test," + " setting children=%s" % len(clients)) + setup['children'] = len(clients) + perchild = int(ceil(len(clients) / float(setup['children'] + 1))) + queue = Queue() + for child in range(setup['children']): + start = child * perchild + end = (child + 1) * perchild + child = Process(target=run_child, + args=(setup, clients[start:end], queue)) + child.start() + + def generate_tests(): + """ Read test results for the clients """ + start = setup['children'] * perchild + for client in clients[start:]: + yield ClientTest(core, client, ignore) + + for i in range(start): # pylint: disable=W0612 + yield ClientTestFromQueue(*queue.get()) + else: + def generate_tests(): + """ Run tests for the clients """ + for client in clients: + yield ClientTest(core, client, ignore) + + TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], + suite=LazySuite(generate_tests), exit=False) + + # block until all children have completed -- should be + # immediate since we've already gotten all the results we + # expect + for child in active_children(): + child.join() core.shutdown() os._exit(0) # pylint: disable=W0212 + if __name__ == "__main__": sys.exit(main()) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index ba6f30406..7e5c03fd5 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -129,7 +129,7 @@ class DepSolver(object): err = sys.exc_info()[1] self.logger.warning(err) return [] - + if ptype == "default": return [p for p, d in list(group.default_packages.items()) @@ -254,6 +254,6 @@ def main(): rv[gdata['group']] = list(packages) print(json.dumps(rv)) - + if __name__ == '__main__': sys.exit(main()) |