diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/lib/Bcfg2/Bcfg2Py3Incompat.py | 2 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Bcfg2Py3k.py | 11 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Component.py | 4 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Options.py | 10 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 6 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugin.py | 37 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 21 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 14 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TGenshi.py | 10 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/settings.py | 8 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-info | 43 | ||||
| -rwxr-xr-x[-rw-r--r--] | src/sbin/bcfg2-test | 104 |
12 files changed, 169 insertions, 101 deletions
diff --git a/src/lib/Bcfg2/Bcfg2Py3Incompat.py b/src/lib/Bcfg2/Bcfg2Py3Incompat.py deleted file mode 100644 index 6b66e72b0..000000000 --- a/src/lib/Bcfg2/Bcfg2Py3Incompat.py +++ /dev/null @@ -1,2 +0,0 @@ -def fprint(s, f): - print(s, file=f) diff --git a/src/lib/Bcfg2/Bcfg2Py3k.py b/src/lib/Bcfg2/Bcfg2Py3k.py index ee05b7e41..6af8b3e5c 100644 --- a/src/lib/Bcfg2/Bcfg2Py3k.py +++ b/src/lib/Bcfg2/Bcfg2Py3k.py @@ -75,17 +75,6 @@ def u_str(string, encoding=None): else: return unicode(string) -""" -In order to use the new syntax for printing to a file, we need to do -a conditional import because there is a syntax incompatibility between -the two versions of python. -""" -if sys.hexversion >= 0x03000000: - from Bcfg2.Bcfg2Py3Incompat import fprint -else: - def fprint(s, f): - print >> f, s - if sys.hexversion >= 0x03000000: from io import FileIO as file else: diff --git a/src/lib/Bcfg2/Component.py b/src/lib/Bcfg2/Component.py index e21b5d359..eb9ea166a 100644 --- a/src/lib/Bcfg2/Component.py +++ b/src/lib/Bcfg2/Component.py @@ -14,7 +14,7 @@ import Bcfg2.Logger from Bcfg2.Statistics import Statistics from Bcfg2.SSLServer import XMLRPCServer # Compatibility import -from Bcfg2.Bcfg2Py3k import xmlrpclib, urlparse, fprint +from Bcfg2.Bcfg2Py3k import xmlrpclib, urlparse logger = logging.getLogger() @@ -55,7 +55,7 @@ def run_component(component_cls, listen_all, location, daemon, pidfile_name, os.chdir(os.sep) pidfile = open(pidfile_name or "/dev/null", "w") - fprint(os.getpid(), pidfile) + pidfile.write("%s\n" % os.getpid()) pidfile.close() component = component_cls(cfile=cfile, **cls_kwargs) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 6a6b83d61..f6273924a 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -3,6 +3,7 @@ import getopt import os import sys +import shlex import Bcfg2.Client.Tools # Compatibility imports from Bcfg2.Bcfg2Py3k import ConfigParser @@ -353,6 +354,15 @@ CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90, cmd='-t', cf=('communication', 'timeout'), odesc='<timeout>') +# bcfg2-test options +TEST_NOSEOPTS = Option('Options to pass to nosetests', default=[], + cmd='--nose-options', cf=('bcfg2_test', 'nose_options'), + odesc='<opts>', long_arg=True, cook=shlex.split) +TEST_IGNORE = Option('Ignore these entries if they fail to build.', default=[], + cmd='--ignore', + cf=('bcfg2_test', 'ignore_entries'), long_arg=True, + odesc='<Type>:<name>,<Type>:<name>', cook=list_split) + # APT client tool options CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path', cf=('APT', 'install_path'), diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 3647ee622..112c484b1 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -6,6 +6,8 @@ import select import sys import threading import time +from traceback import format_exc + try: import lxml.etree except ImportError: @@ -239,12 +241,12 @@ class Core(Component): self.Bind(entry, metadata) except PluginExecutionError, exc: if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) + entry.set('failure', 'bind error: %s' % format_exc()) logger.error("Failed to bind entry: %s %s" % \ (entry.tag, entry.get('name'))) except Exception, exc: if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) + entry.set('failure', 'bind error: %s' % format_exc()) logger.error("Unexpected failure in BindStructure: %s %s" \ % (entry.tag, entry.get('name')), exc_info=1) diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index 3cdfdbb41..11e6c5c20 100644 --- a/src/lib/Bcfg2/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -592,6 +592,29 @@ class SingleXMLFileBacked(XMLFileBacked): self.fam = fam self.fam.AddMonitor(filename, self) + def _follow_xincludes(self, fname=None, xdata=None): + ''' follow xincludes, adding included files to fam and to + self.extras ''' + if xdata is None: + if fname is None: + xdata = self.xdata.getroottree() + else: + xdata = lxml.etree.parse(fname) + included = [ent.get('href') + for ent in xdata.findall('//{http://www.w3.org/2001/XInclude}include')] + for name in included: + if name not in self.extras: + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(self.name), name) + self.add_monitor(fpath, name) + self._follow_xincludes(fname=fpath) + + def add_monitor(self, fpath, fname): + self.fam.AddMonitor(fpath, self) + self.extras.append(fname) + def Index(self): """Build local data structures.""" try: @@ -601,22 +624,14 @@ class SingleXMLFileBacked(XMLFileBacked): logger.error("Failed to parse %s: %s" % (self.name, err)) raise Bcfg2.Server.Plugin.PluginInitError - included = [ent.get('href') - for ent in self.xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.fam.AddMonitor(os.path.join(os.path.dirname(self.name), - name), - self) - self.extras.append(name) + self._follow_xincludes() + if self.extras: try: self.xdata.getroottree().xinclude() except lxml.etree.XIncludeError: err = sys.exc_info()[1] logger.error("XInclude failed on %s: %s" % (self.name, err)) - self.entries = self.xdata.getchildren() if self.__identifier__ is not None: self.label = self.xdata.attrib[self.__identifier__] @@ -844,7 +859,7 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): self._matches(entry, metadata, src.cache[1][entry.tag]))] if len(matching) == 0: - raise PluginExecutionError + raise PluginExecutionError('No matching source for entry when retrieving attributes for %s(%s)' % (entry.tag, entry.attrib.get('name'))) elif len(matching) == 1: index = 0 else: diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index f39993496..5ba4de12f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -36,13 +36,16 @@ class MetadataRuntimeError(Exception): pass -class XMLMetadataConfig(object): +class XMLMetadataConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked): """Handles xml config files and all XInclude statements""" def __init__(self, metadata, watch_clients, basefile): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, + os.path.join(metadata.data, + basefile), + metadata.core.fam) self.metadata = metadata self.basefile = basefile self.should_monitor = watch_clients - self.extras = [] self.data = None self.basedata = None self.basedir = metadata.data @@ -62,11 +65,10 @@ class XMLMetadataConfig(object): raise MetadataRuntimeError return self.basedata - def add_monitor(self, fname): + def add_monitor(self, fpath, fname): """Add a fam monitor for an included file""" if self.should_monitor: - self.metadata.core.fam.AddMonitor(os.path.join(self.basedir, fname), - self.metadata) + self.metadata.core.fam.AddMonitor(fpath, self.metadata) self.extras.append(fname) def load_xml(self): @@ -76,13 +78,10 @@ class XMLMetadataConfig(object): except lxml.etree.XMLSyntaxError: self.logger.error('Failed to parse %s' % self.basefile) return + self.extras = [] self.basedata = copy.copy(xdata) - included = [ent.get('href') for ent in \ - xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.add_monitor(name) + self._follow_xincludes(xdata=xdata) + if self.extras: try: xdata.xinclude() except lxml.etree.XIncludeError: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 3511cfc3d..8d0067b6a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -44,8 +44,18 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, def HandleEvent(self, event=None): Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event) - if event.filename != self.name: - self.parsed.add(os.path.basename(event.filename)) + if event and event.filename != self.name: + for fname in self.extras: + fpath = None + if fname.startswith("/"): + fpath = os.path.abspath(fname) + else: + fpath = \ + os.path.abspath(os.path.join(os.path.dirname(self.name), + fname)) + if fpath == os.path.abspath(event.filename): + self.parsed.add(fname) + break if sorted(list(self.parsed)) == sorted(self.extras): self.logger.info("Reloading Packages plugin") diff --git a/src/lib/Bcfg2/Server/Plugins/TGenshi.py b/src/lib/Bcfg2/Server/Plugins/TGenshi.py index 3ba0f4272..c4dd40614 100644 --- a/src/lib/Bcfg2/Server/Plugins/TGenshi.py +++ b/src/lib/Bcfg2/Server/Plugins/TGenshi.py @@ -114,13 +114,13 @@ class TemplateFile: if entry.text == '': entry.set('empty', 'true') except TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - raise Bcfg2.Server.Plugin.PluginExecutionError + err = sys.exc_info()[1] + logger.exception('Genshi template error') + raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template error: %s' % err) except AttributeError: err = sys.exc_info()[1] - logger.error('Genshi template loading error: %s' % err) - raise Bcfg2.Server.Plugin.PluginExecutionError + logger.exception('Genshi template loading error') + raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template loading error: %s' % err) class TGenshi(Bcfg2.Server.Plugin.GroupSpool): diff --git a/src/lib/Bcfg2/Server/Reports/settings.py b/src/lib/Bcfg2/Server/Reports/settings.py index c8ceb5d88..4d567f1a2 100644 --- a/src/lib/Bcfg2/Server/Reports/settings.py +++ b/src/lib/Bcfg2/Server/Reports/settings.py @@ -64,10 +64,10 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 2: # Local time zone for this installation. All choices can be found here: # http://docs.djangoproject.com/en/dev/ref/settings/#time-zone -try: - TIME_ZONE = c.get('statistics', 'time_zone') -except: - if django.VERSION[0] == 1 and django.VERSION[1] > 2: +if django.VERSION[0] == 1 and django.VERSION[1] > 2: + try: + TIME_ZONE = c.get('statistics', 'time_zone') + except: TIME_ZONE = None # Language code for this installation. All choices can be found here: diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 1e7ec4d49..580de5248 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -1,5 +1,4 @@ #!/usr/bin/env python - """This tool loads the Bcfg2 core into an interactive debugger.""" from code import InteractiveConsole @@ -521,7 +520,7 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): if len(source.whitelist): print(" Whitelist: %s" % ", ".join(source.whitelist)) print("") - + def do_profile(self, arg): """.""" if not have_profile: @@ -545,27 +544,25 @@ if __name__ == '__main__': optinfo = { 'configfile': Bcfg2.Options.CFILE, 'help': Bcfg2.Options.HELP, - } - optinfo.update({ - 'event debug': Bcfg2.Options.DEBUG, - 'profile': Bcfg2.Options.CORE_PROFILE, - 'encoding': Bcfg2.Options.ENCODING, - # Server options - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'mconnect': Bcfg2.Options.SERVER_MCONNECT, - 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, - 'location': Bcfg2.Options.SERVER_LOCATION, - 'static': Bcfg2.Options.SERVER_STATIC, - 'key': Bcfg2.Options.SERVER_KEY, - 'cert': Bcfg2.Options.SERVER_CERT, - 'ca': Bcfg2.Options.SERVER_CA, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'protocol': Bcfg2.Options.SERVER_PROTOCOL, - # More options - 'logging': Bcfg2.Options.LOGGING_FILE_PATH - }) + 'event debug': Bcfg2.Options.DEBUG, + 'profile': Bcfg2.Options.CORE_PROFILE, + 'encoding': Bcfg2.Options.ENCODING, + # Server options + 'repo': Bcfg2.Options.SERVER_REPOSITORY, + 'plugins': Bcfg2.Options.SERVER_PLUGINS, + 'password': Bcfg2.Options.SERVER_PASSWORD, + 'mconnect': Bcfg2.Options.SERVER_MCONNECT, + 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, + 'location': Bcfg2.Options.SERVER_LOCATION, + 'static': Bcfg2.Options.SERVER_STATIC, + 'key': Bcfg2.Options.SERVER_KEY, + 'cert': Bcfg2.Options.SERVER_CERT, + 'ca': Bcfg2.Options.SERVER_CA, + 'password': Bcfg2.Options.SERVER_PASSWORD, + 'protocol': Bcfg2.Options.SERVER_PROTOCOL, + # More options + 'logging': Bcfg2.Options.LOGGING_FILE_PATH + } setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "Usage:\n %s\n%s" % (setup.buildHelpMessage(), USAGE) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index af2225245..01a2a4893 100644..100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -1,41 +1,60 @@ #!/usr/bin/env python -"""This tool verifies that all clients known to the server build without failures""" +"""This tool verifies that all clients known to the server build +without failures""" import sys +import fnmatch +import logging +import Bcfg2.Logger import Bcfg2.Server.Core from nose.core import TestProgram +from nose.suite import LazySuite from unittest import TestCase class ClientTest(TestCase): """ - A test case representing the build of all of the configuration for a single host. - Checks that none of the build config entities has had a failure when it is building. - Optionally ignores some config files that we know will cause errors (because they - are private files we don't have access to, for instance) + A test case representing the build of all of the configuration for + a single host. Checks that none of the build config entities has + had a failure when it is building. Optionally ignores some config + files that we know will cause errors (because they are private + files we don't have access to, for instance) """ __test__ = False # Do not collect - def __init__(self, bcfg2_core, client): + def __init__(self, bcfg2_core, client, ignore=None): TestCase.__init__(self) self.bcfg2_core = bcfg2_core self.client = client + if ignore is None: + self.ignore = dict() + else: + self.ignore = ignore + + def ignore_entry(self, tag, name): + if tag in self.ignore: + if name in self.ignore[tag]: + return True + else: + # try wildcard matching + for pattern in self.ignore[tag]: + if fnmatch.fnmatch(name, pattern): + return True + return False def runTest(self): config = self.bcfg2_core.BuildConfiguration(self.client) - failures = config.xpath('//*[@failure]') - def format_failure(failure): - return "%s(%s): %s" % ( - failure.tag, - failure.attrib.get('name'), - failure.attrib.get('failure') - ) + failures = [] + msg = ["Failures:"] + for failure in config.xpath('//*[@failure]'): + if not self.ignore_entry(failure.tag, failure.get('name')): + failures.append(failure) + msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), + failure.get("failure"))) + + assert len(failures) == 0, "\n".join(msg) - assert len(failures) == 0, "Failures:\n%s" % "\n".join( - [format_failure(failure) for failure in failures] - ) - def __str__(self): return "ClientTest(%s)" % self.client @@ -43,26 +62,55 @@ class ClientTest(TestCase): def main(): optinfo = { - 'configfile': Bcfg2.Options.CFILE, - 'help': Bcfg2.Options.HELP, - 'encoding': Bcfg2.Options.ENCODING, - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'password': Bcfg2.Options.SERVER_PASSWORD, - } + 'configfile': Bcfg2.Options.CFILE, + 'help': Bcfg2.Options.HELP, + 'encoding': Bcfg2.Options.ENCODING, + 'repo': Bcfg2.Options.SERVER_REPOSITORY, + 'plugins': Bcfg2.Options.SERVER_PLUGINS, + 'password': Bcfg2.Options.SERVER_PASSWORD, + 'verbose': Bcfg2.Options.VERBOSE, + 'noseopts': Bcfg2.Options.TEST_NOSEOPTS, + 'ignore': Bcfg2.Options.TEST_IGNORE, + } setup = Bcfg2.Options.OptionParser(optinfo) + setup.hm = \ + "bcfg2-test [options] [client] [client] [...]\nOptions:\n %s" % \ + setup.buildHelpMessage() setup.parse(sys.argv[1:]) + + if setup['verbose']: + Bcfg2.Logger.setup_logging("bcfg2-test", to_syslog=False) + core = Bcfg2.Server.Core.Core( setup['repo'], setup['plugins'], setup['password'], setup['encoding'], filemonitor='pseudo' - ) - core.fam.handle_events_in_interval(0.1) - suite = [ClientTest(core, client) for client in core.metadata.clients] + ) + + ignore = dict() + for entry in setup['ignore']: + tag, name = entry.split(":") + try: + ignore[tag].append(name) + except KeyError: + ignore[tag] = [name] + + def run_tests(): + core.fam.handle_events_in_interval(0.1) + + if setup['args']: + clients = setup['args'] + else: + clients = core.metadata.clients + + for client in clients: + logging.info("Building %s" % client) + yield ClientTest(core, client, ignore) - TestProgram(argv=sys.argv[0:1], suite = suite) + TestProgram(argv=sys.argv[0:1] + setup['noseopts'], + suite=LazySuite(run_tests)) if __name__ == "__main__": sys.exit(main()) |