summaryrefslogtreecommitdiffstats
path: root/src/sbin/bcfg2-test
blob: 7c38a65d8107630cdb33baa63f0eb38a0bd13a4f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
#!/usr/bin/env python

"""This tool verifies that all clients known to the server build
without failures"""

import os
import sys
import signal
import fnmatch
import logging
import Bcfg2.Logger
import Bcfg2.Server.Core
from math import ceil
from nose.core import TestProgram
from nose.suite import LazySuite
from unittest import TestCase

try:
    from multiprocessing import Process, Queue, active_children
    HAS_MULTIPROC = True
except ImportError:
    HAS_MULTIPROC = False
    active_children = lambda: []  # pylint: disable=C0103


class CapturingLogger(object):
    """ Fake logger that captures logging output so that errors are
    only displayed for clients that fail tests """
    def __init__(self, *args, **kwargs):  # pylint: disable=W0613
        self.output = []

    def error(self, msg):
        """ discard error messages """
        self.output.append(msg)

    def warning(self, msg):
        """ discard error messages """
        self.output.append(msg)

    def info(self, msg):
        """ discard error messages """
        self.output.append(msg)

    def debug(self, msg):
        """ discard error messages """
        self.output.append(msg)

    def reset_output(self):
        """ Reset the captured output """
        self.output = []


class ClientTestFromQueue(TestCase):
    """ A test case that tests a value that has been enqueued by a
    child test process.  ``client`` is the name of the client that has
    been tested; ``result`` is the result from the :class:`ClientTest`
    test.  ``None`` indicates a successful test; a string value
    indicates a failed test; and an exception indicates an error while
    running the test. """
    __test__ = False  # Do not collect

    def __init__(self, client, result):
        TestCase.__init__(self)
        self.client = client
        self.result = result

    def shortDescription(self):
        return "Building configuration for %s" % self.client

    def runTest(self):
        """ parse the result from this test """
        if isinstance(self.result, Exception):
            raise self.result
        assert self.result is None, self.result


class ClientTest(TestCase):
    """ A test case representing the build of all of the configuration for
    a single host.  Checks that none of the build config entities has
    had a failure when it is building.  Optionally ignores some config
    files that we know will cause errors (because they are private
    files we don't have access to, for instance) """
    __test__ = False  # Do not collect
    divider = "-" * 70

    def __init__(self, core, client, ignore=None):
        TestCase.__init__(self)
        self.core = core
        self.core.logger = CapturingLogger()
        self.client = client
        if ignore is None:
            self.ignore = dict()
        else:
            self.ignore = ignore

    def ignore_entry(self, tag, name):
        """ return True if an error on a given entry should be ignored
        """
        if tag in self.ignore:
            if name in self.ignore[tag]:
                return True
            else:
                # try wildcard matching
                for pattern in self.ignore[tag]:
                    if fnmatch.fnmatch(name, pattern):
                        return True
        return False

    def shortDescription(self):
        return "Building configuration for %s" % self.client

    def runTest(self):
        """ run this individual test """
        config = self.core.BuildConfiguration(self.client)
        output = self.core.logger.output[:]
        if output:
            output.append(self.divider)
        self.core.logger.reset_output()

        # check for empty client configuration
        assert len(config.findall("Bundle")) > 0, \
            "\n".join(output + ["%s has no content" % self.client])

        # check for missing bundles
        metadata = self.core.build_metadata(self.client)
        sbundles = [el.get('name') for el in config.findall("Bundle")]
        missing = [b for b in metadata.bundles if b not in sbundles]
        assert len(missing) == 0, \
            "\n".join(output + ["Configuration is missing bundle(s): %s" %
                                ':'.join(missing)])

        # check for unknown packages
        unknown_pkgs = [el.get("name")
                        for el in config.xpath('//Package[@type="unknown"]')
                        if not self.ignore_entry(el.tag, el.get("name"))]
        assert len(unknown_pkgs) == 0, \
            "Configuration contains unknown packages: %s" % \
            ", ".join(unknown_pkgs)

        failures = []
        msg = output + ["Failures:"]
        for failure in config.xpath('//*[@failure]'):
            if not self.ignore_entry(failure.tag, failure.get('name')):
                failures.append(failure)
                msg.append("%s:%s: %s" % (failure.tag, failure.get("name"),
                                          failure.get("failure")))

        assert len(failures) == 0, "\n".join(msg)

    def __str__(self):
        return "ClientTest(%s)" % self.client

    id = __str__


def get_core(setup):
    """ Get a server core, with events handled """
    core = Bcfg2.Server.Core.BaseCore(setup)
    core.load_plugins()
    core.block_for_fam_events(handle_events=True)
    return core


def get_ignore(setup):
    """ Given an options dict, get a dict of entry tags and names to
    ignore errors from """
    ignore = dict()
    for entry in setup['test_ignore']:
        tag, name = entry.split(":")
        try:
            ignore[tag].append(name)
        except KeyError:
            ignore[tag] = [name]
    return ignore


def run_child(setup, clients, queue):
    """ Run tests for the given clients in a child process, returning
    results via the given Queue """
    core = get_core(setup)
    ignore = get_ignore(setup)
    for client in clients:
        try:
            ClientTest(core, client, ignore).runTest()
            queue.put((client, None))
        except AssertionError:
            queue.put((client, str(sys.exc_info()[1])))
        except:
            queue.put((client, sys.exc_info()[1]))

    core.shutdown()


def get_sigint_handler(core):
    """ Get a function that handles SIGINT/Ctrl-C by shutting down the
    core and exiting properly."""

    def hdlr(sig, frame):  # pylint: disable=W0613
        """ Handle SIGINT/Ctrl-C by shutting down the core and exiting
        properly. """
        core.shutdown()
        os._exit(1)  # pylint: disable=W0212

    return hdlr


def parse_args():
    """ Parse command line arguments. """
    optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS)

    optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS)
    optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS)
    setup = Bcfg2.Options.OptionParser(optinfo)
    setup.hm = \
        "bcfg2-test [options] [client] [client] [...]\nOptions:\n     %s" % \
        setup.buildHelpMessage()
    setup.parse(sys.argv[1:])

    if setup['debug']:
        level = logging.DEBUG
    elif setup['verbose']:
        level = logging.INFO
    else:
        level = logging.WARNING
    Bcfg2.Logger.setup_logging("bcfg2-test",
                               to_console=setup['verbose'] or setup['debug'],
                               to_syslog=False,
                               to_file=setup['logging'],
                               level=level)
    logger = logging.getLogger(sys.argv[0])
    if (setup['debug'] or setup['verbose']) and "-v" not in setup['noseopts']:
        setup['noseopts'].append("-v")

    if setup['children'] and not HAS_MULTIPROC:
        logger.warning("Python multiprocessing library not found, running "
                       "with no children")
        setup['children'] = 0

    if (setup['children'] and ('--with-xunit' in setup['noseopts'] or
                               '--xunit-file' in setup['noseopts'])):
        logger.warning("Use the --xunit option to bcfg2-test instead of the "
                       "--with-xunit or --xunit-file options to nosetest")
        xunitfile = None
        if '--with-xunit' in setup['noseopts']:
            setup['noseopts'].remove('--with-xunit')
            xunitfile = "nosetests.xml"
        if '--xunit-file' in setup['noseopts']:
            idx = setup['noseopts'].index('--xunit-file')
            try:
                setup['noseopts'].pop(idx)  # remove --xunit-file
                # remove the argument to it
                xunitfile = setup['noseopts'].pop(idx)
            except IndexError:
                pass
        if xunitfile and not setup['xunit']:
            setup['xunit'] = xunitfile
    return setup


def main():
    setup = parse_args()
    logger = logging.getLogger(sys.argv[0])
    core = get_core(setup)
    signal.signal(signal.SIGINT, get_sigint_handler(core))

    if setup['args']:
        clients = setup['args']
    else:
        clients = core.metadata.clients

    ignore = get_ignore(setup)

    if setup['children']:
        if setup['children'] > len(clients):
            logger.info("Refusing to spawn more children than clients to test,"
                        " setting children=%s" % len(clients))
            setup['children'] = len(clients)
        perchild = int(ceil(len(clients) / float(setup['children'] + 1)))
        queue = Queue()
        for child in range(setup['children']):
            start = child * perchild
            end = (child + 1) * perchild
            child = Process(target=run_child,
                            args=(setup, clients[start:end], queue))
            child.start()

        def generate_tests():
            """ Read test results for the clients """
            start = setup['children'] * perchild
            for client in clients[start:]:
                yield ClientTest(core, client, ignore)

            for i in range(start):  # pylint: disable=W0612
                yield ClientTestFromQueue(*queue.get())
    else:
        def generate_tests():
            """ Run tests for the clients """
            for client in clients:
                yield ClientTest(core, client, ignore)

    result = TestProgram(argv=sys.argv[:1] + core.setup['noseopts'],
                         suite=LazySuite(generate_tests), exit=False)

    # block until all children have completed -- should be
    # immediate since we've already gotten all the results we
    # expect
    for child in active_children():
        child.join()

    core.shutdown()
    if result.success:
        os._exit(0)  # pylint: disable=W0212
    else:
        os._exit(1)  # pylint: disable=W0212


if __name__ == "__main__":
    sys.exit(main())