diff options
| -rw-r--r-- | src/lib/Bcfg2/Server/Info.py | 861 | ||||
| -rwxr-xr-x | src/sbin/bcfg2-info | 790 | ||||
| -rwxr-xr-x | tools/bcfg2-profile-templates.py | 138 |
3 files changed, 863 insertions, 926 deletions
diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py new file mode 100644 index 000000000..76da861ba --- /dev/null +++ b/src/lib/Bcfg2/Server/Info.py @@ -0,0 +1,861 @@ +""" Subcommands and helpers for bcfg2-info """ +# -*- coding: utf-8 -*- + +import os +import sys +import cmd +import math +import time +import copy +import pipes +import fnmatch +import argparse +import operator +import lxml.etree +from code import InteractiveConsole +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Server.Plugin +import Bcfg2.Client.Tools.POSIX +from Bcfg2.Compat import any # pylint: disable=W0622 + +try: + try: + import cProfile as profile + except ImportError: + import profile + import pstats + HAS_PROFILE = True +except ImportError: + HAS_PROFILE = False + + +def print_tabular(rows): + """Print data in tabular format.""" + cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 + for index in range(len(rows[0]))]) + fstring = (" %%-%ss |" * len(cmax)) % cmax + fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax + print(fstring % rows[0]) + print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') + for row in rows[1:]: + print(fstring % row) + + +def display_trace(trace): + """ display statistics from a profile trace """ + stats = pstats.Stats(trace) + stats.sort_stats('cumulative', 'calls', 'time') + stats.print_stats(200) + + +def load_interpreters(): + """ Load a dict of available Python interpreters """ + interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) + default = "python" + try: + import bpython.cli + interpreters["bpython"] = lambda v: bpython.cli.main(args=[], + locals_=v) + default = "bpython" + except ImportError: + pass + + try: + # whether ipython is actually better than bpython is + # up for debate, but this is the behavior that existed + # before --interpreter was added, so we call IPython + # better + import IPython + # pylint: disable=E1101 + if hasattr(IPython, "Shell"): + interpreters["ipython"] = lambda v: \ + IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() + default = "ipython" + elif hasattr(IPython, "embed"): + interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) + default = "ipython" + else: + print("Unknown IPython API version") + # pylint: enable=E1101 + except ImportError: + pass + + return (interpreters, default) + + +class InfoCmd(Bcfg2.Options.Subcommand): + """ Base class for bcfg2-info subcommands """ + + def _expand_globs(self, globs, candidates): + # special cases to speed things up: + if globs is None or '*' in globs: + return candidates + has_wildcards = False + for glob in globs: + # check if any wildcard characters are in the string + if set('*?[]') & set(glob): + has_wildcards = True + break + if not has_wildcards: + return globs + + rv = set() + cset = set(candidates) + for glob in globs: + rv.update(c for c in cset if fnmatch.fnmatch(c, glob)) + cset.difference_update(rv) + return list(rv) + + def get_client_list(self, globs): + """ given a list of host globs, get a list of clients that + match them """ + return self._expand_globs(globs, self.core.metadata.clients) + + def get_group_list(self, globs): + """ given a list of group glob, get a list of groups that + match them""" + # special cases to speed things up: + return self._expand_globs(globs, + list(self.core.metadata.groups.keys())) + + +class Help(InfoCmd, Bcfg2.Options.HelpCommand): + """ Get help on a specific subcommand """ + def command_registry(self): + return self.core.commands + + +class Debug(InfoCmd): + """ Shell out to a Python interpreter """ + interpreters, default_interpreter = load_interpreters() + options = [ + Bcfg2.Options.BooleanOption( + "-n", "--non-interactive", + help="Do not enter the interactive debugger"), + Bcfg2.Options.PathOption( + "-f", dest="cmd_list", type=argparse.FileType('r'), + help="File containing commands to run"), + Bcfg2.Options.Option( + "--interpreter", cf=("bcfg2-info", "interpreter"), + env="BCFG2_INFO_INTERPRETER", + choices=interpreters.keys(), default=default_interpreter)] + + def run(self, setup): + if setup.cmd_list: + console = InteractiveConsole(locals()) + for command in setup.cmd_list.readlines(): + command = command.strip() + if command: + console.push(command) + if not setup.non_interactive: + print("Dropping to interpreter; press ^D to resume") + self.interpreters[setup.interpreter](self.core.get_locals()) + + +class Build(InfoCmd): + """ Build config for hostname, writing to filename """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("filename", nargs='?', + default=sys.stdout, + type=argparse.FileType('w'))] + + def run(self, setup): + try: + lxml.etree.ElementTree( + self.core.BuildConfiguration(setup.hostname)).write( + setup.filename, + encoding='UTF-8', xml_declaration=True, + pretty_print=True) + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.filename, err)) + + +class Builddir(InfoCmd): + """ Build config for hostname, writing separate files to directory + """ + + # don't try to isntall these types of entries + blacklisted_types = ["nonexistent", "permissions"] + + options = Bcfg2.Client.Tools.POSIX.POSIX.options + [ + Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PathOption("directory")] + + help = """Generates a config for client <hostname> and writes the +individual configuration files out separately in a tree under <output +dir>. This only handles file entries, and does not respect 'owner' or +'group' attributes unless run as root. """ + + def run(self, setup): + setup.paranoid = False + client_config = self.core.BuildConfiguration(setup.hostname) + if client_config.tag == 'error': + print("Building client configuration failed.") + return 1 + + entries = [] + for struct in client_config: + for entry in struct: + if (entry.tag == 'Path' and + entry.get("type") not in self.blacklisted_types): + failure = entry.get("failure") + if failure is not None: + print("Skipping entry %s:%s with bind failure: %s" % + (entry.tag, entry.get("name"), failure)) + continue + entry.set('name', + os.path.join(setup.directory, + entry.get('name').lstrip("/"))) + entries.append(entry) + + Bcfg2.Client.Tools.POSIX.POSIX(client_config).Install(entries) + + +class Buildfile(InfoCmd): + """ Build config file for hostname """ + + options = [ + Bcfg2.Options.Option("-f", "--outfile", metavar="<path>", + type=argparse.FileType('w'), default=sys.stdout), + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + entry = lxml.etree.Element('Path', name=setup.filename) + if setup.altsrc: + entry.set("altsrc", setup.altsrc) + try: + self.core.Bind(entry, self.core.build_metadata(setup.hostname)) + except: # pylint: disable=W0702 + print("Failed to build entry %s for host %s" % (setup.filename, + setup.hostname)) + raise + try: + setup.outfile.write( + lxml.etree.tostring(entry, + xml_declaration=False).decode('UTF-8')) + setup.outfile.write("\n") + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.outfile.name, err)) + + +class BuildAllMixin(object): + """ InfoCmd mixin to make a version of an existing command that + applies to multiple hosts""" + + directory_arg = Bcfg2.Options.PathOption("directory") + hostname_arg = Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[]) + options = [directory_arg, hostname_arg] + + @property + def _parent(self): + """ the parent command """ + for cls in self.__class__.__mro__: + if (cls != InfoCmd and cls != self.__class__ and + issubclass(cls, InfoCmd)): + return cls + + def run(self, setup): + try: + os.makedirs(setup.directory) + except OSError: + err = sys.exc_info()[1] + if err.errno != 17: + print("Could not create %s: %s" % (setup.directory, err)) + return 1 + clients = self.get_client_list(setup.hostname) + for client in clients: + csetup = self._get_setup(client, copy.copy(setup)) + csetup.hostname = client + self._parent.run(self, csetup) # pylint: disable=E1101 + + def _get_setup(self, client, setup): + raise NotImplementedError + + +class Buildallfile(Buildfile, BuildAllMixin): + """ Build config file for all clients in directory """ + + options = [BuildAllMixin.directory_arg, + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + BuildAllMixin.hostname_arg] + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.outfile = open(os.path.join(setup.directory, client), 'w') + return setup + + +class Buildall(Build, BuildAllMixin): + """ Build configs for all clients in directory """ + + options = BuildAllMixin.options + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.filename = os.path.join(setup.directory, client + ".xml") + return setup + + +class Buildbundle(InfoCmd): + """ Render a templated bundle for hostname """ + + options = [Bcfg2.Options.PositionalArgument("bundle"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + bundler = self.core.plugins['Bundler'] + bundle = None + if setup.bundle in bundler.entries: + bundle = bundler.entries[setup.bundle] + elif not setup.bundle.endswith(".xml"): + fname = setup.bundle + ".xml" + if fname in bundler.entries: + bundle = bundler.entries[bundle] + if not bundle: + print("No such bundle %s" % setup.bundle) + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(bundle.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + except: # pylint: disable=W0702 + print("Failed to render bundle %s for host %s: %s" % + (setup.bundle, client, sys.exc_info()[1])) + raise + + +class Automatch(InfoCmd): + """ Perform automatch on a Properties file """ + + options = [ + Bcfg2.Options.BooleanOption( + "-f", "--force", + help="Force automatch even if it's disabled"), + Bcfg2.Options.PositionalArgument("propertyfile"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + props = self.core.plugins['Properties'] + except KeyError: + print("Properties plugin not enabled") + return 1 + + pfile = props.entries[setup.propertyfile] + if (not Bcfg2.Options.setup.force and + not Bcfg2.Options.setup.automatch and + pfile.xdata.get("automatch", "false").lower() != "true"): + print("Automatch not enabled on %s" % setup.propertyfile) + else: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(pfile.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class Bundles(InfoCmd): + """ Print out group/bundle info """ + + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def run(self, setup): + data = [('Group', 'Bundles')] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + ','.join(self.core.metadata.groups[group][0]))) + print_tabular(data) + + +class Clients(InfoCmd): + """ Print out client/profile info """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[])] + + def run(self, setup): + data = [('Client', 'Profile')] + for client in sorted(self.get_client_list(setup.hostname)): + imd = self.core.metadata.get_initial_metadata(client) + data.append((client, imd.profile)) + print_tabular(data) + + +class Config(InfoCmd): + """ Print out the current configuration of Bcfg2""" + + options = [ + Bcfg2.Options.BooleanOption( + "--raw", + help="Produce more accurate but less readable raw output")] + + def run(self, setup): + parser = Bcfg2.Options.get_parser() + data = [('Description', 'Value')] + for option in parser.option_list: + if hasattr(setup, option.dest): + value = getattr(setup, option.dest) + if any(issubclass(a.__class__, + Bcfg2.Options.ComponentAction) + for a in option.actions.values()): + if not setup.raw: + try: + if option.action.islist: + value = [v.__name__ for v in value] + else: + value = value.__name__ + except AttributeError: + # just use the value as-is + pass + if setup.raw: + value = repr(value) + data.append((getattr(option, "help", option.dest), value)) + print_tabular(data) + + +class Probes(InfoCmd): + """ Get probes for the given host """ + + options = [ + Bcfg2.Options.BooleanOption("-p", "--pretty", + help="Human-readable output"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + if setup.pretty: + probes = [] + else: + probes = lxml.etree.Element('probes') + metadata = self.core.build_metadata(setup.hostname) + for plugin in self.core.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for probe in plugin.GetProbes(metadata): + probes.append(probe) + if setup.pretty: + for probe in probes: + pname = probe.get("name") + print("=" * (len(pname) + 2)) + print(" %s" % pname) + print("=" * (len(pname) + 2)) + print("") + print(probe.text) + print("") + else: + print(lxml.etree.tostring(probes, xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class Showentries(InfoCmd): + """ Show abstract configuration entries for a given host """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("type", nargs='?')] + + def run(self, setup): + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + structures = self.core.GetStructures(metadata) + output = [('Entry Type', 'Name')] + etypes = None + if setup.type: + etypes = [setup.type, "Bound%s" % setup.type] + for item in structures: + output.extend((child.tag, child.get('name')) + for child in item.getchildren() + if not etypes or child.tag in etypes) + print_tabular(output) + + +class Groups(InfoCmd): + """ Print out group info """ + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def _profile_flag(self, group): + if self.core.metadata.groups[group].is_profile: + return 'yes' + else: + return 'no' + + def run(self, setup): + data = [("Groups", "Profile", "Category")] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + self._profile_flag(group), + self.core.metadata.groups[group].category)) + print_tabular(data) + + +class Showclient(InfoCmd): + """ Show metadata for the given hosts """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*')] + + def run(self, setup): + for client in self.get_client_list(setup.hostname): + try: + metadata = self.core.build_metadata(client) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Could not build metadata for %s: %s" % + (client, sys.exc_info()[1])) + continue + fmt = "%-10s %s" + print(fmt % ("Hostname:", metadata.hostname)) + print(fmt % ("Profile:", metadata.profile)) + + group_fmt = "%-10s %-30s %s" + header = False + for group in list(metadata.groups): + category = "" + for cat, grp in metadata.categories.items(): + if grp == group: + category = "Category: %s" % cat + break + if not header: + print(group_fmt % ("Groups:", group, category)) + header = True + else: + print(group_fmt % ("", group, category)) + + if metadata.bundles: + print(fmt % ("Bundles:", list(metadata.bundles)[0])) + for bnd in list(metadata.bundles)[1:]: + print(fmt % ("", bnd)) + if metadata.connectors: + print("Connector data") + print("=" * 80) + for conn in metadata.connectors: + if getattr(metadata, conn): + print(fmt % (conn + ":", getattr(metadata, conn))) + print("=" * 80) + + +class Mappings(InfoCmd): + """ Print generator mappings for optional type and name """ + + options = [Bcfg2.Options.PositionalArgument("type", nargs='?'), + Bcfg2.Options.PositionalArgument("name", nargs='?')] + + def run(self, setup): + data = [('Plugin', 'Type', 'Name')] + for generator in self.core.plugins_by_type( + Bcfg2.Server.Plugin.Generator): + etypes = setup.type or list(generator.Entries.keys()) + if setup.name: + interested = [(etype, [setup.name]) for etype in etypes] + else: + interested = [(etype, generator.Entries[etype]) + for etype in etypes + if etype in generator.Entries] + for etype, names in interested: + data.extend((generator.name, etype, name) + for name in names + if name in generator.Entries.get(etype, {})) + print_tabular(data) + + +class Packageresolve(InfoCmd): + """ Resolve packages for the given host""" + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("package", nargs="*")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + + metadata = self.core.build_metadata(setup.hostname) + + indep = lxml.etree.Element("Independent", + name=self.__class__.__name__.lower()) + if setup.package: + structures = [lxml.etree.Element("Bundle", name="packages")] + for package in setup.package: + lxml.etree.SubElement(structures[0], "Package", name=package) + else: + structures = self.core.GetStructures(metadata) + + pkgs._build_packages(metadata, indep, # pylint: disable=W0212 + structures) + print("%d new packages added" % len(indep.getchildren())) + if len(indep.getchildren()): + print(" %s" % "\n ".join(lxml.etree.tostring(p) + for p in indep.getchildren())) + + +class Packagesources(InfoCmd): + """ Show package sources """ + + options = [Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + return 1 + print(pkgs.get_collection(metadata).sourcelist()) + + +class Query(InfoCmd): + """ Query clients """ + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.Option( + "-g", "--group", metavar="<group>", dest="querygroups", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-p", "--profile", metavar="<profile>", dest="queryprofiles", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-b", "--bundle", metavar="<bundle>", dest="querybundles", + type=Bcfg2.Options.Types.comma_list), + required=True)] + + def run(self, setup): + if setup.queryprofiles: + res = self.core.metadata.get_client_names_by_profiles( + setup.queryprofiles) + elif setup.querygroups: + res = self.core.metadata.get_client_names_by_groups( + setup.querygroups) + elif setup.querybundles: + res = self.core.metadata.get_client_names_by_bundles( + setup.querybundles) + print("\n".join(res)) + + +class Shell(InfoCmd): + """ Open an interactive shell to run multiple bcfg2-info commands """ + interactive = False + + def run(self, setup): + loop = True + while loop: + try: + self.core.cmdloop('Welcome to bcfg2-info\n' + 'Type "help" for more information') + except SystemExit: + raise + except Bcfg2.Server.Plugin.PluginExecutionError: + continue + except KeyboardInterrupt: + print("Ctrl-C pressed, exiting...") + loop = False + except: + self.core.logger.error("Command failure", exc_info=1) + + +class ProfileTemplates(InfoCmd): + """ Benchmark template rendering times """ + + options = [ + Bcfg2.Options.Option( + "--clients", type=Bcfg2.Options.Types.comma_list, + help="Benchmark templates for the named clients"), + Bcfg2.Options.Option( + "--runs", help="Number of rendering passes per template", + default=5, type=int), + Bcfg2.Options.PositionalArgument( + "templates", nargs="*", default=[], + help="Profile the named templates instead of all templates")] + + def profile_entry(self, entry, metadata, runs=5): + times = [] + for i in range(runs): # pylint: disable=W0612 + start = time.time() + try: + self.core.Bind(entry, metadata) + times.append(time.time() - start) + except: # pylint: disable=W0702 + break + if times: + avg = sum(times) / len(times) + if avg: + self.logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) + return times + + def profile_struct(self, struct, metadata, templates=None, runs=5): + times = dict() + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates is None or entry.get("name") in templates: + self.logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, + len(entries))) + times.setdefault(entry.get("name"), + self.profile_entry(entry, metadata, + runs=runs)) + return times + + def profile_client(self, metadata, templates=None, runs=5): + structs = self.core.GetStructures(metadata) + struct_count = 0 + times = dict() + for struct in structs: + struct_count += 1 + self.logger.info("Rendering templates from structure %s:%s " + "(%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + times.update(self.profile_struct(struct, metadata, + templates=templates, runs=runs)) + return times + + def stdev(self, nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) + + def run(self, setup): + clients = self.get_client_list(setup.clients) + + times = dict() + client_count = 0 + for client in clients: + client_count += 1 + self.logger.info("Rendering templates for client %s (%s/%s)" % + (client, client_count, len(clients))) + times.update(self.profile_client(self.core.build_metadata(client), + templates=setup.templates, + runs=setup.runs)) + + # print out per-file results + tmpltimes = [] + for tmpl, ptimes in times.items(): + try: + mean = float(sum(ptimes)) / len(ptimes) + except ZeroDivisionError: + continue + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = self.stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or setup.templates: + tmpltimes.append((tmpl, mean, median, std)) + else: + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + + +if HAS_PROFILE: + class Profile(InfoCmd): + """ Profile a single bcfg2-info command """ + + options = [Bcfg2.Options.PositionalArgument("command"), + Bcfg2.Options.PositionalArgument("args", nargs="*")] + + def run(self, setup): + prof = profile.Profile() + cls = self.core.commands[setup.command] + prof.runcall(cls, " ".join(pipes.quote(a) for a in setup.args)) + display_trace(prof) + + +class InfoCore(cmd.Cmd, + Bcfg2.Server.Core.Core, + Bcfg2.Options.CommandRegistry): + """Main class for bcfg2-info.""" + + def __init__(self): + cmd.Cmd.__init__(self) + Bcfg2.Server.Core.Core.__init__(self) + Bcfg2.Options.CommandRegistry.__init__(self) + self.prompt = '> ' + + def get_locals(self): + return locals() + + def do_quit(self, _): + """ quit|exit - Exit program """ + raise SystemExit(0) + + do_EOF = do_quit + do_exit = do_quit + + def do_eventdebug(self, _): + """ eventdebug - Enable debugging output for FAM events """ + self.fam.set_debug(True) + + do_event_debug = do_eventdebug + + def do_update(self, _): + """ update - Process pending filesystem events """ + self.fam.handle_events_in_interval(0.1) + + def run(self): + self.load_plugins() + self.fam.handle_events_in_interval(1) + + def _daemonize(self): + pass + + def _run(self): + pass + + def _block(self): + pass + + def shutdown(self): + Bcfg2.Options.CommandRegistry.shutdown(self) + Bcfg2.Server.Core.Core.shutdown(self) + + +class CLI(object): + options = [Bcfg2.Options.BooleanOption("-p", "--profile", help="Profile")] + + def __init__(self): + Bcfg2.Options.register_commands(InfoCore, globals().values(), + parent=InfoCmd) + parser = Bcfg2.Options.get_parser( + description="Inspect a running Bcfg2 server", + components=[self, InfoCore]) + parser.parse() + + if Bcfg2.Options.setup.profile and HAS_PROFILE: + prof = profile.Profile() + self.core = prof.runcall(InfoCore) + display_trace(prof) + else: + if Bcfg2.Options.setup.profile: + print("Profiling functionality not available.") + self.core = InfoCore() + + for command in self.core.commands.values(): + command.core = self.core + + def run(self): + if Bcfg2.Options.setup.subcommand != 'help': + self.core.run() + return self.core.runcommand() diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 1fd9bc067..adfa96852 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -1,794 +1,8 @@ #!/usr/bin/env python """This tool loads the Bcfg2 core into an interactive debugger.""" -import os -import re import sys -import cmd -import getopt -import fnmatch -import logging -import lxml.etree -import traceback -from code import InteractiveConsole -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core -import Bcfg2.Server.Plugin -import Bcfg2.Client.Tools.POSIX - -try: - try: - import cProfile as profile - except ImportError: - import profile - import pstats - HAS_PROFILE = True -except ImportError: - HAS_PROFILE = False - - -class MockLog(object): - """ Fake logger that just discards all messages in order to mask - errors from builddir being unable to chown files it creates """ - def error(self, *args, **kwargs): - """ discard error messages """ - pass - - def warning(self, *args, **kwargs): - """ discard warning messages """ - pass - - def info(self, *args, **kwargs): - """ discard info messages """ - pass - - def debug(self, *args, **kwargs): - """ discard debug messages """ - pass - - -class FileNotBuilt(Exception): - """Thrown when File entry contains no content.""" - def __init__(self, value): - Exception.__init__(self) - self.value = value - - def __str__(self): - return repr(self.value) - - -def print_tabular(rows): - """Print data in tabular format.""" - cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 - for index in range(len(rows[0]))]) - fstring = (" %%-%ss |" * len(cmax)) % cmax - fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax - print(fstring % rows[0]) - print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') - for row in rows[1:]: - print(fstring % row) - - -def display_trace(trace): - """ display statistics from a profile trace """ - stats = pstats.Stats(trace) - stats.sort_stats('cumulative', 'calls', 'time') - stats.print_stats(200) - - -def load_interpreters(): - """ Load a dict of available Python interpreters """ - interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) - best = "python" - try: - import bpython.cli - interpreters["bpython"] = lambda v: bpython.cli.main(args=[], - locals_=v) - best = "bpython" - except ImportError: - pass - - try: - # whether ipython is actually better than bpython is - # up for debate, but this is the behavior that existed - # before --interpreter was added, so we call IPython - # better - import IPython - # pylint: disable=E1101 - if hasattr(IPython, "Shell"): - interpreters["ipython"] = lambda v: \ - IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() - best = "ipython" - elif hasattr(IPython, "embed"): - interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) - best = "ipython" - else: - print("Unknown IPython API version") - # pylint: enable=E1101 - except ImportError: - pass - - interpreters['best'] = interpreters[best] - return interpreters - - -class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): - """Main class for bcfg2-info.""" - def __init__(self): - cmd.Cmd.__init__(self) - Bcfg2.Server.Core.BaseCore.__init__(self) - self.prompt = '> ' - self.cont = True - - def _get_client_list(self, hostglobs): - """ given a host glob, get a list of clients that match it """ - # special cases to speed things up: - if '*' in hostglobs: - return self.metadata.clients - has_wildcards = False - for glob in hostglobs: - # check if any wildcard characters are in the string - if set('*?[]') & set(glob): - has_wildcards = True - break - if not has_wildcards: - return hostglobs - - rv = set() - clist = set(self.metadata.clients) - for glob in hostglobs: - for client in clist: - if fnmatch.fnmatch(client, glob): - rv.update(client) - clist.difference_update(rv) - return list(rv) - - def _get_usage(self, func): - """ get the short usage message for a given function """ - return "Usage: " + re.sub(r'\s+', ' ', func.__doc__).split(" - ", 1)[0] - - def do_loop(self): - """Looping.""" - self.cont = True - while self.cont: - try: - self.cmdloop('Welcome to bcfg2-info\n' - 'Type "help" for more information') - except SystemExit: - raise - except Bcfg2.Server.Plugin.PluginExecutionError: - continue - except KeyboardInterrupt: - print("Ctrl-C pressed exiting...") - self.do_exit([]) - except: - self.logger.error("Command failure", exc_info=1) - - def do_debug(self, args): - """ debug [-n] [-f <command list>] - Shell out to native - python interpreter """ - try: - opts, _ = getopt.getopt(args.split(), 'nf:') - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self._get_usage(self.do_debug)) - return - self.cont = False - scriptmode = False - interactive = True - for opt in opts: - if opt[0] == '-f': - scriptmode = True - spath = opt[1] - elif opt[0] == '-n': - interactive = False - if scriptmode: - console = InteractiveConsole(locals()) - for command in [c.strip() for c in open(spath).readlines()]: - if command: - console.push(command) - if interactive: - interpreters = load_interpreters() - if self.setup['interpreter'] in interpreters: - print("Dropping to %s interpreter; press ^D to resume" % - self.setup['interpreter']) - interpreters[self.setup['interpreter']](locals()) - else: - self.logger.error("Invalid interpreter %s" % - self.setup['interpreter']) - self.logger.error("Valid interpreters are: %s" % - ", ".join(interpreters.keys())) - - def do_quit(self, _): - """ quit|exit - Exit program """ - self.shutdown() - os._exit(0) # pylint: disable=W0212 - - do_EOF = do_quit - do_exit = do_quit - - def do_help(self, _): - """ help - Print this list of available commands """ - print(USAGE) - - def do_update(self, _): - """ update - Process pending filesystem events""" - self.fam.handle_events_in_interval(0.1) - - def do_build(self, args): - """ build [-f] <hostname> <filename> - Build config for - hostname, writing to filename""" - alist = args.split() - path_force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, ofile = alist - if not ofile.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - try: - lxml.etree.ElementTree(self.BuildConfiguration(client)).write( - ofile, - encoding='UTF-8', xml_declaration=True, - pretty_print=True) - except IOError: - err = sys.exc_info()[1] - print("Failed to write File %s: %s" % (ofile, err)) - else: - print(self._get_usage(self.do_build)) - - def help_builddir(self): - """Display help for builddir command.""" - print("""Usage: builddir [-f] <hostname> <output dir> - -Generates a config for client <hostname> and writes the -individual configuration files out separately in a tree -under <output dir>. The <output dir> directory must be -rooted under /tmp unless the -f argument is provided, in -which case it can be located anywhere. - -NOTE: Currently only handles file entries and writes -all content with the default owner and permissions. These -could be much more permissive than would be created by the -Bcfg2 client itself.""") - - def do_builddir(self, args): - """ builddir [-f] <hostname> <dirname> - Build config for - hostname, writing separate files to dirname""" - alist = args.split() - path_force = False - if '-f' in args: - alist.remove('-f') - path_force = True - if len(alist) == 2: - client, odir = alist - if not odir.startswith('/tmp') and not path_force: - print("Refusing to write files outside of /tmp without -f " - "option") - return - client_config = self.BuildConfiguration(client) - if client_config.tag == 'error': - print("Building client configuration failed.") - return - - for struct in client_config: - for entry in struct: - if entry.tag == 'Path': - entry.set('name', odir + '/' + entry.get('name')) - - posix = Bcfg2.Client.Tools.POSIX.POSIX(MockLog(), - self.setup, - client_config) - states = posix.Inventory() - posix.Install(list(states.keys())) - else: - print('Error: Incorrect number of parameters.') - self.help_builddir() - - def do_buildall(self, args): - """ buildall <directory> [<hostnames*>] - Build configs for - all clients in directory """ - alist = args.split() - if len(alist) < 1: - print(self._get_usage(self.do_buildall)) - return - - destdir = alist[0] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(alist) > 1: - clients = self._get_client_list(alist[1:]) - else: - clients = self.metadata.clients - for client in clients: - self.do_build("%s %s" % (client, os.path.join(destdir, - client + ".xml"))) - - def do_buildallfile(self, args): - """ buildallfile <directory> <filename> [<hostnames*>] - Build - config file for all clients in directory """ - try: - opts, args = getopt.gnu_getopt(args.split(), '', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self._get_usage(self.do_buildallfile)) - return - altsrc = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - if len(args) < 2: - print(self._get_usage(self.do_buildallfile)) - return - - destdir = args[0] - filename = args[1] - try: - os.mkdir(destdir) - except OSError: - err = sys.exc_info()[1] - if err.errno != 17: - print("Could not create %s: %s" % (destdir, err)) - if len(args) > 2: - clients = self._get_client_list(args[1:]) - else: - clients = self.metadata.clients - if altsrc: - args = "--altsrc %s -f %%s %%s %%s" % altsrc - else: - args = "-f %s %s %s" - for client in clients: - self.do_buildfile(args % (os.path.join(destdir, client), - filename, client)) - - def do_buildfile(self, args): - """ buildfile [-f <outfile>] [--altsrc=<altsrc>] <filename> - <hostname> - Build config file for hostname (not written to - disk)""" - try: - opts, alist = getopt.gnu_getopt(args.split(), 'f:', ['altsrc=']) - except getopt.GetoptError: - print(str(sys.exc_info()[1])) - print(self.do_buildfile.__doc__) - return - altsrc = None - outfile = None - for opt in opts: - if opt[0] == '--altsrc': - altsrc = opt[1] - elif opt[0] == '-f': - outfile = opt[1] - if len(alist) != 2: - print(self.do_buildfile.__doc__) - return - - fname, client = alist - entry = lxml.etree.Element('Path', type='file', name=fname) - if altsrc: - entry.set("altsrc", altsrc) - try: - metadata = self.build_metadata(client) - self.Bind(entry, metadata) - data = lxml.etree.tostring(entry, - xml_declaration=False).decode('UTF-8') - except Exception: - print("Failed to build entry %s for host %s: %s" % - (fname, client, traceback.format_exc().splitlines()[-1])) - raise - try: - if outfile: - open(outfile, 'w').write(data) - else: - print(data) - except IOError: - err = sys.exc_info()[1] - print("Could not write to %s: %s" % (outfile, err)) - print(data) - - def do_buildbundle(self, args): - """ buildbundle <bundle> <hostname> - Render a templated - bundle for hostname (not written to disk) """ - if len(args.split()) != 2: - print(self._get_usage(self.do_buildbundle)) - return 1 - - bname, client = args.split() - try: - metadata = self.build_metadata(client) - bundle = self.plugins['Bundler'].entries[bname] - print(lxml.etree.tostring(bundle.get_xml_value(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - except KeyError: - print("No such bundle %s" % bname) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Failed to render bundle %s for host %s: %s" % (bname, - client, - err)) - - def do_automatch(self, args): - """ automatch [-f] <propertyfile> <hostname> - Perform automatch on - a Properties file """ - alist = args.split() - force = False - for arg in alist: - if arg == '-f': - alist.remove('-f') - force = True - if len(alist) != 2: - print(self._get_usage(self.do_automatch)) - return - - if 'Properties' not in self.plugins: - print("Properties plugin not enabled") - return - - pname, client = alist - automatch = self.setup.cfp.getboolean("properties", "automatch", - default=False) - pfile = self.plugins['Properties'].store.entries[pname] - if (not force and - not automatch and - pfile.xdata.get("automatch", "false").lower() != "true"): - print("Automatch not enabled on %s" % pname) - else: - metadata = self.build_metadata(client) - print(lxml.etree.tostring(pfile.XMLMatch(metadata), - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_bundles(self, _): - """ bundles - Print out group/bundle info """ - data = [('Group', 'Bundles')] - groups = list(self.metadata.groups.keys()) - groups.sort() - for group in groups: - data.append((group, - ','.join(self.metadata.groups[group][0]))) - print_tabular(data) - - def do_clients(self, _): - """ clients - Print out client/profile info """ - data = [('Client', 'Profile')] - for client in sorted(self.metadata.list_clients()): - imd = self.metadata.get_initial_metadata(client) - data.append((client, imd.profile)) - print_tabular(data) - - def do_config(self, _): - """ config - Print out the current configuration of Bcfg2""" - output = [ - ('Description', 'Value'), - ('Path Bcfg2 repository', self.setup['repo']), - ('Plugins', self.setup['plugins']), - ('Password', self.setup['password']), - ('Filemonitor', self.setup['filemonitor']), - ('Server address', self.setup['location']), - ('Path to key', self.setup['key']), - ('Path to SSL certificate', self.setup['cert']), - ('Path to SSL CA certificate', self.setup['ca']), - ('Protocol', self.setup['protocol']), - ('Logging', self.setup['logging'])] - print_tabular(output) - - def do_probes(self, args): - """ probes [-p] <hostname> - Get probe list for the given - host, in XML (the default) or human-readable pretty (with -p) - format""" - alist = args.split() - pretty = False - if '-p' in alist: - pretty = True - alist.remove('-p') - if len(alist) != 1: - print(self._get_usage(self.do_probes)) - return - hostname = alist[0] - if pretty: - probes = [] - else: - probes = lxml.etree.Element('probes') - metadata = self.build_metadata(hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): - for probe in plugin.GetProbes(metadata): - probes.append(probe) - if pretty: - for probe in probes: - pname = probe.get("name") - print("=" * (len(pname) + 2)) - print(" %s" % pname) - print("=" * (len(pname) + 2)) - print("") - print(probe.text) - print("") - else: - print(lxml.etree.tostring(probes, - xml_declaration=False, - pretty_print=True).decode('UTF-8')) - - def do_showentries(self, args): - """ showentries <hostname> <type> - Show abstract - configuration entries for a given host """ - arglen = len(args.split()) - if arglen not in [1, 2]: - print(self._get_usage(self.do_showentries)) - return - client = args.split()[0] - try: - meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to find metadata for host %s" % client) - return - structures = self.GetStructures(meta) - output = [('entrytype', 'name')] - if arglen == 1: - for item in structures: - for child in item.getchildren(): - output.append((child.tag, child.get('name'))) - if arglen == 2: - etype = args.split()[1] - for item in structures: - for child in item.getchildren(): - if child.tag in [etype, "Bound%s" % etype]: - output.append((child.tag, child.get('name'))) - print_tabular(output) - - def do_groups(self, _): - """ groups - Print out group info """ - data = [("Groups", "Profile", "Category")] - grouplist = list(self.metadata.groups.keys()) - grouplist.sort() - for group in grouplist: - if self.metadata.groups[group].is_profile: - prof = 'yes' - else: - prof = 'no' - cat = self.metadata.groups[group].category - data.append((group, prof, cat)) - print_tabular(data) - - def do_showclient(self, args): - """ showclient <client> [<client> ...] - Show metadata for the - given hosts """ - if not len(args): - print(self._get_usage(self.do_showclient)) - return - for client in args.split(): - try: - client_meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Client %s not defined" % client) - continue - fmt = "%-10s %s" - print(fmt % ("Hostname:", client_meta.hostname)) - print(fmt % ("Profile:", client_meta.profile)) - - group_fmt = "%-10s %-30s %s" - header = False - for group in list(client_meta.groups): - category = "" - for cat, grp in client_meta.categories.items(): - if grp == group: - category = "Category: %s" % cat - break - if not header: - print(group_fmt % ("Groups:", group, category)) - header = True - else: - print(group_fmt % ("", group, category)) - - if client_meta.bundles: - print(fmt % ("Bundles:", list(client_meta.bundles)[0])) - for bnd in list(client_meta.bundles)[1:]: - print(fmt % ("", bnd)) - if client_meta.connectors: - print("Connector data") - print("=" * 80) - for conn in client_meta.connectors: - if getattr(client_meta, conn): - print(fmt % (conn + ":", getattr(client_meta, conn))) - print("=" * 80) - - def do_mappings(self, args): - """ mappings <type*> <name*> - Print generator mappings for - optional type and name """ - # Dump all mappings unless type specified - data = [('Plugin', 'Type', 'Name')] - arglen = len(args.split()) - for generator in self.plugins_by_type(Bcfg2.Server.Plugin.Generator): - if arglen == 0: - etypes = list(generator.Entries.keys()) - else: - etypes = [args.split()[0]] - if arglen == 2: - interested = [(etype, [args.split()[1]]) - for etype in etypes] - else: - interested = [(etype, generator.Entries[etype]) - for etype in etypes - if etype in generator.Entries] - for etype, names in interested: - for name in [name for name in names if name in - generator.Entries.get(etype, {})]: - data.append((generator.name, etype, name)) - print_tabular(data) - - def do_event_debug(self, _): - """ event_debug - Display filesystem events as they are - processed """ - self.fam.debug = True - - def do_packageresolve(self, args): - """ packageresolve <hostname> [<package> [<package>...]] - - Resolve packages for the given host, optionally specifying a - set of packages """ - arglist = args.split(" ") - if len(arglist) < 1: - print(self._get_usage(self.do_packageresolve)) - return - - try: - pkgs = self.plugins['Packages'] - except KeyError: - print("Packages plugin not enabled") - return - pkgs.toggle_debug() - - hostname = arglist[0] - metadata = self.build_metadata(hostname) - - indep = lxml.etree.Element("Independent") - if len(arglist) > 1: - structures = [lxml.etree.Element("Bundle", name="packages")] - for arg in arglist[1:]: - lxml.etree.SubElement(structures[0], "Package", name=arg) - else: - structures = self.GetStructures(metadata) - - pkgs._build_packages(metadata, indep, # pylint: disable=W0212 - structures) - print("%d new packages added" % len(indep.getchildren())) - if len(indep.getchildren()): - print(" %s" % "\n ".join(lxml.etree.tostring(p) - for p in indep.getchildren())) - - def do_packagesources(self, args): - """ packagesources <hostname> - Show package sources """ - if not args: - print(self._get_usage(self.do_packagesources)) - return - if 'Packages' not in self.plugins: - print("Packages plugin not enabled") - return - try: - metadata = self.build_metadata(args) - except Bcfg2.Server.Plugin.MetadataConsistencyError: - print("Unable to build metadata for host %s" % args) - return - collection = self.plugins['Packages'].get_collection(metadata) - print(collection.sourcelist()) - - def do_query(self, args): - """ query <-g group|-p profile|-b bundle> - Query clients """ - if len(args) == 0: - print("\n".join(self.metadata.clients)) - return - arglist = args.split(" ") - if len(arglist) != 2: - print(self._get_usage(self.do_query)) - return - - qtype, qparam = arglist - if qtype == '-p': - res = self.metadata.get_client_names_by_profiles(qparam.split(',')) - elif qtype == '-g': - res = self.metadata.get_client_names_by_groups(qparam.split(',')) - elif qtype == '-b': - res = self.metadata.get_client_names_by_bundles(qparam.split(',')) - else: - print(self._get_usage(self.do_query)) - return - print("\n".join(res)) - - def do_profile(self, arg): - """ profile <command> <args> - Profile a single bcfg2-info - command """ - if not HAS_PROFILE: - print("Profiling functionality not available.") - return - if len(arg) == 0: - print(self._get_usage(self.do_profile)) - return - prof = profile.Profile() - prof.runcall(self.onecmd, arg) - display_trace(prof) - - def run(self, args): # pylint: disable=W0221 - try: - self.load_plugins() - self.fam.handle_events_in_interval(1) - if args: - self.onecmd(" ".join(args)) - else: - self.do_loop() - finally: - self.shutdown() - - def _daemonize(self): - pass - - def _run(self): - pass - - def _block(self): - pass - - -def build_usage(): - """ build usage message """ - cmd_blacklist = ["do_loop", "do_EOF"] - usage = dict() - for attrname in dir(InfoCore): - attr = getattr(InfoCore, attrname) - - # shim for python 2.4, __func__ is im_func - funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None)) - if (funcattr is not None and - funcattr.func_name not in cmd_blacklist and - funcattr.func_name.startswith("do_") and - funcattr.func_doc): - usage[attr.__name__] = re.sub(r'\s+', ' ', attr.__doc__) - return "Commands:\n" + "\n".join(usage[k] for k in sorted(usage.keys())) - - -USAGE = build_usage() - - -def main(): - optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, - interactive=Bcfg2.Options.INTERACTIVE, - interpreter=Bcfg2.Options.INTERPRETER) - optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", - "Options:", - setup.buildHelpMessage(), - USAGE]) - - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-info', to_syslog=False, level=level) - - if setup['args'] and setup['args'][0] == 'help': - print(setup.hm) - sys.exit(0) - elif setup['profile'] and HAS_PROFILE: - prof = profile.Profile() - loop = prof.runcall(InfoCore) - display_trace(prof) - else: - if setup['profile']: - print("Profiling functionality not available.") - loop = InfoCore() - - loop.run(setup['args']) - +from Bcfg2.Server.Info import CLI if __name__ == '__main__': - sys.exit(main()) + sys.exit(CLI().run()) diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py deleted file mode 100755 index f4069e454..000000000 --- a/tools/bcfg2-profile-templates.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python -Ott -# -*- coding: utf-8 -*- -""" Benchmark template rendering times """ - -import sys -import time -import math -import signal -import logging -import operator -import Bcfg2.Logger -import Bcfg2.Options -import Bcfg2.Server.Core - - -def stdev(nums): - mean = float(sum(nums)) / len(nums) - return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) - - -def get_sigint_handler(core): - """ Get a function that handles SIGINT/Ctrl-C by shutting down the - core and exiting properly.""" - - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - core.shutdown() - os._exit(1) # pylint: disable=W0212 - - return hdlr - - -def main(): - optinfo = dict( - client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - runs=Bcfg2.Options.Option("Number of rendering passes per template", - cmd="--runs", - odesc="<runs>", - long_arg=True, - default=5, - cook=int)) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - - if setup['debug']: - level = logging.DEBUG - elif setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - Bcfg2.Logger.setup_logging("bcfg2-test", - to_console=setup['verbose'] or setup['debug'], - to_syslog=False, - to_file=setup['logging'], - level=level) - logger = logging.getLogger(sys.argv[0]) - - core = Bcfg2.Server.Core.BaseCore(setup) - signal.signal(signal.SIGINT, get_sigint_handler(core)) - logger.info("Bcfg2 server core loaded") - core.load_plugins() - logger.debug("Plugins loaded") - core.fam.handle_events_in_interval(0.1) - logger.debug("Repository events processed") - - if setup['args']: - templates = setup['args'] - else: - templates = [] - - if setup['client'] is None: - clients = [core.build_metadata(c) for c in core.metadata.clients] - else: - clients = [core.build_metadata(setup['client'])] - - times = dict() - client_count = 0 - for metadata in clients: - client_count += 1 - logger.info("Rendering templates for client %s (%s/%s)" % - (metadata.hostname, client_count, len(clients))) - structs = core.GetStructures(metadata) - struct_count = 0 - for struct in structs: - struct_count += 1 - logger.info("Rendering templates from structure %s:%s (%s/%s)" % - (struct.tag, struct.get("name"), struct_count, - len(structs))) - entries = struct.xpath("//Path") - entry_count = 0 - for entry in entries: - entry_count += 1 - if templates and entry.get("name") not in templates: - continue - logger.info("Rendering Path:%s (%s/%s)..." % - (entry.get("name"), entry_count, len(entries))) - ptimes = times.setdefault(entry.get("name"), []) - for i in range(setup['runs']): - start = time.time() - try: - core.Bind(entry, metadata) - ptimes.append(time.time() - start) - except: - break - if ptimes: - avg = sum(ptimes) / len(ptimes) - if avg: - logger.debug(" %s: %.02f sec" % - (metadata.hostname, avg)) - - # print out per-file results - tmpltimes = [] - for tmpl, ptimes in times.items(): - try: - mean = float(sum(ptimes)) / len(ptimes) - except ZeroDivisionError: - continue - ptimes.sort() - median = ptimes[len(ptimes) / 2] - std = stdev(ptimes) - if mean > 0.01 or median > 0.01 or std > 1 or templates: - tmpltimes.append((tmpl, mean, median, std)) - print("%-50s %-9s %-11s %6s" % - ("Template", "Mean Time", "Median Time", "σ")) - for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %9.02f %11.02f %6.02f" % info) - core.shutdown() - - -if __name__ == "__main__": - sys.exit(main()) |