diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/bcfg2-profile-templates.py | 110 | ||||
| -rwxr-xr-x | tools/bcfg2_local.py | 9 | ||||
| -rwxr-xr-x | tools/export.py | 43 | ||||
| -rwxr-xr-x | tools/posixusers_baseline.py | 4 | ||||
| -rwxr-xr-x | tools/selinux_baseline.py | 5 | ||||
| -rw-r--r-- | tools/upgrade/1.3/README | 5 | ||||
| -rwxr-xr-x | tools/upgrade/1.3/migrate_dbstats.py | 19 | ||||
| -rwxr-xr-x | tools/upgrade/1.3/migrate_info.py | 6 | ||||
| -rwxr-xr-x | tools/upgrade/1.3/migrate_perms_to_mode.py | 5 | ||||
| -rwxr-xr-x | tools/upgrade/1.3/migrate_probe_groups_to_db.py | 68 | ||||
| -rwxr-xr-x | tools/yum-listpkgs-xml.py | 2 |
11 files changed, 212 insertions, 64 deletions
diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index 3cd3786f9..f4069e454 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -1,25 +1,49 @@ #!/usr/bin/python -Ott +# -*- coding: utf-8 -*- """ Benchmark template rendering times """ -import os import sys import time +import math +import signal import logging import operator import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.Server.Core -LOGGER = None + +def stdev(nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) + + +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr def main(): - optinfo = \ - dict(client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - ) + optinfo = dict( + client=Bcfg2.Options.Option("Benchmark templates for one client", + cmd="--client", + odesc="<client>", + long_arg=True, + default=None), + runs=Bcfg2.Options.Option("Number of rendering passes per template", + cmd="--runs", + odesc="<runs>", + long_arg=True, + default=5, + cook=int)) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) @@ -39,13 +63,13 @@ def main(): logger = logging.getLogger(sys.argv[0]) core = Bcfg2.Server.Core.BaseCore(setup) + signal.signal(signal.SIGINT, get_sigint_handler(core)) logger.info("Bcfg2 server core loaded") + core.load_plugins() + logger.debug("Plugins loaded") core.fam.handle_events_in_interval(0.1) logger.debug("Repository events processed") - # how many times to render each template for each client - runs = 5 - if setup['args']: templates = setup['args'] else: @@ -57,41 +81,57 @@ def main(): clients = [core.build_metadata(setup['client'])] times = dict() + client_count = 0 for metadata in clients: - for struct in core.GetStructures(metadata): - logger.info("Rendering templates from structure %s:%s" % - (struct.tag, struct.get("name"))) - for entry in struct.xpath("//Path"): - path = entry.get("name") - logger.info("Rendering %s..." % path) - times[path] = dict() - avg = 0.0 - for i in range(runs): + client_count += 1 + logger.info("Rendering templates for client %s (%s/%s)" % + (metadata.hostname, client_count, len(clients))) + structs = core.GetStructures(metadata) + struct_count = 0 + for struct in structs: + struct_count += 1 + logger.info("Rendering templates from structure %s:%s (%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates and entry.get("name") not in templates: + continue + logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, len(entries))) + ptimes = times.setdefault(entry.get("name"), []) + for i in range(setup['runs']): start = time.time() try: core.Bind(entry, metadata) - avg += (time.time() - start) / runs + ptimes.append(time.time() - start) except: break - if avg: - logger.debug(" %s: %.02f sec" % (metadata.hostname, avg)) - times[path][metadata.hostname] = avg + if ptimes: + avg = sum(ptimes) / len(ptimes) + if avg: + logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) # print out per-file results tmpltimes = [] - for tmpl, clients in times.items(): + for tmpl, ptimes in times.items(): try: - avg = sum(clients.values()) / len(clients) + mean = float(sum(ptimes)) / len(ptimes) except ZeroDivisionError: continue - if avg > 0.01 or templates: - tmpltimes.append((tmpl, avg)) - print("%-50s %s" % ("Template", "Average Render Time")) - for tmpl, avg in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %.02f" % (tmpl, avg)) - - # TODO: complain about templates that on average were quick but - # for which some clients were slow + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + core.shutdown() if __name__ == "__main__": diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 2b9d39342..8c164e52e 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -19,7 +19,8 @@ class LocalCore(BaseCore): setup['logging'] = None Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) setup['syslog'], setup['logging'] = saved - self.fam.handle_events_in_interval(4) + self.load_plugins() + self.fam.handle_events_in_interval(0.1) def _daemonize(self): return True @@ -64,6 +65,12 @@ class LocalClient(Client): def main(): optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + if 'bundle_quick' in optinfo: + # CLIENT_BUNDLEQUICK option uses -Q, just like the server repo + # option. the server repo is more important for this + # application. + optinfo['bundle_quick'] = Bcfg2.Options.Option('bundlequick', + default=False) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) diff --git a/tools/export.py b/tools/export.py index 716c831d9..7c3c56db2 100755 --- a/tools/export.py +++ b/tools/export.py @@ -136,8 +136,7 @@ E.G. 1.2.0pre1 is a valid version. tarname = '/tmp/%s-%s.tar.gz' % (pkgname, version) - newchangelog = \ -"""bcfg2 (%s-0.0) unstable; urgency=low + newchangelog = """bcfg2 (%s-0.0) unstable; urgency=low * New upstream release @@ -177,7 +176,9 @@ E.G. 1.2.0pre1 is a valid version. "- New upstream release\n", "\n"] # write out the new RPM changelog - specs = ["misc/bcfg2.spec", "misc/bcfg2-selinux.spec", "redhat/bcfg2.spec.in"] + specs = ["misc/bcfg2.spec", + "misc/bcfg2-selinux.spec", + "redhat/bcfg2.spec.in"] if options.dryrun: print("*** Add the following to the top of the %%changelog section in %s:\n%s\n" % (rpmchangelog, " and ".join(specs))) @@ -227,8 +228,28 @@ E.G. 1.2.0pre1 is a valid version. 'VERSION="%s"\n' % version, startswith=True, dryrun=options.dryrun) - # set new version in setup.py - find_and_replace('setup.py', 'version=', ' version="%s",\n' % version, + # update solaris IPS version + find_and_replace('solaris-ips/Makefile', 'VERS=', + 'VERS=%s-1\n' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/MANIFEST.bcfg2.header', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2@', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2@%s"' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/MANIFEST.bcfg2-server.header', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@%s"' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/pkginfo.bcfg2', 'VERSION=', + 'VERSION="%s"\n' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/pkginfo.bcfg2-server', 'VERSION=', + 'VERSION="%s"\n' % version, + startswith=True, dryrun=options.dryrun) # set new version in Bcfg2/version.py find_and_replace('src/lib/Bcfg2/version.py', @@ -252,30 +273,30 @@ E.G. 1.2.0pre1 is a valid version. else: find_and_replace('misc/bcfg2.spec', 'Release: ', 'Release: 0.%s.%s\n' % - (version_info['build'][-1], version_info['build']), + (version_info['build'][-1], version_info['build']), dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', 'Release: ', 'Release: 0.%s.%s\n' % - (version_info['build'][-1], version_info['build']), + (version_info['build'][-1], version_info['build']), dryrun=options.dryrun) find_and_replace('misc/bcfg2.spec', '%setup', '%%setup -q -n %%{name}-%%{version}%s\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', '%setup', '%%setup -q -n %%{name}-%%{version}%s\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2.spec', 'BuildRoot', 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', 'BuildRoot', 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) # fix pre problem noted in diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py index a4abca42d..c45e54f1a 100755 --- a/tools/posixusers_baseline.py +++ b/tools/posixusers_baseline.py @@ -61,8 +61,8 @@ def main(): if entry.tag == 'POSIXUser': entry.set("group", grp.getgrgid(data[3])[0]) for group in users.user_supplementary_groups(entry): - memberof = lxml.etree.SubElement(entry, "MemberOf") - memberof.text = group[0] + memberof = lxml.etree.SubElement(entry, "MemberOf", + group=group[0]) entry.tag = "Bound" + entry.tag baseline.append(entry) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index b6997bb29..507a16f43 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -42,7 +42,10 @@ def main(): baseline.append(lxml.etree.Comment("%s entries" % etype)) extra = handler.FindExtra() for entry in extra: - entry.tag = "BoundSELinux" + if etype != "SEModule": + entry.tag = "Bound%s" % etype + else: + entry.tag = "%s" % etype baseline.extend(extra) print(lxml.etree.tostring(baseline, pretty_print=True)) diff --git a/tools/upgrade/1.3/README b/tools/upgrade/1.3/README index 2831d8f00..1a919f869 100644 --- a/tools/upgrade/1.3/README +++ b/tools/upgrade/1.3/README @@ -19,3 +19,8 @@ migrate_dbstats.py migrate_perms_to_mode.py - Convert perms attribute to mode (note that if you have info/:info files, you should run migrate_info.py first) + +migrate_probe_groups_to_db.py + - Migrate Probe host and group data from XML to DB backend for Metadata + and Probe plugins. Does not migrate individual probe return data. Assumes + migration to BOTH Metadata and Probe to database backends. diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index cbd2a6099..07def2ac8 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) _our_backend = None + def _quote(value): """ Quote a string to use as a table name or column @@ -44,12 +45,12 @@ def _migrate_perms(): fperms = {} logger.info("Creating FilePerms objects") - for data in ( ('owner', 'group', 'perms'), + for data in (('owner', 'group', 'perms'), ('current_owner', 'current_group', 'current_perms')): for grp in legacy_models.Reason.objects.values_list(*data).distinct(): if grp in fperms: continue - fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) + fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) fp.save() fperms[grp] = fp @@ -60,7 +61,7 @@ def _migrate_perms(): def _migrate_transaction(inter, entries, fperms): """helper""" - logger.debug("Migrating interaction %s for %s" % + logger.debug("Migrating interaction %s for %s" % (inter.id, inter.client.name)) newint = new_models.Interaction(id=inter.id, @@ -107,7 +108,7 @@ def _migrate_transaction(inter, entries, fperms): elif ent.kind == 'Package': act_dict['target_version'] = ei.reason.version act_dict['current_version'] = ei.reason.current_version - logger.debug("Adding package %s %s" % + logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict)) elif ent.kind == 'Path': @@ -116,7 +117,7 @@ def _migrate_transaction(inter, entries, fperms): act_dict['target_perms'] = fperms[( ei.reason.owner, - ei.reason.group, + ei.reason.group, ei.reason.perms )] @@ -141,7 +142,6 @@ def _migrate_transaction(inter, entries, fperms): act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED act_dict['details'] = ei.reason.unpruned - if ei.reason.is_sensitive: act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE elif ei.reason.is_binary: @@ -164,7 +164,7 @@ def _migrate_transaction(inter, entries, fperms): for entry_type in updates.keys(): i = 0 while(i < len(updates[entry_type])): - getattr(newint, entry_type).add(*updates[entry_type][i:i+100]) + getattr(newint, entry_type).add(*updates[entry_type][i:i + 100]) i += 100 for perf in inter.performance_items.all(): @@ -220,8 +220,8 @@ def _restructure(): # run any migrations from the previous schema try: - from Bcfg2.Server.Reports.updatefix import update_database - update_database() + from Bcfg2.Server.Reports.updatefix import update_database + update_database() except: logger.error("Failed to run legacy schema updates", exc_info=1) return False @@ -295,4 +295,3 @@ if __name__ == '__main__': Reports(setup).__call__(['update']) _restructure() - diff --git a/tools/upgrade/1.3/migrate_info.py b/tools/upgrade/1.3/migrate_info.py index e72599daf..3ccbf0285 100755 --- a/tools/upgrade/1.3/migrate_info.py +++ b/tools/upgrade/1.3/migrate_info.py @@ -1,12 +1,16 @@ #!/usr/bin/env python import os +import re import sys import lxml.etree import Bcfg2.Options from Bcfg2.Server.Plugin import INFO_REGEX +PERMS_REGEX = re.compile(r'perms:\s*(?P<perms>\w+)') + + def convert(info_file): info_xml = os.path.join(os.path.dirname(info_file), "info.xml") if os.path.exists(info_xml): @@ -16,7 +20,7 @@ def convert(info_file): fileinfo = lxml.etree.Element("FileInfo") info = lxml.etree.SubElement(fileinfo, "Info") for line in open(info_file).readlines(): - match = INFO_REGEX.match(line) + match = INFO_REGEX.match(line) or PERMS_REGEX.match(line) if match: mgd = match.groupdict() for key, value in list(mgd.items()): diff --git a/tools/upgrade/1.3/migrate_perms_to_mode.py b/tools/upgrade/1.3/migrate_perms_to_mode.py index e061558d3..18abffec2 100755 --- a/tools/upgrade/1.3/migrate_perms_to_mode.py +++ b/tools/upgrade/1.3/migrate_perms_to_mode.py @@ -13,6 +13,7 @@ def setmodeattr(elem): elem.set('mode', elem.get('perms')) del elem.attrib['perms'] return True + return False def writefile(f, xdata): @@ -32,7 +33,7 @@ def convertinfo(ifile): return found = False for i in xdata.findall('//Info'): - found = setmodeattr(i) + found |= setmodeattr(i) if found: writefile(ifile, xdata) @@ -47,7 +48,7 @@ def convertstructure(structfile): return found = False for path in xdata.xpath('//BoundPath|//Path'): - found = setmodeattr(path) + found |= setmodeattr(path) if found: writefile(structfile, xdata) diff --git a/tools/upgrade/1.3/migrate_probe_groups_to_db.py b/tools/upgrade/1.3/migrate_probe_groups_to_db.py new file mode 100755 index 000000000..73339e787 --- /dev/null +++ b/tools/upgrade/1.3/migrate_probe_groups_to_db.py @@ -0,0 +1,68 @@ +#!/bin/env python +""" Migrate Probe host and group data from XML to DB backend for Metadata +and Probe plugins. Does not migrate individual probe return data. Assumes +migration to BOTH Metadata and Probe to database backends. """ + +import os +os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' + +import lxml.etree +import sys +import Bcfg2.Options + +from Bcfg2.Server.Plugins.Metadata import MetadataClientModel +from Bcfg2.Server.Plugins.Probes import ProbesGroupsModel + +def migrate(xclient): + """ Helper to do the migration given a <Client/> XML element """ + client_name = xclient.get('name') + try: + try: + client = MetadataClientModel.objects.get(hostname=client_name) + except MetadataClientModel.DoesNotExist: + client = MetadataClientModel(hostname=client_name) + client.save() + except: + print("Failed to migrate client %s" % (client)) + return False + + try: + cgroups = [] + for xgroup in xclient.findall('Group'): + group_name = xgroup.get('name') + cgroups.append(group_name) + try: + group = ProbesGroupsModel.objects.get(hostname=client_name, group=group_name) + except ProbesGroupsModel.DoesNotExist: + group = ProbesGroupsModel(hostname=client_name, group=group_name) + group.save() + + ProbesGroupsModel.objects.filter( + hostname=client.hostname).exclude( + group__in=cgroups).delete() + + except: + print("Failed to migrate groups") + return False + return True + +def main(): + """ Main """ + opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) + setup = Bcfg2.Options.OptionParser(opts) + setup.parse(sys.argv[1:]) + + probefile = os.path.join(setup['repo'], 'Probes', "probed.xml") + + try: + xdata = lxml.etree.parse(probefile) + except lxml.etree.XMLSyntaxError: + err = sys.exc_info()[1] + print("Could not parse %s, skipping: %s" % (probefile, err)) + + for xclient in xdata.findall('Client'): + print "Migrating Metadata and Probe groups for %s" % xclient.get('name') + migrate(xclient) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/yum-listpkgs-xml.py b/tools/yum-listpkgs-xml.py index a052e75af..b4c5f6589 100755 --- a/tools/yum-listpkgs-xml.py +++ b/tools/yum-listpkgs-xml.py @@ -39,5 +39,5 @@ try: sys.argv = [sys.argv[0], '-d', '0', 'list'] yummain.main(sys.argv[1:]) except KeyboardInterrupt: - print("\n\nExiting on user cancel.", file=sys.stderr) + sys.stderr.write("\n\nExiting on user cancel.") sys.exit(1) |