From d7d90ca8626cae96d5aad3aac81967fa31cf7c4f Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Tue, 29 Jan 2013 09:38:45 -0600 Subject: add old schema to migration --- tools/upgrade/1.3/migrate_dbstats.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'tools') diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index 15bd328f9..69d9514df 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +os.environ['BCFG2_LEGACY_MODELS'] = '1' os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' import sys @@ -217,6 +218,14 @@ def migrate_stage1(): def _restructure(): """major restructure of reporting data""" + # run any migrations from the previous schema + try: + from Bcfg2.Server.Reports.updatefix import update_database + update_database() + except: + logger.error("Failed to run legacy schema updates", exc_info=1) + return False + # try to avoid dangling transactions if not migrate_stage1(): return -- cgit v1.2.3-1-g7c22 From 9c5c8d1076eca02d991da6a03136cf21bf9f7f5a Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 29 Jan 2013 11:36:32 -0500 Subject: bcfg2-profile-templates.py: various improvements --- tools/bcfg2-profile-templates.py | 104 +++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 60 deletions(-) (limited to 'tools') diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index fe7146d27..cc7a1a3d8 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -1,62 +1,61 @@ #!/usr/bin/python -Ott """ Benchmark template rendering times """ +import os import sys import time +import signal import logging -import logging.handlers import operator -import lxml.etree +import Bcfg2.Logger import Bcfg2.Server.Core LOGGER = None -def get_logger(setup=None): - """ set up logging according to the verbose level given on the - command line """ - global LOGGER - if LOGGER is None: - if setup is None: - setup = dict() - LOGGER = logging.getLogger(sys.argv[0]) - stderr = logging.StreamHandler() - level = logging.WARNING - lformat = "%(message)s" - if setup.get("debug", False): - stderr.setFormatter(logging.Formatter("%(asctime)s: %(levelname)s: %(message)s")) - level = logging.DEBUG - elif setup.get("verbose", False): - level = logging.INFO - LOGGER.setLevel(level) - LOGGER.addHandler(stderr) - syslog = logging.handlers.SysLogHandler("/dev/log") - syslog.setFormatter(logging.Formatter("%(name)s: %(message)s")) - LOGGER.addHandler(syslog) - return LOGGER + +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr + def main(): optinfo = \ - dict(configfile=Bcfg2.Options.CFILE, - help=Bcfg2.Options.HELP, - encoding=Bcfg2.Options.ENCODING, - repo=Bcfg2.Options.SERVER_REPOSITORY, - plugins=Bcfg2.Options.SERVER_PLUGINS, - password=Bcfg2.Options.SERVER_PASSWORD, - debug=Bcfg2.Options.DEBUG, - verbose=Bcfg2.Options.VERBOSE, - client=Bcfg2.Options.Option("Benchmark templates for one client", + dict(client=Bcfg2.Options.Option("Benchmark templates for one client", cmd="--client", odesc="", long_arg=True, default=None), ) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) - logger = get_logger(setup) + + if setup['debug']: + level = logging.DEBUG + elif setup['verbose']: + level = logging.INFO + else: + level = logging.WARNING + Bcfg2.Logger.setup_logging("bcfg2-test", + to_console=setup['verbose'] or setup['debug'], + to_syslog=False, + to_file=setup['logging'], + level=level) + logger = logging.getLogger(sys.argv[0]) core = Bcfg2.Server.Core.BaseCore(setup) + signal.signal(signal.SIGINT, get_sigint_handler(core)) logger.info("Bcfg2 server core loaded") - core.fam.handle_events_in_interval(4) + core.fam.handle_events_in_interval(0.1) logger.debug("Repository events processed") # how many times to render each template for each client @@ -73,34 +72,19 @@ def main(): clients = [core.build_metadata(setup['client'])] times = dict() - for plugin in ['Cfg', 'TGenshi', 'TCheetah']: - if plugin not in core.plugins: - logger.debug("Skipping disabled plugin %s" % plugin) - continue - logger.info("Rendering templates from plugin %s" % plugin) - - entrysets = [] - for template in templates: - try: - entrysets.append(core.plugins[plugin].entries[template]) - except KeyError: - logger.debug("Template %s not found in plugin %s" % - (template, plugin)) - if not entrysets: - logger.debug("Using all entrysets in plugin %s" % plugin) - entrysets = core.plugins[plugin].entries.values() - - for eset in entrysets: - path = eset.path.replace(setup['repo'], '') - logger.info("Rendering %s..." % path) - times[path] = dict() - for metadata in clients: + for metadata in clients: + for struct in core.GetStructures(metadata): + logger.info("Rendering templates from structure %s:%s" % + (struct.tag, struct.get("name"))) + for entry in struct.xpath("//Path"): + path = entry.get("name") + logger.info("Rendering %s..." % path) + times[path] = dict() avg = 0.0 for i in range(runs): - entry = lxml.etree.Element("Path") start = time.time() try: - eset.bind_entry(entry, metadata) + core.Bind(entry, metadata) avg += (time.time() - start) / runs except: break @@ -108,7 +92,7 @@ def main(): logger.debug(" %s: %.02f sec" % (metadata.hostname, avg)) times[path][metadata.hostname] = avg - # print out per-template results + # print out per-file results tmpltimes = [] for tmpl, clients in times.items(): try: -- cgit v1.2.3-1-g7c22 From 213290b04fe5e92c364619c76eb884be5445f271 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 4 Feb 2013 10:12:32 -0500 Subject: removed obsolete accounts2.xml.py --- tools/README | 4 -- tools/accounts2xml.py | 153 -------------------------------------------------- 2 files changed, 157 deletions(-) delete mode 100755 tools/accounts2xml.py (limited to 'tools') diff --git a/tools/README b/tools/README index 335363898..9e7f667e3 100644 --- a/tools/README +++ b/tools/README @@ -1,9 +1,5 @@ This directory contains repository maintenance tools. -accounts2xml.py - - Generate an XML description of accounts on a machine from - /etc/passwd - basebuilder.py - builds v2 base.xml from bcfg1 repo diff --git a/tools/accounts2xml.py b/tools/accounts2xml.py deleted file mode 100755 index 749f3b68c..000000000 --- a/tools/accounts2xml.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python -#=============================================================================== -# -# FILE: accounts2xml.py -# -# USAGE: ./accounts2xml.py filename node_name -# -# DESCRIPTION: A python script to generate accounts.xml containing only the login -# users from the given /etc/passwd file -# -# OPTIONS: --- -# REQUIREMENTS: --- -# BUGS: --- -# NOTES: --- -# AUTHOR: DongInn Kim (), dikim@cs.indiana.edu -# ORGANIZATION: Center for Research in Extreme Scale Technologies -# VERSION: 1.0 -# CREATED: 05/13/2012 01:44:43 PM -# REVISION: --- -#=============================================================================== - -# encoding: utf-8 - -""" -accounts2xml.py - -This script coverts a csv file to an XML. -The script takes 1 paramenters -* filename - -e.g., ./accounts2xml.py /etc/passwd - -Created by Giovanni Collazo on 2011-02-19. -Copyright (c) 2011 24veces.com. All rights reserved. - -Modified by DongInn Kim on 2012-05-13 -Copyright (c) 2012 Indiana University. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -""" - -import sys -import csv -import os -import re -import grp -from xml.dom.minidom import Document - -def main(args): - - try: - filename = "./copied_passwd" - with file(args[1], 'r') as original: data = original.read() - with file(filename, 'w') as modified: modified.write("name:pass:uid:gid:gecos:home:shell\n" + data); modified.close() - safe_filename = "Properties" - except IndexError: - print("ERROR: Please provide a filename.csv as the first argument") - sys.exit() - - node_user = "UnixUser" - node_group = "UnixGroup" - - f = csv.reader(open(filename, 'rb'), delimiter=':') - - doc = Document() - root_element = doc.createElement(safe_filename) - doc.appendChild(root_element) - - columns = f.next() - - groups = dict() - for row in f: - match = re.search(r'/bin/\w*sh', row[6]) # adjust this line to match the right shell path - if match: - item = doc.createElement(node_user) - root_element.appendChild(item) - extra_groups = os.popen("groups %s" % row[0]).readline()[:-1] - p_group = os.popen("id -gn %s" % row[0]).readline()[:-1] - extra_groups_str = extra_groups.split(' : ')[1] - populate_groups(groups, extra_groups_str) - item.setAttribute('extra_groups', get_extra_group_str(extra_groups_str, p_group)) - create_col_nodes(columns, item, doc, row) - - for gkey, gval in groups.items(): - item = doc.createElement(node_group) - root_element.appendChild(item) - item.setAttribute('name', gkey) - (gid,extra) = gval.split(':') - item.setAttribute('gid', gid) - - output_file = "accounts.xml" - doc.writexml(open(output_file, 'w'), addindent=' ', newl='\n') # Write file - - print("Done: Created %s" % output_file) - os.remove(filename) - -def get_extra_group_str(group_str, p_group): - groups = group_str.split(' ') - groups = [x for x in groups if p_group != x] - return ' '.join(groups) - - -def create_col_nodes(cols, item, doc, row): - for col in cols: - if col == "gid": - att = doc.createAttribute("group") - att.nodeValue = grp.getgrgid(int(row.pop(0)))[0] - else: - att = doc.createAttribute(str.replace(col, " ", "_").lower()) - att.nodeValue = row.pop(0) - - if col != "pass": - item.setAttributeNode(att) - -def populate_groups(group_dic, group_str): - for g in group_str.split(' '): - if not group_dic.has_key(g): - group_ent = os.popen("getent group %s" % g).readline()[:-1].split(':') - gid = group_ent[2] - extra = group_ent[3] - extra_list = list(extra) - for e in extra_list: - if e == ',': - loc = extra_list.index(e) - extra_list[loc] = ' ' - extra = "".join(extra_list) - group_dic[g] = gid + ":" + extra - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) - -# vim:set sr et ts=4 sw=4 ft=python fenc=utf-8: // See Vim, :help 'modeline' -# Created: Sun, 13 May 2012 13:44:43 -0400 - - -- cgit v1.2.3-1-g7c22