From 90b1276b8039642f95f5987ff1d0df413454a6d9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 17 Oct 2012 10:26:13 -0400 Subject: expanded pylint coverage to Admin modes, removed some old/broken admin modes --- src/lib/Bcfg2/Server/Admin/Backup.py | 22 ++++----- src/lib/Bcfg2/Server/Admin/Bundle.py | 80 -------------------------------- src/lib/Bcfg2/Server/Admin/Client.py | 18 +++---- src/lib/Bcfg2/Server/Admin/Compare.py | 9 ++-- src/lib/Bcfg2/Server/Admin/Init.py | 3 -- src/lib/Bcfg2/Server/Admin/Minestruct.py | 26 ++++------- src/lib/Bcfg2/Server/Admin/Perf.py | 10 ++-- src/lib/Bcfg2/Server/Admin/Pull.py | 56 +++++++++------------- src/lib/Bcfg2/Server/Admin/Query.py | 77 ------------------------------ src/lib/Bcfg2/Server/Admin/Reports.py | 57 +++++++++++------------ src/lib/Bcfg2/Server/Admin/Snapshots.py | 6 +-- src/lib/Bcfg2/Server/Admin/Syncdb.py | 23 +++++---- src/lib/Bcfg2/Server/Admin/Tidy.py | 66 -------------------------- src/lib/Bcfg2/Server/Admin/Viz.py | 59 +++++++++++------------ src/lib/Bcfg2/Server/Admin/Xcmd.py | 10 ++-- src/lib/Bcfg2/Server/Admin/__init__.py | 30 ++++++++++-- src/lib/Bcfg2/Server/Plugins/Metadata.py | 8 +++- src/sbin/bcfg2-admin | 31 +++++++------ src/sbin/bcfg2-info | 19 ++++++++ testsuite/Testsrc/test_code_checks.py | 18 ++----- 20 files changed, 198 insertions(+), 430 deletions(-) delete mode 100644 src/lib/Bcfg2/Server/Admin/Bundle.py delete mode 100644 src/lib/Bcfg2/Server/Admin/Query.py delete mode 100644 src/lib/Bcfg2/Server/Admin/Tidy.py diff --git a/src/lib/Bcfg2/Server/Admin/Backup.py b/src/lib/Bcfg2/Server/Admin/Backup.py index 3744abca3..be208d9b6 100644 --- a/src/lib/Bcfg2/Server/Admin/Backup.py +++ b/src/lib/Bcfg2/Server/Admin/Backup.py @@ -1,5 +1,6 @@ +""" Make a backup of the Bcfg2 repository """ + import os -import sys import time import tarfile import Bcfg2.Server.Admin @@ -7,19 +8,16 @@ import Bcfg2.Options class Backup(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Make a backup of the Bcfg2 repository" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin backup\n") - #"\n\nbcfg2-admin backup restore") - __usage__ = ("bcfg2-admin backup") + """ Make a backup of the Bcfg2 repository """ def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - self.datastore = self.setup['repo'] + datastore = self.setup['repo'] timestamp = time.strftime('%Y%m%d%H%M%S') - format = 'gz' - mode = 'w:' + format - filename = timestamp + '.tar' + '.' + format - out = tarfile.open(self.datastore + '/' + filename, mode=mode) - out.add(self.datastore, os.path.basename(self.datastore)) + fmt = 'gz' + mode = 'w:' + fmt + filename = timestamp + '.tar' + '.' + fmt + out = tarfile.open(os.path.join(datastore, filename), mode=mode) + out.add(datastore, os.path.basename(datastore)) out.close() - print("Archive %s was stored under %s" % (filename, self.datastore)) + print("Archive %s was stored under %s" % (filename, datastore)) diff --git a/src/lib/Bcfg2/Server/Admin/Bundle.py b/src/lib/Bcfg2/Server/Admin/Bundle.py deleted file mode 100644 index dbadaa3af..000000000 --- a/src/lib/Bcfg2/Server/Admin/Bundle.py +++ /dev/null @@ -1,80 +0,0 @@ -import lxml.etree -import glob -import sys -import re -import Bcfg2.Server.Admin -import Bcfg2.Options -from Bcfg2.Server.Plugin import MetadataConsistencyError - - -class Bundle(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "List and view bundle entries" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin bundle list-xml" - "\nbcfg2-admin bundle list-genshi" - "\nbcfg2-admin bundle show\n") - __usage__ = ("bcfg2-admin bundle [options] [list-xml|list-genshi|show]") - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - rg = re.compile(r'([^.]+\.(?:[a-z][a-z\-]+))(?![\w\.])', - re.IGNORECASE | re.DOTALL) - - # Get all bundles out of the Bundle/ directory - repo = self.setup['repo'] - xml_list = glob.glob("%s/Bundler/*.xml" % repo) - genshi_list = glob.glob("%s/Bundler/*.genshi" % repo) - - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin bundle help for usage.") - # Lists all available xml bundles - elif args[0] in ['list-xml', 'ls-xml']: - bundle_name = [] - for bundle_path in xml_list: - bundle_name.append(rg.search(bundle_path).group(1)) - for bundle in bundle_name: - print(bundle.split('.')[0]) - # Lists all available genshi bundles - elif args[0] in ['list-genshi', 'ls-gen']: - bundle_name = [] - for bundle_path in genshi_list: - bundle_name.append(rg.search(bundle_path).group(1)) - for bundle in bundle_name: - print(bundle.split('.')[0]) - # Shows a list of all available bundles and prints bundle - # details after the user choose one bundle. - # FIXME: Add support for detailed output of genshi bundles - # FIXME: This functionality is almost identical with - # bcfg2-info bundles - elif args[0] in ['show']: - bundle_name = [] - bundle_list = xml_list + genshi_list - for bundle_path in bundle_list: - bundle_name.append(rg.search(bundle_path).group(1)) - text = "Available bundles (Number of bundles: %s)" % \ - (len(bundle_list)) - print(text) - print("%s" % (len(text) * "-")) - for i in range(len(bundle_list)): - print("[%i]\t%s" % (i, bundle_name[i])) - try: - lineno = raw_input("Enter the line number of a bundle for details: ") - except NameError: - lineno = input("Enter the line number of a bundle for details: ") - if int(lineno) >= int(len(bundle_list)): - print("No line with this number.") - else: - if '%s/Bundler/%s' % \ - (repo, bundle_name[int(lineno)]) in genshi_list: - print("Detailed output for *.genshi bundles is not supported.") - else: - print('Details for the "%s" bundle:' % \ - (bundle_name[int(lineno)].split('.')[0])) - tree = lxml.etree.parse(bundle_list[int(lineno)]) - names = ['Action', 'Package', 'Path', 'Service'] - for name in names: - for node in tree.findall("//" + name): - print("%s:\t%s" % (name, node.attrib["name"])) - else: - print("No command specified") - raise SystemExit(1) diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py index 277e7e3d5..ca9de27c7 100644 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -1,14 +1,13 @@ -import lxml.etree +""" Create, delete, or list client entries """ + +import sys import Bcfg2.Server.Admin from Bcfg2.Server.Plugin import MetadataConsistencyError class Client(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Create, delete, or list client entries" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin client add " - "\nbcfg2-admin client list" - "\nbcfg2-admin client del \n") - __usage__ = ("bcfg2-admin client [options] [add|del|list] [attr=val]") + """ Create, delete, or list client entries """ + __usage__ = "[options] [add|del|list] [attr=val]" def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) @@ -19,13 +18,15 @@ class Client(Bcfg2.Server.Admin.MetadataCore): try: self.metadata.add_client(args[1]) except MetadataConsistencyError: - print("Error in adding client") + err = sys.exc_info()[1] + print("Error in adding client: %s" % err) raise SystemExit(1) elif args[0] in ['delete', 'remove', 'del', 'rm']: try: self.metadata.remove_client(args[1]) except MetadataConsistencyError: - print("Error in deleting client") + err = sys.exc_info()[1] + print("Error in deleting client: %s" % err) raise SystemExit(1) elif args[0] in ['list', 'ls']: for client in self.metadata.list_clients(): @@ -33,4 +34,3 @@ class Client(Bcfg2.Server.Admin.MetadataCore): else: print("No command specified") raise SystemExit(1) - diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index b5e3c3c01..c56dd0a8f 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -1,15 +1,12 @@ import lxml.etree import os - import Bcfg2.Server.Admin class Compare(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("Determine differences between files or " - "directories of client specification instances") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin compare " - "\nbcfg2-admin compare -r ") - __usage__ = ("bcfg2-admin compare \n\n" + """ Determine differences between files or directories of client + specification instances """ + __usage__ = (" \n\n" " -r\trecursive") def __init__(self, setup): diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 286c52d5d..869dc1aca 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -133,9 +133,6 @@ def create_conf(confpath, confdata): class Init(Bcfg2.Server.Admin.Mode): """Interactively initialize a new repository.""" - __shorthelp__ = ("Interactively initialize a new repository.") - __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin init" - __usage__ = "bcfg2-admin init" options = {'configfile': Bcfg2.Options.CFILE, 'plugins': Bcfg2.Options.SERVER_PLUGINS, 'proto': Bcfg2.Options.SERVER_PROTOCOL, diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 2db80f5c5..6d0dab106 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -1,22 +1,17 @@ +""" Extract extra entry lists from statistics """ import getopt import lxml.etree import sys - import Bcfg2.Server.Admin + class Minestruct(Bcfg2.Server.Admin.StructureMode): - """Pull extra entries out of statistics.""" - __shorthelp__ = "Extract extra entry lists from statistics" - __longhelp__ = (__shorthelp__ + - "\n\nbcfg2-admin minestruct [-f filename] " - "[-g groups] client\n") - __usage__ = ("bcfg2-admin minestruct [options] \n\n" + """ Extract extra entry lists from statistics """ + __usage__ = ("[options] \n\n" " %-25s%s\n" " %-25s%s\n" % - ("-f ", - "build a particular file", - "-g ", - "only build config for groups")) + ("-f ", "build a particular file", + "-g ", "only build config for groups")) def __call__(self, args): if len(args) == 0: @@ -25,7 +20,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: (opts, args) = getopt.getopt(args, 'f:g:h') except: - self.log.error(self.__shorthelp__) + self.log.error(self.__doc__) raise SystemExit(1) client = args[0] @@ -54,11 +49,10 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): root = lxml.etree.Element("Base") self.log.info("Found %d extra entries" % (len(extra))) add_point = root - for g in groups: - add_point = lxml.etree.SubElement(add_point, "Group", name=g) + for grp in groups: + add_point = lxml.etree.SubElement(add_point, "Group", name=grp) for tag, name in extra: self.log.info("%s: %s" % (tag, name)) lxml.etree.SubElement(add_point, tag, name=name) - tree = lxml.etree.ElementTree(root) - tree.write(output, pretty_print=True) + lxml.etree.ElementTree(root).write(output, pretty_print=True) diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py index 76bd24ca7..86eb6810d 100644 --- a/src/lib/Bcfg2/Server/Admin/Perf.py +++ b/src/lib/Bcfg2/Server/Admin/Perf.py @@ -1,14 +1,13 @@ -import sys +""" Get performance data from server """ +import sys import Bcfg2.Options import Bcfg2.Proxy import Bcfg2.Server.Admin class Perf(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("Query server for performance data") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin perf\n") - __usage__ = ("bcfg2-admin perf") + """ Get performance data from server """ def __call__(self, args): output = [('Name', 'Min', 'Max', 'Mean', 'Count')] @@ -34,5 +33,6 @@ class Perf(Bcfg2.Server.Admin.Mode): for key in sorted(data.keys()): output.append((key, ) + tuple(["%.06f" % item - for item in data[key][:-1]] + [data[key][-1]])) + for item in data[key][:-1]] + \ + [data[key][-1]])) self.print_table(output) diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index acbb44e9e..3cc7a2cc5 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -1,33 +1,24 @@ +""" Retrieves entries from clients and integrates the information into +the repository """ + import getopt import sys - import Bcfg2.Server.Admin -from Bcfg2.Compat import input +from Bcfg2.Compat import input # pylint: disable=W0622 class Pull(Bcfg2.Server.Admin.MetadataCore): - """Pull mode retrieves entries from clients and - integrates the information into the repository. - """ - __shorthelp__ = ("Integrate configuration information " - "from clients into the server repository") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin pull [-v] [-f][-I] [-s] " - " \n") - __usage__ = ("bcfg2-admin pull [options] " - "\n\n" + """ Retrieves entries from clients and integrates the information + into the repository """ + __usage__ = ("[options] \n\n" " %-25s%s\n" " %-25s%s\n" " %-25s%s\n" " %-25s%s\n" % - ("-v", - "be verbose", - "-f", - "force", - "-I", - "interactive", - "-s", - "stdin")) - allowed = ['Metadata', "DBStats", "Statistics", "Cfg", "SSHbase"] + ("-v", "be verbose", + "-f", "force", + "-I", "interactive", + "-s", "stdin")) def __init__(self, setup): Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) @@ -40,7 +31,7 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): try: opts, gargs = getopt.getopt(args, 'vfIs') except: - print(self.__shorthelp__) + print(self.__doc__) raise SystemExit(1) for opt in opts: if opt[0] == '-v': @@ -48,7 +39,7 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): elif opt[0] == '-f': self.mode = 'force' elif opt[0] == '-I': - self.mode == 'interactive' + self.mode = 'interactive' elif opt[0] == '-s': use_stdin = True @@ -61,8 +52,7 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): except: print("Bad entry: %s" % line.strip()) elif len(gargs) < 3: - print(self.__longhelp__) - raise SystemExit(1) + self.usage() else: self.PullEntry(gargs[0], gargs[1], gargs[2]) @@ -90,9 +80,9 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): print("Unable to build entry. " "Do you have a statistics plugin enabled?") raise SystemExit(1) - for k, v in list(data.items()): - if v: - new_entry[k] = v + for key, val in list(data.items()): + if val: + new_entry[key] = val return new_entry def Choose(self, choices): @@ -129,20 +119,20 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): glist = [gen for gen in self.bcore.generators if ename in gen.Entries.get(etype, {})] if len(glist) != 1: - self.errExit("Got wrong numbers of matching generators for entry:" \ - + "%s" % ([g.name for g in glist])) + self.errExit("Got wrong numbers of matching generators for entry:" + "%s" % ([g.name for g in glist])) plugin = glist[0] if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): - self.errExit("Configuration upload not supported by plugin %s" \ - % (plugin.name)) + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) try: choices = plugin.AcceptChoices(new_entry, meta) specific = self.Choose(choices) if specific: plugin.AcceptPullData(specific, new_entry, self.log) except Bcfg2.Server.Plugin.PluginExecutionError: - self.errExit("Configuration upload not supported by plugin %s" \ - % (plugin.name)) + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) # Commit if running under a VCS for vcsplugin in list(self.bcore.plugins.values()): if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): diff --git a/src/lib/Bcfg2/Server/Admin/Query.py b/src/lib/Bcfg2/Server/Admin/Query.py deleted file mode 100644 index e63ca61b8..000000000 --- a/src/lib/Bcfg2/Server/Admin/Query.py +++ /dev/null @@ -1,77 +0,0 @@ -import sys -import logging -import Bcfg2.Logger -import Bcfg2.Server.Admin - - -class Query(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Query clients" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin query [-n] [-c] " - "[-f filename] g=group p=profile b=bundle") - __usage__ = ("bcfg2-admin query [options] \n\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-n", - "query results delimited with newlines", - "-c", - "query results delimited with commas", - "-f filename", - "write query to file")) - - __plugin_blacklist__ = ['DBStats', 'Cfg', 'Pkgmgr', 'Packages' ] - - def __init__(self, setup): - Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) - logging.root.setLevel(100) - Bcfg2.Logger.setup_logging(100, to_console=False, - to_syslog=setup['syslog']) - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - clients = self.metadata.clients - filename_arg = False - filename = None - for arg in args: - if filename_arg == True: - filename = arg - filename_arg = False - continue - if arg in ['-n', '-c']: - continue - if arg in ['-f']: - filename_arg = True - continue - try: - k, v = arg.split('=') - except: - print("Unknown argument %s" % arg) - continue - if k == 'p': - nc = self.metadata.get_client_names_by_profiles(v.split(',')) - elif k == 'g': - nc = self.metadata.get_client_names_by_groups(v.split(',')) - # add probed groups (if present) - for conn in self.bcore.connectors: - if isinstance(conn, Bcfg2.Server.Plugins.Probes.Probes): - for c, glist in list(conn.cgroups.items()): - for g in glist: - if g in v.split(','): - nc.append(c) - elif k == 'b': - nc = self.metadata.get_client_names_by_bundles(v.split(',')) - else: - print("One of g=, p= or b= must be specified") - raise SystemExit(1) - clients = [c for c in clients if c in nc] - if '-n' in args: - for client in clients: - print(client) - else: - print(','.join(clients)) - if '-f' in args: - f = open(filename, "w") - for client in clients: - f.write(client + "\n") - f.close() - print("Wrote results to %s" % (filename)) diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 1ff250bdf..15ff79a35 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -3,14 +3,8 @@ import Bcfg2.Logger import Bcfg2.Server.Admin import datetime import os -import logging -import pickle -import platform import sys import traceback - -from Bcfg2.Compat import md5 - from Bcfg2 import settings # Load django and reports stuff _after_ we know we can load settings @@ -25,7 +19,7 @@ sys.path.pop() # Set DJANGO_SETTINGS_MODULE appropriately. os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name -from django.db import connection, transaction +from django.db import transaction from Bcfg2.Reporting.models import Client, Interaction, \ Performance, Bundle, Group, FailureEntry, PathEntry, \ @@ -57,21 +51,20 @@ def printStats(fn): class Reports(Bcfg2.Server.Admin.Mode): - '''Admin interface for dynamic reports''' - __shorthelp__ = "Manage dynamic reports" - __longhelp__ = (__shorthelp__) + """ Manage dynamic reports """ django_commands = ['dbshell', 'shell', 'sqlall', 'validate'] - __usage__ = ("bcfg2-admin reports [command] [options]\n" - "\n" + __usage__ = ("[command] [options]\n" " Commands:\n" " init Initialize the database\n" " purge Purge records\n" " --client [n] Client to operate on\n" " --days [n] Records older then n days\n" " --expired Expired clients only\n" - " scrub Scrub the database for duplicate reasons and orphaned entries\n" + " scrub Scrub the database for duplicate " + "reasons and orphaned entries\n" " stats print database statistics\n" - " update Apply any updates to the reporting database\n" + " update Apply any updates to the reporting " + "database\n" "\n" " Django commands:\n " \ + "\n ".join(django_commands)) @@ -108,8 +101,9 @@ class Reports(Bcfg2.Server.Admin.Mode): management.call_command("syncdb", verbosity=vrb) management.call_command("migrate", verbosity=vrb) except: - print("Update failed: %s" % traceback.format_exc().splitlines()[-1]) - raise SystemExit(-1) + print("Update failed: %s" % + traceback.format_exc().splitlines()[-1]) + raise SystemExit(1) elif args[0] == 'purge': expired = False client = None @@ -127,9 +121,11 @@ class Reports(Bcfg2.Server.Admin.Mode): if maxdate: self.errExit("Max date specified multiple times") try: - maxdate = datetime.datetime.now() - datetime.timedelta(days=int(args[i + 1])) + maxdate = datetime.datetime.now() - \ + datetime.timedelta(days=int(args[i + 1])) except: - self.log.error("Invalid number of days: %s" % args[i + 1]) + self.log.error("Invalid number of days: %s" % + args[i + 1]) raise SystemExit(-1) i = i + 1 elif args[i] == '--expired': @@ -150,18 +146,17 @@ class Reports(Bcfg2.Server.Admin.Mode): ''' Perform a thorough scrub and cleanup of the database ''' # Cleanup unused entries - for cls in (Group, Bundle, FailureEntry, ActionEntry, PathEntry, - PackageEntry, PathEntry): + for cls in (Group, Bundle, FailureEntry, ActionEntry, PathEntry, + PackageEntry, PathEntry): try: start_count = cls.objects.count() cls.prune_orphans() self.log.info("Pruned %d %s records" % \ (start_count - cls.objects.count(), cls.__class__.__name__)) except: - print("Failed to prune %s: %s" % \ - (cls.__class__.__name__, - traceback.format_exc().splitlines()[-1])) - + print("Failed to prune %s: %s" % + (cls.__class__.__name__, + traceback.format_exc().splitlines()[-1])) def django_command_proxy(self, command): '''Call a django command''' @@ -170,7 +165,6 @@ class Reports(Bcfg2.Server.Admin.Mode): else: management.call_command(command) - @printStats def purge(self, client=None, maxdate=None, state=None): '''Purge historical data from the database''' @@ -197,15 +191,16 @@ class Reports(Bcfg2.Server.Admin.Mode): self.log.debug("Filtering by maxdate: %s" % maxdate) ipurge = ipurge.filter(timestamp__lt=maxdate) - if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': + if settings.DATABASES['default']['ENGINE'] == \ + 'django.db.backends.sqlite3': grp_limit = 100 else: grp_limit = 1000 if state: filtered = True if state not in ('dirty', 'clean', 'modified'): - raise TypeError("state is not one of the following values " + \ - "('dirty','clean','modified')") + raise TypeError("state is not one of the following values: " + "dirty, clean, modified") self.log.debug("Filtering by state: %s" % state) ipurge = ipurge.filter(state=state) @@ -217,7 +212,8 @@ class Reports(Bcfg2.Server.Admin.Mode): # just in case... if not grp: break - Interaction.objects.filter(id__in=[x['id'] for x in grp]).delete() + Interaction.objects.filter(id__in=[x['id'] + for x in grp]).delete() rnum += len(grp) self.log.debug("Deleted %s of %s" % (rnum, count)) except: @@ -235,7 +231,7 @@ class Reports(Bcfg2.Server.Admin.Mode): m2m.prune_orphans() if client and not filtered: - '''Delete the client, ping data is automatic''' + # Delete the client, ping data is automatic try: self.log.debug("Purging client %s" % client) cobj.delete() @@ -271,4 +267,3 @@ class Reports(Bcfg2.Server.Admin.Mode): for cls in classes: print("%s has %s records" % (cls().__class__.__name__, cls.objects.count())) - diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index 36e3dfe02..c2d279391 100644 --- a/src/lib/Bcfg2/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -12,10 +12,8 @@ from Bcfg2.Server.Snapshots.model import Snapshot, Client, Metadata, Base, \ from Bcfg2.Compat import u_str class Snapshots(Bcfg2.Server.Admin.Mode): - __shorthelp__ = "Interact with the Snapshots system" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin snapshots init" - "\nbcfg2-admin query qtype\n") - __usage__ = ("bcfg2-admin snapshots [init|query qtype]") + """ Interact with the Snapshots system """ + __usage__ = "[init|query qtype]" q_dispatch = {'client': Client, 'group': Group, diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py index 4126c18af..4ba840b86 100644 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py @@ -1,3 +1,4 @@ +import sys import Bcfg2.settings import Bcfg2.Options import Bcfg2.Server.Admin @@ -6,27 +7,25 @@ from django.core.management import setup_environ, call_command class Syncdb(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("Sync the Django ORM with the configured database") - __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin syncdb" - __usage__ = "bcfg2-admin syncdb" + """ Sync the Django ORM with the configured database """ options = {'configfile': Bcfg2.Options.WEB_CFILE} def __call__(self, args): # Parse options - self.opts = Bcfg2.Options.OptionParser(self.options) - self.opts.parse(args) + opts = Bcfg2.Options.OptionParser(self.options) + opts.parse(args) setup_environ(Bcfg2.settings) - Bcfg2.Server.models.load_models(cfile=self.opts['configfile']) + Bcfg2.Server.models.load_models(cfile=opts['configfile']) try: call_command("syncdb", interactive=False, verbosity=0) self._database_available = True except ImproperlyConfigured: - self.logger.error("Django configuration problem: %s" % - format_exc().splitlines()[-1]) - raise SystemExit(-1) + err = sys.exc_info()[1] + self.log.error("Django configuration problem: %s" % err) + raise SystemExit(1) except: - self.logger.error("Database update failed: %s" % - format_exc().splitlines()[-1]) - raise SystemExit(-1) + err = sys.exc_info()[1] + self.log.error("Database update failed: %s" % err) + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Server/Admin/Tidy.py b/src/lib/Bcfg2/Server/Admin/Tidy.py deleted file mode 100644 index 8a417a427..000000000 --- a/src/lib/Bcfg2/Server/Admin/Tidy.py +++ /dev/null @@ -1,66 +0,0 @@ -import os -import re -import socket - -import Bcfg2.Server.Admin -from Bcfg2.Compat import input - - -class Tidy(Bcfg2.Server.Admin.Mode): - __shorthelp__ = "Clean up useless files in the repo" - __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin tidy [-f] [-I]\n" - __usage__ = ("bcfg2-admin tidy [options]\n\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-f", - "force", - "-I", - "interactive")) - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - badfiles = self.buildTidyList() - if '-f' in args or '-I' in args: - if '-I' in args: - for name in badfiles[:]: - answer = input("Unlink file %s? [yN] " % name) - if answer not in ['y', 'Y']: - badfiles.remove(name) - for name in badfiles: - try: - os.unlink(name) - except IOError: - print("Failed to unlink %s" % name) - else: - for name in badfiles: - print(name) - - def buildTidyList(self): - """Clean up unused or unusable files from the repository.""" - hostmatcher = re.compile('.*\.H_(\S+)$') - to_remove = [] - good = [] - bad = [] - - # clean up unresolvable hosts in SSHbase - for name in os.listdir("%s/SSHbase" % self.setup['repo']): - if hostmatcher.match(name): - hostname = hostmatcher.match(name).group(1) - if hostname in good + bad: - continue - try: - socket.gethostbyname(hostname) - good.append(hostname) - except: - bad.append(hostname) - for name in os.listdir("%s/SSHbase" % self.setup['repo']): - if not hostmatcher.match(name): - to_remove.append("%s/SSHbase/%s" % (self.setup['repo'], - name)) - else: - if hostmatcher.match(name).group(1) in bad: - to_remove.append("%s/SSHbase/%s" % - (self.setup['repo'], name)) - # clean up file~ - # clean up files without parsable names in Cfg - return to_remove diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py index b190dd62a..1d9d25f16 100644 --- a/src/lib/Bcfg2/Server/Admin/Viz.py +++ b/src/lib/Bcfg2/Server/Admin/Viz.py @@ -1,17 +1,14 @@ +""" Produce graphviz diagrams of metadata structures """ + import getopt from subprocess import Popen, PIPE -import sys import pipes import Bcfg2.Server.Admin class Viz(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Produce graphviz diagrams of metadata structures" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin viz [--includehosts] " - "[--includebundles] [--includekey] " - "[--only-client clientname] " - "[-o output.]\n") - __usage__ = ("bcfg2-admin viz [options]\n\n" + """ Produce graphviz diagrams of metadata structures """ + __usage__ = ("[options]\n\n" " %-32s%s\n" " %-32s%s\n" " %-32s%s\n" @@ -32,23 +29,20 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): 'indianred1', 'limegreen', 'orange1', 'lightblue2', 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] - __plugin_blacklist__ = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', 'Packages', - 'Rules', 'Account', 'Decisions', 'Deps', 'Git', - 'Svn', 'Fossil', 'Bzr', 'Bundler', 'TGenshi', - 'Base'] + __plugin_blacklist__ = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', + 'Packages', 'Rules', 'Account', 'Decisions', + 'Deps', 'Git', 'Svn', 'Fossil', 'Bzr', 'Bundler', + 'TGenshi', 'Base'] def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) # First get options to the 'viz' subcommand try: opts, args = getopt.getopt(args, 'Hbkc:o:', ['includehosts', 'includebundles', - 'includekey', 'only-client=', 'outfile=']) + 'includekey', 'only-client=', + 'outfile=']) except getopt.GetoptError: - msg = sys.exc_info()[1] - print(msg) - print(self.__longhelp__) - raise SystemExit(1) + self.usage() hset = False bset = False @@ -67,21 +61,19 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): elif opt in ("-o", "--outfile"): outputfile = arg - data = self.Visualize(self.setup['repo'], hset, bset, - kset, only_client, outputfile) + data = self.Visualize(hset, bset, kset, only_client, outputfile) if data: print(data) - raise SystemExit(0) - def Visualize(self, repopath, hosts=False, - bundles=False, key=False, only_client=None, output=False): + def Visualize(self, hosts=False, bundles=False, key=False, + only_client=None, output=None): """Build visualization of groups file.""" if output: - format = output.split('.')[-1] + fmt = output.split('.')[-1] else: - format = 'png' + fmt = 'png' - cmd = ["dot", "-T", format] + cmd = ["dot", "-T", fmt] if output: cmd.extend(["-o", output]) try: @@ -91,7 +83,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): # shell=True. on others (Gentoo with Python 2.7), you # must. In yet others (RHEL 5), either way works. I have # no idea what the difference is, but it's kind of a PITA. - cmd = ["dot", "-T", pipes.quote(format)] + cmd = ["dot", "-T", pipes.quote(fmt)] if output: cmd.extend(["-o", pipes.quote(output)]) dotpipe = Popen(cmd, shell=True, @@ -106,13 +98,14 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): key, only_client, self.colors)) if key: dotpipe.stdin.write("\tsubgraph cluster_key {\n") - dotpipe.stdin.write('''\tstyle="filled";\n''') - dotpipe.stdin.write('''\tcolor="lightblue";\n''') - dotpipe.stdin.write('''\tBundle [ shape="septagon" ];\n''') - dotpipe.stdin.write('''\tGroup [shape="ellipse"];\n''') - dotpipe.stdin.write('''\tProfile [style="bold", shape="ellipse"];\n''') - dotpipe.stdin.write('''\tHblock [label="Host1|Host2|Host3", shape="record"];\n''') - dotpipe.stdin.write('''\tlabel="Key";\n''') + dotpipe.stdin.write('\tstyle="filled";\n') + dotpipe.stdin.write('\tcolor="lightblue";\n') + dotpipe.stdin.write('\tBundle [ shape="septagon" ];\n') + dotpipe.stdin.write('\tGroup [shape="ellipse"];\n') + dotpipe.stdin.write('\tProfile [style="bold", shape="ellipse"];\n') + dotpipe.stdin.write('\tHblock [label="Host1|Host2|Host3", ' + 'shape="record"];\n') + dotpipe.stdin.write('\tlabel="Key";\n') dotpipe.stdin.write("\t}\n") dotpipe.stdin.write("}\n") dotpipe.stdin.close() diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index ca07f50aa..79eeebc7c 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -1,17 +1,15 @@ -import sys +""" XML-RPC Command Interface for bcfg2-admin""" +import sys import Bcfg2.Options import Bcfg2.Proxy import Bcfg2.Server.Admin - -# Compatibility import from Bcfg2.Compat import xmlrpclib class Xcmd(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("XML-RPC Command Interface") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin xcmd command\n") - __usage__ = ("bcfg2-admin xcmd ") + """ XML-RPC Command Interface """ + __usage__ = "" def __call__(self, args): optinfo = { diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index 6a6e54cb2..df9a45cd5 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -19,10 +19,10 @@ __all__ = [ 'Xcmd' ] +import re +import sys import logging import lxml.etree -import sys - import Bcfg2.Server.Core import Bcfg2.Options from Bcfg2.Compat import ConfigParser @@ -32,8 +32,6 @@ class Mode(object): """ Base object for admin modes. Docstrings are used as help messages, so if you are seeing this, a help message has not yet been added for this mode. """ - __shorthelp__ = 'Shorthelp not defined yet' - __longhelp__ = 'Longhelp not defined yet' __usage__ = None __args__ = [] @@ -42,8 +40,10 @@ class Mode(object): self.configfile = setup['configfile'] self.__cfp = False self.log = logging.getLogger('Bcfg2.Server.Admin.Mode') + usage = "bcfg2-admin %s" % self.__class__.__name__.lower() if self.__usage__ is not None: - setup.hm = self.__usage__ + usage += " " + self.__usage__ + setup.hm = usage def getCFP(self): """ get a config parser for the Bcfg2 config file """ @@ -57,6 +57,22 @@ class Mode(object): def __call__(self, args): raise NotImplementedError + @classmethod + def usage(cls, rv=1): + """ Exit with a long usage message """ + print(re.sub(r'\s{2,}', ' ', cls.__doc__.strip())) + print("") + print("Usage:") + usage = "bcfg2-admin %s" % cls.__name__.lower() + if cls.__usage__ is not None: + usage += " " + cls.__usage__ + print(" %s" % usage) + raise SystemExit(rv) + + def shutdown(self): + """ Perform any necessary shtudown tasks for this mode """ + pass + def errExit(self, emsg): """ exit with an error """ print(emsg) @@ -132,6 +148,10 @@ class MetadataCore(Mode): self.bcore.fam.handle_event_set() self.metadata = self.bcore.metadata + def shutdown(self): + if hasattr(self, 'bcore'): + self.bcore.shutdown() + class StructureMode(MetadataCore): # pylint: disable=W0223 """ Base class for admin modes that handle structure plugins """ diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 606f34ea3..932c35545 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1070,7 +1070,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ return a list of names of clients in the given profile groups """ rv = [] for client in list(self.clients): - mdata = self.get_initial_metadata(client) + mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) return rv @@ -1226,6 +1226,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def viz(self, hosts, bundles, key, only_client, colors): """Admin mode viz support.""" + clientmeta = None if only_client: clientmeta = self.core.build_metadata(only_client) @@ -1233,9 +1234,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = {'default': 'grey83'} viz_str = [] egroups = groups.findall("Group") + groups.findall('.//Groups/Group') + color = 0 for group in egroups: if not group.get('category') in categories: - categories[group.get('category')] = colors.pop() + categories[group.get('category')] = colors[color] + color = (color + 1) % len(colors) group.set('color', categories[group.get('category')]) if None in categories: del categories[None] @@ -1332,6 +1335,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if include_group(group.get('name')): rv.append('"group-%s" -> "group-%s";' % (group.get('name'), parent.get('name'))) + return rv class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 47dd923d4..fa4307702 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -1,17 +1,15 @@ #!/usr/bin/env python -"""bcfg2-admin is a script that helps to administrate a Bcfg2 deployment.""" +""" bcfg2-admin is a script that helps to administer a Bcfg2 +deployment. """ +import re import sys import logging import Bcfg2.Logger import Bcfg2.Options import Bcfg2.Server.Admin -# Compatibility import from Bcfg2.Compat import StringIO -log = logging.getLogger('bcfg2-admin') - - def mode_import(modename): """Load Bcfg2.Server.Admin..""" modname = modename.capitalize() @@ -32,10 +30,10 @@ def create_description(): description.write("Available modes are:\n\n") for mode in modes: try: - description.write((" %-15s %s\n" % - (mode, mode_import(mode).__shorthelp__))) + doc = re.sub(r'\s{2,}', ' ', mode_import(mode).__doc__.strip()) except (ImportError, SystemExit): - pass + continue + description.write((" %-15s %s\n" % (mode, doc))) return description.getvalue() @@ -58,6 +56,8 @@ def main(): Bcfg2.Logger.setup_logging('bcfg2-admin', to_syslog=setup['syslog'], level=level) + log = logging.getLogger('bcfg2-admin') + # Provide help if requested or no args were specified if (not setup['args'] or len(setup['args']) < 1 or setup['args'][0] == 'help' or setup['help']): @@ -72,18 +72,19 @@ def main(): if setup['args'][0] in get_modes(): modname = setup['args'][0].capitalize() if len(setup['args']) > 1 and setup['args'][1] == 'help': - print(mode_import(modname).__longhelp__) - raise SystemExit(0) + mode_cls = mode_import(modname) + mode_cls.usage(rv=0) try: mode_cls = mode_import(modname) except ImportError: - e = sys.exc_info()[1] - log.error("Failed to load admin mode %s: %s" % (modname, e)) + err = sys.exc_info()[1] + log.error("Failed to load admin mode %s: %s" % (modname, err)) raise SystemExit(1) mode = mode_cls(setup) - mode(setup['args'][1:]) - if hasattr(mode, 'bcore'): - mode.bcore.shutdown() + try: + mode(setup['args'][1:]) + finally: + mode.shutdown() else: log.error("Error: Unknown mode '%s'\n" % setup['args'][0]) print(create_description()) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 16e1e8ca1..d6d529dec 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -687,6 +687,25 @@ Bcfg2 client itself.""") collection = self.plugins['Packages'].get_collection(metadata) print(collection.sourcelist()) + def do_query(self, args): + """ query {-g group|-p profile|-b bundle} - Query clients """ + arglist = args.split(" ") + if len(arglist) != 2: + print(self._get_usage(self.do_query)) + return + + qtype, qparam = arglist + if qtype == '-p': + res = self.metadata.get_client_names_by_profiles(qparam.split(',')) + elif qtype == '-g': + res = self.metadata.get_client_names_by_groups(qparam.split(',')) + elif qtype == '-b': + res = self.metadata.get_client_names_by_bundles(qparam.split(',')) + else: + print(self._get_usage(self.do_query)) + return + print("\n".join(res)) + def do_profile(self, arg): """ profile - Profile a single bcfg2-info command """ diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index f5a4655ae..d4ab0aa12 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -55,23 +55,11 @@ contingent_checks = { # perform only error checking on the listed files error_checks = { - "sbin": ["bcfg2-build-reports", "bcfg2-admin", "bcfg2-reports"], + "sbin": ["bcfg2-build-reports", "bcfg2-reports"], "lib/Bcfg2": ["Proxy.py", "SSLServer.py", "Reporting"], "lib/Bcfg2/Server": ["Reports", "SchemaUpdater"], - "lib/Bcfg2/Server/Admin": ["Backup.py", - "Bundle.py", - "Client.py", - "Compare.py", - "Minestruct.py", - "Perf.py", - "Pull.py", - "Query.py", - "Reports.py", - "Snapshots.py", - "Syncdb.py", - "Tidy.py", - "Viz.py", - "Xcmd.py"], + "lib/Bcfg2/Server/Admin": ["Compare.py", + "Snapshots.py"], "lib/Bcfg2/Client/Tools": ["launchd.py", "OpenCSW.py", "Blast.py", -- cgit v1.2.3-1-g7c22