From dab1d03d81c538966d03fb9318a4588a9e803b44 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 24 Mar 2012 11:20:07 -0500 Subject: Allow to run directly from a git checkout (#1037) Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Admin/Backup.py | 25 ++ src/lib/Bcfg2/Server/Admin/Bundle.py | 96 ++++++++ src/lib/Bcfg2/Server/Admin/Client.py | 64 ++++++ src/lib/Bcfg2/Server/Admin/Compare.py | 151 +++++++++++++ src/lib/Bcfg2/Server/Admin/Group.py | 63 ++++++ src/lib/Bcfg2/Server/Admin/Init.py | 357 +++++++++++++++++++++++++++++ src/lib/Bcfg2/Server/Admin/Minestruct.py | 65 ++++++ src/lib/Bcfg2/Server/Admin/Perf.py | 37 +++ src/lib/Bcfg2/Server/Admin/Pull.py | 154 +++++++++++++ src/lib/Bcfg2/Server/Admin/Query.py | 72 ++++++ src/lib/Bcfg2/Server/Admin/Reports.py | 376 +++++++++++++++++++++++++++++++ src/lib/Bcfg2/Server/Admin/Snapshots.py | 165 ++++++++++++++ src/lib/Bcfg2/Server/Admin/Tidy.py | 69 ++++++ src/lib/Bcfg2/Server/Admin/Viz.py | 119 ++++++++++ src/lib/Bcfg2/Server/Admin/Xcmd.py | 60 +++++ src/lib/Bcfg2/Server/Admin/__init__.py | 136 +++++++++++ 16 files changed, 2009 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Admin/Backup.py create mode 100644 src/lib/Bcfg2/Server/Admin/Bundle.py create mode 100644 src/lib/Bcfg2/Server/Admin/Client.py create mode 100644 src/lib/Bcfg2/Server/Admin/Compare.py create mode 100644 src/lib/Bcfg2/Server/Admin/Group.py create mode 100644 src/lib/Bcfg2/Server/Admin/Init.py create mode 100644 src/lib/Bcfg2/Server/Admin/Minestruct.py create mode 100644 src/lib/Bcfg2/Server/Admin/Perf.py create mode 100644 src/lib/Bcfg2/Server/Admin/Pull.py create mode 100644 src/lib/Bcfg2/Server/Admin/Query.py create mode 100644 src/lib/Bcfg2/Server/Admin/Reports.py create mode 100644 src/lib/Bcfg2/Server/Admin/Snapshots.py create mode 100644 src/lib/Bcfg2/Server/Admin/Tidy.py create mode 100644 src/lib/Bcfg2/Server/Admin/Viz.py create mode 100644 src/lib/Bcfg2/Server/Admin/Xcmd.py create mode 100644 src/lib/Bcfg2/Server/Admin/__init__.py (limited to 'src/lib/Bcfg2/Server/Admin') diff --git a/src/lib/Bcfg2/Server/Admin/Backup.py b/src/lib/Bcfg2/Server/Admin/Backup.py new file mode 100644 index 000000000..3744abca3 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Backup.py @@ -0,0 +1,25 @@ +import os +import sys +import time +import tarfile +import Bcfg2.Server.Admin +import Bcfg2.Options + + +class Backup(Bcfg2.Server.Admin.MetadataCore): + __shorthelp__ = "Make a backup of the Bcfg2 repository" + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin backup\n") + #"\n\nbcfg2-admin backup restore") + __usage__ = ("bcfg2-admin backup") + + def __call__(self, args): + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + self.datastore = self.setup['repo'] + timestamp = time.strftime('%Y%m%d%H%M%S') + format = 'gz' + mode = 'w:' + format + filename = timestamp + '.tar' + '.' + format + out = tarfile.open(self.datastore + '/' + filename, mode=mode) + out.add(self.datastore, os.path.basename(self.datastore)) + out.close() + print("Archive %s was stored under %s" % (filename, self.datastore)) diff --git a/src/lib/Bcfg2/Server/Admin/Bundle.py b/src/lib/Bcfg2/Server/Admin/Bundle.py new file mode 100644 index 000000000..89c099602 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Bundle.py @@ -0,0 +1,96 @@ +import lxml.etree +import glob +import sys +import re +import Bcfg2.Server.Admin +import Bcfg2.Options +from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError + + +class Bundle(Bcfg2.Server.Admin.MetadataCore): + __shorthelp__ = "Create or delete bundle entries" + # TODO: add/del functions + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin bundle list-xml" + "\nbcfg2-admin bundle list-genshi" + "\nbcfg2-admin bundle show\n") + __usage__ = ("bcfg2-admin bundle [options] [add|del] [group]") + + def __call__(self, args): + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + rg = re.compile(r'([^.]+\.(?:[a-z][a-z\-]+))(?![\w\.])', + re.IGNORECASE | re.DOTALL) + + # Get all bundles out of the Bundle/ directory + repo = self.setup['repo'] + xml_list = glob.glob("%s/Bundler/*.xml" % repo) + genshi_list = glob.glob("%s/Bundler/*.genshi" % repo) + + if len(args) == 0: + self.errExit("No argument specified.\n" + "Please see bcfg2-admin bundle help for usage.") +# if args[0] == 'add': +# try: +# self.metadata.add_bundle(args[1]) +# except MetadataConsistencyError: +# print("Error in adding bundle.") +# raise SystemExit(1) +# elif args[0] in ['delete', 'remove', 'del', 'rm']: +# try: +# self.metadata.remove_bundle(args[1]) +# except MetadataConsistencyError: +# print("Error in deleting bundle.") +# raise SystemExit(1) + # Lists all available xml bundles + elif args[0] in ['list-xml', 'ls-xml']: + bundle_name = [] + for bundle_path in xml_list: + bundle_name.append(rg.search(bundle_path).group(1)) + for bundle in bundle_name: + print(bundle.split('.')[0]) + # Lists all available genshi bundles + elif args[0] in ['list-genshi', 'ls-gen']: + bundle_name = [] + for bundle_path in genshi_list: + bundle_name.append(rg.search(bundle_path).group(1)) + for bundle in bundle_name: + print(bundle.split('.')[0]) + # Shows a list of all available bundles and prints bundle + # details after the user choose one bundle. + # FIXME: Add support for detailed output of genshi bundles + # FIXME: This functionality is almost identical with + # bcfg2-info bundles + elif args[0] in ['show']: + bundle_name = [] + bundle_list = xml_list + genshi_list + for bundle_path in bundle_list: + print "matching %s" % bundle_path + bundle_name.append(rg.search(bundle_path).group(1)) + text = "Available bundles (Number of bundles: %s)" % \ + (len(bundle_list)) + print(text) + print("%s" % (len(text) * "-")) + for i in range(len(bundle_list)): + print("[%i]\t%s" % (i, bundle_name[i])) + try: + lineno = raw_input("Enter the line number of a bundle for details: ") + except NameError: + lineno = input("Enter the line number of a bundle for details: ") + if int(lineno) >= int(len(bundle_list)): + print("No line with this number.") + else: + if '%s/Bundler/%s' % \ + (repo, bundle_name[int(lineno)]) in genshi_list: + print("Detailed output for *.genshi bundles is not supported.") + else: + print('Details for the "%s" bundle:' % \ + (bundle_name[int(lineno)].split('.')[0])) + tree = lxml.etree.parse(bundle_list[int(lineno)]) + #Prints bundle content + #print(lxml.etree.tostring(tree)) + names = ['Action', 'Package', 'Path', 'Service'] + for name in names: + for node in tree.findall("//" + name): + print("%s:\t%s" % (name, node.attrib["name"])) + else: + print("No command specified") + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py new file mode 100644 index 000000000..4d580c54c --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -0,0 +1,64 @@ +import lxml.etree +import Bcfg2.Server.Admin +from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError + + +class Client(Bcfg2.Server.Admin.MetadataCore): + __shorthelp__ = "Create, delete, or modify client entries" + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin client add " + "attr1=val1 attr2=val2" + "\nbcfg2-admin client update " + "attr1=val1 attr2=val2" + "\nbcfg2-admin client list" + "\nbcfg2-admin client del \n") + __usage__ = ("bcfg2-admin client [options] [add|del|update|list] [attr=val]") + + def __call__(self, args): + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + if len(args) == 0: + self.errExit("No argument specified.\n" + "Please see bcfg2-admin client help for usage.") + if args[0] == 'add': + attr_d = {} + for i in args[2:]: + attr, val = i.split('=', 1) + if attr not in ['profile', 'uuid', 'password', + 'location', 'secure', 'address', + 'auth']: + print("Attribute %s unknown" % attr) + raise SystemExit(1) + attr_d[attr] = val + try: + self.metadata.add_client(args[1], attr_d) + except MetadataConsistencyError: + print("Error in adding client") + raise SystemExit(1) + elif args[0] in ['update', 'up']: + attr_d = {} + for i in args[2:]: + attr, val = i.split('=', 1) + if attr not in ['profile', 'uuid', 'password', + 'location', 'secure', 'address', + 'auth']: + print("Attribute %s unknown" % attr) + raise SystemExit(1) + attr_d[attr] = val + try: + self.metadata.update_client(args[1], attr_d) + except MetadataConsistencyError: + print("Error in updating client") + raise SystemExit(1) + elif args[0] in ['delete', 'remove', 'del', 'rm']: + try: + self.metadata.remove_client(args[1]) + except MetadataConsistencyError: + print("Error in deleting client") + raise SystemExit(1) + elif args[0] in ['list', 'ls']: + tree = lxml.etree.parse(self.metadata.data + "/clients.xml") + tree.xinclude() + for node in tree.findall("//Client"): + print(node.attrib["name"]) + else: + print("No command specified") + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py new file mode 100644 index 000000000..050dd69f8 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -0,0 +1,151 @@ +import lxml.etree +import os + +import Bcfg2.Server.Admin + + +class Compare(Bcfg2.Server.Admin.Mode): + __shorthelp__ = ("Determine differences between files or " + "directories of client specification instances") + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin compare " + "\nbcfg2-admin compare -r ") + __usage__ = ("bcfg2-admin compare \n\n" + " -r\trecursive") + + def __init__(self, setup): + Bcfg2.Server.Admin.Mode.__init__(self, setup) + self.important = {'Path': ['name', 'type', 'owner', 'group', 'perms', + 'important', 'paranoid', 'sensitive', + 'dev_type', 'major', 'minor', 'prune', + 'encoding', 'empty', 'to', 'recursive', + 'vcstype', 'sourceurl', 'revision'], + 'Package': ['name', 'type', 'version', 'simplefile', + 'verify'], + 'Service': ['name', 'type', 'status', 'mode', + 'target', 'sequence', 'parameters'], + 'Action': ['name', 'timing', 'when', 'status', + 'command'], + 'PostInstall': ['name'] + } + + def compareStructures(self, new, old): + if new.tag == 'Independent': + bundle = 'Base' + else: + bundle = new.get('name') + + identical = True + + for child in new.getchildren(): + if child.tag not in self.important: + print(" %s in (new) bundle %s:\n tag type not handled!" % + (child.tag, bundle)) + continue + equiv = old.xpath('%s[@name="%s"]' % + (child.tag, child.get('name'))) + if len(equiv) == 0: + print(" %s %s in bundle %s:\n only in new configuration" % + (child.tag, child.get('name'), bundle)) + identical = False + continue + diff = [] + if child.tag == 'Path' and child.get('type') == 'file' and \ + child.text != equiv[0].text: + diff.append('contents') + attrdiff = [field for field in self.important[child.tag] if \ + child.get(field) != equiv[0].get(field)] + if attrdiff: + diff.append('attributes (%s)' % ', '.join(attrdiff)) + if diff: + print(" %s %s in bundle %s:\n %s differ" % (child.tag, \ + child.get('name'), bundle, ' and '.join(diff))) + identical = False + + for child in old.getchildren(): + if child.tag not in self.important: + print(" %s in (old) bundle %s:\n tag type not handled!" % + (child.tag, bundle)) + elif len(new.xpath('%s[@name="%s"]' % + (child.tag, child.get('name')))) == 0: + print(" %s %s in bundle %s:\n only in old configuration" % + (child.tag, child.get('name'), bundle)) + identical = False + + return identical + + def compareSpecifications(self, path1, path2): + try: + new = lxml.etree.parse(path1).getroot() + except IOError: + print("Failed to read %s" % (path1)) + raise SystemExit(1) + + try: + old = lxml.etree.parse(path2).getroot() + except IOError: + print("Failed to read %s" % (path2)) + raise SystemExit(1) + + for src in [new, old]: + for bundle in src.findall('./Bundle'): + if bundle.get('name')[-4:] == '.xml': + bundle.set('name', bundle.get('name')[:-4]) + + identical = True + + for bundle in old.findall('./Bundle'): + if len(new.xpath('Bundle[@name="%s"]' % (bundle.get('name')))) == 0: + print(" Bundle %s only in old configuration" % + bundle.get('name')) + identical = False + for bundle in new.findall('./Bundle'): + equiv = old.xpath('Bundle[@name="%s"]' % (bundle.get('name'))) + if len(equiv) == 0: + print(" Bundle %s only in new configuration" % + bundle.get('name')) + identical = False + elif not self.compareStructures(bundle, equiv[0]): + identical = False + + i1 = lxml.etree.Element('Independent') + i2 = lxml.etree.Element('Independent') + i1.extend(new.findall('./Independent/*')) + i2.extend(old.findall('./Independent/*')) + if not self.compareStructures(i1, i2): + identical = False + + return identical + + def __call__(self, args): + Bcfg2.Server.Admin.Mode.__call__(self, args) + if len(args) == 0: + self.errExit("No argument specified.\n" + "Please see bcfg2-admin compare help for usage.") + if '-r' in args: + args = list(args) + args.remove('-r') + (oldd, newd) = args + (old, new) = [os.listdir(spot) for spot in args] + old_extra = [] + for item in old: + if item not in new: + old_extra.append(item) + continue + print("File: %s" % item) + state = self.__call__([oldd + '/' + item, newd + '/' + item]) + new.remove(item) + if state: + print("File %s is good" % item) + else: + print("File %s is bad" % item) + if new: + print("%s has extra files: %s" % (newd, ', '.join(new))) + if old_extra: + print("%s has extra files: %s" % (oldd, ', '.join(old_extra))) + return + try: + (old, new) = args + return self.compareSpecifications(new, old) + except IndexError: + print(self.__call__.__doc__) + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Server/Admin/Group.py b/src/lib/Bcfg2/Server/Admin/Group.py new file mode 100644 index 000000000..16a773d6f --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Group.py @@ -0,0 +1,63 @@ +import lxml.etree +import Bcfg2.Server.Admin +from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError + + +class Group(Bcfg2.Server.Admin.MetadataCore): + __shorthelp__ = "Create, delete, or modify group entries" + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin group add " + "attr1=val1 attr2=val2" + "\nbcfg2-admin group update " + "attr1=val1 attr2=val2" + "\nbcfg2-admin group list" + "\nbcfg2-admin group del \n") + __usage__ = ("bcfg2-admin group [options] [add|del|update|list] [attr=val]") + + def __call__(self, args): + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + if len(args) == 0: + self.errExit("No argument specified.\n" + "Please see bcfg2-admin group help for usage.") + if args[0] == 'add': + attr_d = {} + for i in args[2:]: + attr, val = i.split('=', 1) + if attr not in ['profile', 'public', 'default', + 'name', 'auth', 'toolset', 'category', + 'comment']: + print("Attribute %s unknown" % attr) + raise SystemExit(1) + attr_d[attr] = val + try: + self.metadata.add_group(args[1], attr_d) + except MetadataConsistencyError: + print("Error in adding group") + raise SystemExit(1) + elif args[0] in ['update', 'up']: + attr_d = {} + for i in args[2:]: + attr, val = i.split('=', 1) + if attr not in ['profile', 'public', 'default', + 'name', 'auth', 'toolset', 'category', + 'comment']: + print("Attribute %s unknown" % attr) + raise SystemExit(1) + attr_d[attr] = val + try: + self.metadata.update_group(args[1], attr_d) + except MetadataConsistencyError: + print("Error in updating group") + raise SystemExit(1) + elif args[0] in ['delete', 'remove', 'del', 'rm']: + try: + self.metadata.remove_group(args[1]) + except MetadataConsistencyError: + print("Error in deleting group") + raise SystemExit(1) + elif args[0] in ['list', 'ls']: + tree = lxml.etree.parse(self.metadata.data + "/groups.xml") + for node in tree.findall("//Group"): + print(node.attrib["name"]) + else: + print("No command specified") + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py new file mode 100644 index 000000000..832190b7d --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -0,0 +1,357 @@ +import getpass +import os +import random +import socket +import stat +import string +import sys +import subprocess +import Bcfg2.Server.Admin +import Bcfg2.Server.Plugin +import Bcfg2.Options + +# default config file +config = ''' +[server] +repository = %s +plugins = %s + +[statistics] +sendmailpath = %s +database_engine = sqlite3 +# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. +database_name = +# Or path to database file if using sqlite3. +#/etc/brpt.sqlite is default path if left empty +database_user = +# Not used with sqlite3. +database_password = +# Not used with sqlite3. +database_host = +# Not used with sqlite3. +database_port = + +[communication] +protocol = %s +password = %s +certificate = %s +key = %s +ca = %s + +[components] +bcfg2 = %s +''' + +# Default groups +groups = ''' + + + + + + + + + + + + +''' + +# Default contents of clients.xml +clients = ''' + + +''' + +# Mapping of operating system names to groups +os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), + ('SUSE/SLES', 'suse'), + ('Mandrake', 'mandrake'), + ('Debian', 'debian'), + ('Ubuntu', 'ubuntu'), + ('Gentoo', 'gentoo'), + ('FreeBSD', 'freebsd')] + +# Complete list of plugins +plugin_list = ['Account', + 'Base', + 'Bundler', + 'Bzr', + 'Cfg', + 'Decisions', + 'Deps', + 'Git', + 'Guppy', + 'Hg', + 'Metadata', + 'NagiosGen', + 'Ohai', + 'Packages', + 'Pkgmgr', + 'Probes', + 'Properties', + 'Rules', + 'Snapshots', + 'SSHbase', + 'SSLCA', + 'Statistics', + 'Svcmgr', + 'TCheetah', + 'TGenshi'] + +# Default list of plugins to use +default_plugins = Bcfg2.Options.SERVER_PLUGINS.default + + +def get_input(prompt): + """py3k compatible function to get input""" + try: + return raw_input(prompt) + except NameError: + return input(prompt) + + +def gen_password(length): + """Generates a random alphanumeric password with length characters.""" + chars = string.letters + string.digits + newpasswd = '' + for i in range(length): + newpasswd = newpasswd + random.choice(chars) + return newpasswd + + +def create_key(hostname, keypath, certpath, country, state, location): + """Creates a bcfg2.key at the directory specifed by keypath.""" + kcstr = ("openssl req -batch -x509 -nodes -subj '/C=%s/ST=%s/L=%s/CN=%s' " + "-days 1000 -newkey rsa:2048 -keyout %s -noout" % (country, + state, + location, + hostname, + keypath)) + subprocess.call((kcstr), shell=True) + ccstr = ("openssl req -batch -new -subj '/C=%s/ST=%s/L=%s/CN=%s' -key %s " + "| openssl x509 -req -days 1000 -signkey %s -out %s" % (country, + state, + location, + hostname, + keypath, + keypath, + certpath)) + subprocess.call((ccstr), shell=True) + os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 + + +def create_conf(confpath, confdata, keypath): + # Don't overwrite existing bcfg2.conf file + if os.path.exists(confpath): + result = get_input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % confpath) + if result not in ['Y', 'y']: + print("Leaving %s unchanged" % confpath) + return + try: + open(confpath, "w").write(confdata) + os.chmod(confpath, stat.S_IRUSR | stat.S_IWUSR) # 0600 + except Exception: + e = sys.exc_info()[1] + print("Error %s occured while trying to write configuration " + "file to '%s'.\n" % + (e, confpath)) + raise SystemExit(1) + + +class Init(Bcfg2.Server.Admin.Mode): + __shorthelp__ = ("Interactively initialize a new repository.") + __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin init" + __usage__ = "bcfg2-admin init" + options = {'configfile': Bcfg2.Options.CFILE, + 'plugins': Bcfg2.Options.SERVER_PLUGINS, + 'proto': Bcfg2.Options.SERVER_PROTOCOL, + 'repo': Bcfg2.Options.SERVER_REPOSITORY, + 'sendmail': Bcfg2.Options.SENDMAIL_PATH} + repopath = "" + response = "" + + def _set_defaults(self): + """Set default parameters.""" + self.configfile = self.opts['configfile'] + self.repopath = self.opts['repo'] + self.password = gen_password(8) + self.server_uri = "https://%s:6789" % socket.getfqdn() + self.plugins = default_plugins + + def __call__(self, args): + Bcfg2.Server.Admin.Mode.__call__(self, args) + + # Parse options + self.opts = Bcfg2.Options.OptionParser(self.options) + self.opts.parse(args) + self._set_defaults() + + # Prompt the user for input + self._prompt_config() + self._prompt_repopath() + self._prompt_password() + self._prompt_hostname() + self._prompt_server() + self._prompt_groups() + # self._prompt_plugins() + self._prompt_certificate() + + # Initialize the repository + self.init_repo() + + def _prompt_hostname(self): + """Ask for the server hostname.""" + data = get_input("What is the server's hostname [%s]: " % + socket.getfqdn()) + if data != '': + self.shostname = data + else: + self.shostname = socket.getfqdn() + + def _prompt_config(self): + """Ask for the configuration file path.""" + newconfig = get_input("Store Bcfg2 configuration in [%s]: " % + self.configfile) + if newconfig != '': + self.configfile = os.path.abspath(newconfig) + + def _prompt_repopath(self): + """Ask for the repository path.""" + while True: + newrepo = get_input("Location of Bcfg2 repository [%s]: " % + self.repopath) + if newrepo != '': + self.repopath = os.path.abspath(newrepo) + if os.path.isdir(self.repopath): + response = get_input("Directory %s exists. Overwrite? [y/N]:" \ + % self.repopath) + if response.lower().strip() == 'y': + break + else: + break + + def _prompt_password(self): + """Ask for a password or generate one if none is provided.""" + newpassword = getpass.getpass( + "Input password used for communication verification " + "(without echoing; leave blank for a random): ").strip() + if len(newpassword) != 0: + self.password = newpassword + + def _prompt_server(self): + """Ask for the server name.""" + newserver = get_input("Input the server location [%s]: " % + self.server_uri) + if newserver != '': + self.server_uri = newserver + + def _prompt_groups(self): + """Create the groups.xml file.""" + prompt = '''Input base Operating System for clients:\n''' + for entry in os_list: + prompt += "%d: %s\n" % (os_list.index(entry) + 1, entry[0]) + prompt += ': ' + while True: + try: + osidx = int(get_input(prompt)) + self.os_sel = os_list[osidx - 1][1] + break + except ValueError: + continue + + def _prompt_plugins(self): + default = get_input("Use default plugins? (%s) [Y/n]: " % + ''.join(default_plugins)).lower() + if default != 'y' or default != '': + while True: + plugins_are_valid = True + plug_str = get_input("Specify plugins: ") + plugins = plug_str.split(',') + for plugin in plugins: + plugin = plugin.strip() + if not plugin in plugin_list: + plugins_are_valid = False + print("ERROR: Plugin %s not recognized" % plugin) + if plugins_are_valid: + break + + def _prompt_certificate(self): + """Ask for the key details (country, state, and location).""" + print("The following questions affect SSL certificate generation.") + print("If no data is provided, the default values are used.") + newcountry = get_input("Country name (2 letter code) for certificate: ") + if newcountry != '': + if len(newcountry) == 2: + self.country = newcountry + else: + while len(newcountry) != 2: + newcountry = get_input("2 letter country code (eg. US): ") + if len(newcountry) == 2: + self.country = newcountry + break + else: + self.country = 'US' + + newstate = get_input("State or Province Name (full name) for certificate: ") + if newstate != '': + self.state = newstate + else: + self.state = 'Illinois' + + newlocation = get_input("Locality Name (eg, city) for certificate: ") + if newlocation != '': + self.location = newlocation + else: + self.location = 'Argonne' + + def _init_plugins(self): + """Initialize each plugin-specific portion of the repository.""" + for plugin in self.plugins: + if plugin == 'Metadata': + Bcfg2.Server.Plugins.Metadata.Metadata.init_repo(self.repopath, + groups, + self.os_sel, + clients) + else: + try: + module = __import__("Bcfg2.Server.Plugins.%s" % plugin, '', + '', ["Bcfg2.Server.Plugins"]) + cls = getattr(module, plugin) + cls.init_repo(self.repopath) + except Exception: + e = sys.exc_info()[1] + print("Plugin setup for %s failed: %s\n" + "Check that dependencies are installed?" % (plugin, e)) + + def init_repo(self): + """Setup a new repo and create the content of the configuration file.""" + keypath = os.path.dirname(self.configfile) + kpath = os.path.join(keypath, 'bcfg2.key') + cpath = os.path.join(keypath, 'bcfg2.crt') + + confdata = config % (self.repopath, + ','.join(self.plugins), + self.opts['sendmail'], + self.opts['proto'], + self.password, + cpath, + kpath, + cpath, + self.server_uri) + + # Create the configuration file and SSL key + create_conf(self.configfile, confdata, keypath) + create_key(self.shostname, kpath, cpath, self.country, + self.state, self.location) + + # Create the repository + path = os.path.join(self.repopath, 'etc') + try: + os.makedirs(path) + self._init_plugins() + print("Repository created successfuly in %s" % (self.repopath)) + except OSError: + print("Failed to create %s." % path) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py new file mode 100644 index 000000000..b929a9a8c --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -0,0 +1,65 @@ +import getopt +import lxml.etree +import sys + +import Bcfg2.Server.Admin + +class Minestruct(Bcfg2.Server.Admin.StructureMode): + """Pull extra entries out of statistics.""" + __shorthelp__ = "Extract extra entry lists from statistics" + __longhelp__ = (__shorthelp__ + + "\n\nbcfg2-admin minestruct [-f filename] " + "[-g groups] client\n") + __usage__ = ("bcfg2-admin minestruct [options] \n\n" + " %-25s%s\n" + " %-25s%s\n" % + ("-f ", + "build a particular file", + "-g ", + "only build config for groups")) + + def __call__(self, args): + Bcfg2.Server.Admin.Mode.__call__(self, args) + if len(args) == 0: + self.errExit("No argument specified.\n" + "Please see bcfg2-admin minestruct help for usage.") + try: + (opts, args) = getopt.getopt(args, 'f:g:h') + except: + self.log.error(self.__shorthelp__) + raise SystemExit(1) + + client = args[0] + output = sys.stdout + groups = [] + + for (opt, optarg) in opts: + if opt == '-f': + try: + output = open(optarg, 'w') + except IOError: + self.log.error("Failed to open file: %s" % (optarg)) + raise SystemExit(1) + elif opt == '-g': + groups = optarg.split(':') + + try: + extra = set() + for source in self.bcore.pull_sources: + for item in source.GetExtra(client): + extra.add(item) + except: + self.log.error("Failed to find extra entry info for client %s" % + client) + raise SystemExit(1) + root = lxml.etree.Element("Base") + self.log.info("Found %d extra entries" % (len(extra))) + add_point = root + for g in groups: + add_point = lxml.etree.SubElement(add_point, "Group", name=g) + for tag, name in extra: + self.log.info("%s: %s" % (tag, name)) + lxml.etree.SubElement(add_point, tag, name=name) + + tree = lxml.etree.ElementTree(root) + tree.write(output, pretty_print=True) diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py new file mode 100644 index 000000000..411442698 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Perf.py @@ -0,0 +1,37 @@ +import sys + +import Bcfg2.Options +import Bcfg2.Proxy +import Bcfg2.Server.Admin + + +class Perf(Bcfg2.Server.Admin.Mode): + __shorthelp__ = ("Query server for performance data") + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin perf\n") + __usage__ = ("bcfg2-admin perf") + + def __call__(self, args): + output = [('Name', 'Min', 'Max', 'Mean', 'Count')] + optinfo = { + 'ca': Bcfg2.Options.CLIENT_CA, + 'certificate': Bcfg2.Options.CLIENT_CERT, + 'key': Bcfg2.Options.SERVER_KEY, + 'password': Bcfg2.Options.SERVER_PASSWORD, + 'server': Bcfg2.Options.SERVER_LOCATION, + 'user': Bcfg2.Options.CLIENT_USER, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, + } + setup = Bcfg2.Options.OptionParser(optinfo) + setup.parse(sys.argv[1:]) + proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], + setup['user'], + setup['password'], + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca'], + timeout=setup['timeout']) + data = proxy.get_statistics() + for key, value in list(data.items()): + data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]]) + output.append((key, ) + data) + self.print_table(output) diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py new file mode 100644 index 000000000..daf353107 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -0,0 +1,154 @@ +import getopt +import sys + +import Bcfg2.Server.Admin + + +class Pull(Bcfg2.Server.Admin.MetadataCore): + """Pull mode retrieves entries from clients and + integrates the information into the repository. + """ + __shorthelp__ = ("Integrate configuration information " + "from clients into the server repository") + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin pull [-v] [-f][-I] [-s] " + " \n") + __usage__ = ("bcfg2-admin pull [options] " + "\n\n" + " %-25s%s\n" + " %-25s%s\n" + " %-25s%s\n" + " %-25s%s\n" % + ("-v", + "be verbose", + "-f", + "force", + "-I", + "interactive", + "-s", + "stdin")) + allowed = ['Metadata', 'BB', "DBStats", "Statistics", "Cfg", "SSHbase"] + + def __init__(self, setup): + Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) + self.log = False + self.mode = 'interactive' + + def __call__(self, args): + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + use_stdin = False + try: + opts, gargs = getopt.getopt(args, 'vfIs') + except: + print(self.__shorthelp__) + raise SystemExit(1) + for opt in opts: + if opt[0] == '-v': + self.log = True + elif opt[0] == '-f': + self.mode = 'force' + elif opt[0] == '-I': + self.mode == 'interactive' + elif opt[0] == '-s': + use_stdin = True + + if use_stdin: + for line in sys.stdin: + try: + self.PullEntry(*line.split(None, 3)) + except SystemExit: + print(" for %s" % line) + except: + print("Bad entry: %s" % line.strip()) + elif len(gargs) < 3: + print(self.__longhelp__) + raise SystemExit(1) + else: + self.PullEntry(gargs[0], gargs[1], gargs[2]) + + def BuildNewEntry(self, client, etype, ename): + """Construct a new full entry for + given client/entry from statistics. + """ + new_entry = {'type': etype, 'name': ename} + for plugin in self.bcore.pull_sources: + try: + (owner, group, perms, contents) = \ + plugin.GetCurrentEntry(client, etype, ename) + break + except Bcfg2.Server.Plugin.PluginExecutionError: + if plugin == self.bcore.pull_sources[-1]: + print("Pull Source failure; could not fetch current state") + raise SystemExit(1) + + try: + data = {'owner': owner, + 'group': group, + 'perms': perms, + 'text': contents} + except UnboundLocalError: + print("Unable to build entry. " + "Do you have a statistics plugin enabled?") + raise SystemExit(1) + for k, v in list(data.items()): + if v: + new_entry[k] = v + #print new_entry + return new_entry + + def Choose(self, choices): + """Determine where to put pull data.""" + if self.mode == 'interactive': + for choice in choices: + print("Plugin returned choice:") + if id(choice) == id(choices[0]): + print("(current entry) ") + if choice.all: + print(" => global entry") + elif choice.group: + print(" => group entry: %s (prio %d)" % + (choice.group, choice.prio)) + else: + print(" => host entry: %s" % (choice.hostname)) + # py3k compatibility + try: + ans = raw_input("Use this entry? [yN]: ") in ['y', 'Y'] + except NameError: + ans = input("Use this entry? [yN]: ") in ['y', 'Y'] + if ans: + return choice + return False + else: + # mode == 'force' + if not choices: + return False + return choices[0] + + def PullEntry(self, client, etype, ename): + """Make currently recorded client state correct for entry.""" + new_entry = self.BuildNewEntry(client, etype, ename) + + meta = self.bcore.build_metadata(client) + # Find appropriate plugin in bcore + glist = [gen for gen in self.bcore.generators if + ename in gen.Entries.get(etype, {})] + if len(glist) != 1: + self.errExit("Got wrong numbers of matching generators for entry:" \ + + "%s" % ([g.name for g in glist])) + plugin = glist[0] + if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): + self.errExit("Configuration upload not supported by plugin %s" \ + % (plugin.name)) + try: + choices = plugin.AcceptChoices(new_entry, meta) + specific = self.Choose(choices) + if specific: + plugin.AcceptPullData(specific, new_entry, self.log) + except Bcfg2.Server.Plugin.PluginExecutionError: + self.errExit("Configuration upload not supported by plugin %s" \ + % (plugin.name)) + # Commit if running under a VCS + for vcsplugin in list(self.bcore.plugins.values()): + if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): + files = "%s/%s" % (plugin.data, ename) + comment = 'file "%s" pulled from host %s' % (files, client) + vcsplugin.commit_data([files], comment) diff --git a/src/lib/Bcfg2/Server/Admin/Query.py b/src/lib/Bcfg2/Server/Admin/Query.py new file mode 100644 index 000000000..3dd326645 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Query.py @@ -0,0 +1,72 @@ +import sys +import logging +import Bcfg2.Logger +import Bcfg2.Server.Admin + + +class Query(Bcfg2.Server.Admin.MetadataCore): + __shorthelp__ = "Query clients" + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin query [-n] [-c] " + "[-f filename] g=group p=profile") + __usage__ = ("bcfg2-admin query [options] \n\n" + " %-25s%s\n" + " %-25s%s\n" + " %-25s%s\n" % + ("-n", + "query results delimited with newlines", + "-c", + "query results delimited with commas", + "-f filename", + "write query to file")) + + def __init__(self, setup): + Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) + logging.root.setLevel(100) + Bcfg2.Logger.setup_logging(100, to_console=False, to_syslog=False) + + def __call__(self, args): + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + clients = list(self.metadata.clients.keys()) + filename_arg = False + filename = None + for arg in args: + if filename_arg == True: + filename = arg + filename_arg = False + continue + if arg in ['-n', '-c']: + continue + if arg in ['-f']: + filename_arg = True + continue + try: + k, v = arg.split('=') + except: + print("Unknown argument %s" % arg) + continue + if k == 'p': + nc = self.metadata.get_client_names_by_profiles(v.split(',')) + elif k == 'g': + nc = self.metadata.get_client_names_by_groups(v.split(',')) + # add probed groups (if present) + for conn in self.bcore.connectors: + if isinstance(conn, Bcfg2.Server.Plugins.Probes.Probes): + for c, glist in list(conn.cgroups.items()): + for g in glist: + if g in v.split(','): + nc.append(c) + else: + print("One of g= or p= must be specified") + raise SystemExit(1) + clients = [c for c in clients if c in nc] + if '-n' in args: + for client in clients: + print(client) + else: + print(','.join(clients)) + if '-f' in args: + f = open(filename, "w") + for client in clients: + f.write(client + "\n") + f.close() + print("Wrote results to %s" % (filename)) diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py new file mode 100644 index 000000000..974cdff9d --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -0,0 +1,376 @@ +'''Admin interface for dynamic reports''' +import Bcfg2.Logger +import Bcfg2.Server.Admin +import datetime +import os +import logging +import pickle +import platform +import sys +import traceback +from lxml.etree import XML, XMLSyntaxError + +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser + +# FIXME: Remove when server python dep is 2.5 or greater +if sys.version_info >= (2, 5): + from hashlib import md5 +else: + from md5 import md5 + +# Prereq issues can be signaled with ImportError, so no try needed +# FIXME - settings file uses a hardcoded path for /etc/bcfg2.conf +import Bcfg2.Server.Reports.settings + +# Load django and reports stuff _after_ we know we can load settings +import django.core.management +from Bcfg2.Server.Reports.importscript import load_stats +from Bcfg2.Server.Reports.updatefix import update_database +from Bcfg2.Server.Reports.utils import * + +project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) +project_name = os.path.basename(project_directory) +sys.path.append(os.path.join(project_directory, '..')) +project_module = __import__(project_name, '', '', ['']) +sys.path.pop() + +# Set DJANGO_SETTINGS_MODULE appropriately. +os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name +from django.db import connection, transaction + +from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \ + Entries_interactions, Performance, \ + Reason, Ping + + +def printStats(fn): + """ + Print db stats. + + Decorator for purging. Prints database statistics after a run. + """ + def print_stats(self, *data): + start_client = Client.objects.count() + start_i = Interaction.objects.count() + start_ei = Entries_interactions.objects.count() + start_perf = Performance.objects.count() + start_ping = Ping.objects.count() + + fn(self, *data) + + self.log.info("Clients removed: %s" % + (start_client - Client.objects.count())) + self.log.info("Interactions removed: %s" % + (start_i - Interaction.objects.count())) + self.log.info("Interactions->Entries removed: %s" % + (start_ei - Entries_interactions.objects.count())) + self.log.info("Metrics removed: %s" % + (start_perf - Performance.objects.count())) + self.log.info("Ping metrics removed: %s" % + (start_ping - Ping.objects.count())) + + return print_stats + + +class Reports(Bcfg2.Server.Admin.Mode): + '''Admin interface for dynamic reports''' + __shorthelp__ = "Manage dynamic reports" + __longhelp__ = (__shorthelp__) + django_commands = ['syncdb', 'sqlall', 'validate'] + __usage__ = ("bcfg2-admin reports [command] [options]\n" + " -v|--verbose Be verbose\n" + " -q|--quiet Print only errors\n" + "\n" + " Commands:\n" + " init Initialize the database\n" + " load_stats Load statistics data\n" + " -s|--stats Path to statistics.xml file\n" + " -c|--clients-file Path to clients.xml file\n" + " -O3 Fast mode. Duplicates data!\n" + " purge Purge records\n" + " --client [n] Client to operate on\n" + " --days [n] Records older then n days\n" + " --expired Expired clients only\n" + " scrub Scrub the database for duplicate reasons and orphaned entries\n" + " update Apply any updates to the reporting database\n" + "\n" + " Django commands:\n " + "\n ".join(django_commands)) + + def __init__(self, setup): + Bcfg2.Server.Admin.Mode.__init__(self, setup) + self.log.setLevel(logging.INFO) + + def __call__(self, args): + Bcfg2.Server.Admin.Mode.__call__(self, args) + if len(args) == 0 or args[0] == '-h': + print(self.__usage__) + raise SystemExit(0) + + verb = 0 + + if '-v' in args or '--verbose' in args: + self.log.setLevel(logging.DEBUG) + verb = 1 + if '-q' in args or '--quiet' in args: + self.log.setLevel(logging.WARNING) + + # FIXME - dry run + + if args[0] in self.django_commands: + self.django_command_proxy(args[0]) + elif args[0] == 'scrub': + self.scrub() + elif args[0] == 'init': + update_database() + elif args[0] == 'update': + update_database() + elif args[0] == 'load_stats': + quick = '-O3' in args + stats_file = None + clients_file = None + i = 1 + while i < len(args): + if args[i] == '-s' or args[i] == '--stats': + stats_file = args[i + 1] + if stats_file[0] == '-': + self.errExit("Invalid statistics file: %s" % stats_file) + elif args[i] == '-c' or args[i] == '--clients-file': + clients_file = args[i + 1] + if clients_file[0] == '-': + self.errExit("Invalid clients file: %s" % clients_file) + i = i + 1 + self.load_stats(stats_file, clients_file, verb, quick) + elif args[0] == 'purge': + expired = False + client = None + maxdate = None + state = None + i = 1 + while i < len(args): + if args[i] == '-c' or args[i] == '--client': + if client: + self.errExit("Only one client per run") + client = args[i + 1] + print(client) + i = i + 1 + elif args[i] == '--days': + if maxdate: + self.errExit("Max date specified multiple times") + try: + maxdate = datetime.datetime.now() - datetime.timedelta(days=int(args[i + 1])) + except: + self.log.error("Invalid number of days: %s" % args[i + 1]) + raise SystemExit(-1) + i = i + 1 + elif args[i] == '--expired': + expired = True + i = i + 1 + if expired: + if state: + self.log.error("--state is not valid with --expired") + raise SystemExit(-1) + self.purge_expired(maxdate) + else: + self.purge(client, maxdate, state) + else: + print("Unknown command: %s" % args[0]) + + @transaction.commit_on_success + def scrub(self): + ''' Perform a thorough scrub and cleanup of the database ''' + + # Currently only reasons are a problem + try: + start_count = Reason.objects.count() + except Exception: + e = sys.exc_info()[1] + self.log.error("Failed to load reason objects: %s" % e) + return + dup_reasons = [] + + cmp_reasons = dict() + batch_update = [] + for reason in BatchFetch(Reason.objects): + ''' Loop through each reason and create a key out of the data. \ + This lets us take advantage of a fast hash lookup for \ + comparisons ''' + id = reason.id + reason.id = None + key = md5(pickle.dumps(reason)).hexdigest() + reason.id = id + + if key in cmp_reasons: + self.log.debug("Update interactions from %d to %d" \ + % (reason.id, cmp_reasons[key])) + dup_reasons.append([reason.id]) + batch_update.append([cmp_reasons[key], reason.id]) + else: + cmp_reasons[key] = reason.id + self.log.debug("key %d" % reason.id) + + self.log.debug("Done with updates, deleting dupes") + try: + cursor = connection.cursor() + cursor.executemany('update reports_entries_interactions set reason_id=%s where reason_id=%s', batch_update) + cursor.executemany('delete from reports_reason where id = %s', dup_reasons) + transaction.set_dirty() + except Exception: + ex = sys.exc_info()[1] + self.log.error("Failed to delete reasons: %s" % ex) + raise + + self.log.info("Found %d dupes out of %d" % (len(dup_reasons), start_count)) + + # Cleanup orphans + start_count = Reason.objects.count() + Reason.prune_orphans() + self.log.info("Pruned %d Reason records" % (start_count - Reason.objects.count())) + + start_count = Entries.objects.count() + Entries.prune_orphans() + self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count())) + + def django_command_proxy(self, command): + '''Call a django command''' + if command == 'sqlall': + django.core.management.call_command(command, 'reports') + else: + django.core.management.call_command(command) + + def load_stats(self, stats_file=None, clientspath=None, verb=0, quick=False): + '''Load statistics data into the database''' + location = '' + + if not stats_file: + try: + stats_file = "%s/etc/statistics.xml" % self.cfp.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.errExit("Could not read bcfg2.conf; exiting") + try: + statsdata = XML(open(stats_file).read()) + except (IOError, XMLSyntaxError): + self.errExit("StatReports: Failed to parse %s" % (stats_file)) + + try: + encoding = self.cfp.get('components', 'encoding') + except: + encoding = 'UTF-8' + + if not clientspath: + try: + clientspath = "%s/Metadata/clients.xml" % \ + self.cfp.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.errExit("Could not read bcfg2.conf; exiting") + try: + clientsdata = XML(open(clientspath).read()) + except (IOError, XMLSyntaxError): + self.errExit("StatReports: Failed to parse %s" % (clientspath)) + + try: + load_stats(clientsdata, + statsdata, + encoding, + verb, + self.log, + quick=quick, + location=platform.node()) + except: + pass + + @printStats + def purge(self, client=None, maxdate=None, state=None): + '''Purge historical data from the database''' + + filtered = False # indicates whether or not a client should be deleted + + if not client and not maxdate and not state: + self.errExit("Reports.prune: Refusing to prune all data") + + ipurge = Interaction.objects + if client: + try: + cobj = Client.objects.get(name=client) + ipurge = ipurge.filter(client=cobj) + except Client.DoesNotExist: + self.log.error("Client %s not in database" % client) + raise SystemExit(-1) + self.log.debug("Filtering by client: %s" % client) + + if maxdate: + filtered = True + if not isinstance(maxdate, datetime.datetime): + raise TypeError("maxdate is not a DateTime object") + self.log.debug("Filtering by maxdate: %s" % maxdate) + ipurge = ipurge.filter(timestamp__lt=maxdate) + + # Handle ping data as well + ping = Ping.objects.filter(endtime__lt=maxdate) + if client: + ping = ping.filter(client=cobj) + ping.delete() + + if state: + filtered = True + if state not in ('dirty', 'clean', 'modified'): + raise TypeError("state is not one of the following values " + \ + "('dirty','clean','modified')") + self.log.debug("Filtering by state: %s" % state) + ipurge = ipurge.filter(state=state) + + count = ipurge.count() + rnum = 0 + try: + while rnum < count: + grp = list(ipurge[:1000].values("id")) + # just in case... + if not grp: + break + Interaction.objects.filter(id__in=[x['id'] for x in grp]).delete() + rnum += len(grp) + self.log.debug("Deleted %s of %s" % (rnum, count)) + except: + self.log.error("Failed to remove interactions") + (a, b, c) = sys.exc_info() + msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] + del a, b, c + self.log.error(msg) + + # bulk operations bypass the Interaction.delete method + self.log.debug("Pruning orphan Performance objects") + Performance.prune_orphans() + self.log.debug("Pruning orphan Reason objects") + Reason.prune_orphans() + + if client and not filtered: + '''Delete the client, ping data is automatic''' + try: + self.log.debug("Purging client %s" % client) + cobj.delete() + except: + self.log.error("Failed to delete client %s" % client) + (a, b, c) = sys.exc_info() + msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] + del a, b, c + self.log.error(msg) + + @printStats + def purge_expired(self, maxdate=None): + '''Purge expired clients from the database''' + + if maxdate: + if not isinstance(maxdate, datetime.datetime): + raise TypeError("maxdate is not a DateTime object") + self.log.debug("Filtering by maxdate: %s" % maxdate) + clients = Client.objects.filter(expiration__lt=maxdate) + else: + clients = Client.objects.filter(expiration__isnull=False) + + for client in clients: + self.log.debug("Purging client %s" % client) + Interaction.objects.filter(client=client).delete() + client.delete() + self.log.debug("Pruning orphan Performance objects") + Performance.prune_orphans() diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py new file mode 100644 index 000000000..8bc56f1f1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -0,0 +1,165 @@ +from datetime import date +import sys + +# Prereq issues can be signaled with ImportError, so no try needed +import sqlalchemy, sqlalchemy.orm +import Bcfg2.Server.Admin +import Bcfg2.Server.Snapshots +import Bcfg2.Server.Snapshots.model +from Bcfg2.Server.Snapshots.model import Snapshot, Client, Metadata, Base, \ + File, Group, Package, Service +# Compatibility import +from Bcfg2.Bcfg2Py3k import u_str + +class Snapshots(Bcfg2.Server.Admin.Mode): + __shorthelp__ = "Interact with the Snapshots system" + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin snapshots init" + "\nbcfg2-admin query qtype\n") + __usage__ = ("bcfg2-admin snapshots [init|query qtype]") + + q_dispatch = {'client': Client, + 'group': Group, + 'metadata': Metadata, + 'package': Package, + 'snapshot': Snapshot} + + def __init__(self, setup): + Bcfg2.Server.Admin.Mode.__init__(self, setup) + self.session = Bcfg2.Server.Snapshots.setup_session(self.configfile) + self.cfile = self.configfile + + def __call__(self, args): + Bcfg2.Server.Admin.Mode.__call__(self, args) + if len(args) == 0 or args[0] == '-h': + print(self.__usage__) + raise SystemExit(0) + + if args[0] == 'query': + if args[1] in self.q_dispatch: + q_obj = self.q_dispatch[args[1]] + if q_obj == Client: + rows = [] + labels = ('Client', 'Active') + for host in \ + self.session.query(q_obj).filter(q_obj.active == False): + rows.append([host.name, 'No']) + for host in \ + self.session.query(q_obj).filter(q_obj.active == True): + rows.append([host.name, 'Yes']) + self.print_table([labels]+rows, + justify='left', + hdr=True, + vdelim=" ", + padding=1) + elif q_obj == Group: + print("Groups:") + for group in self.session.query(q_obj).all(): + print(" %s" % group.name) + else: + results = self.session.query(q_obj).all() + else: + print('error') + raise SystemExit(1) + elif args[0] == 'init': + # Initialize the Snapshots database + dbpath = Bcfg2.Server.Snapshots.db_from_config(self.cfile) + engine = sqlalchemy.create_engine(dbpath, echo=True) + metadata = Base.metadata + metadata.create_all(engine) + Session = sqlalchemy.orm.sessionmaker() + Session.configure(bind=engine) + session = Session() + session.commit() + elif args[0] == 'dump': + client = args[1] + snap = Snapshot.get_current(self.session, u_str(client)) + if not snap: + print("Current snapshot for %s not found" % client) + sys.exit(1) + print("Client %s last run at %s" % (client, snap.timestamp)) + for pkg in snap.packages: + print("C:", pkg.correct, 'M:', pkg.modified) + print("start", pkg.start.name, pkg.start.version) + print("end", pkg.end.name, pkg.end.version) + elif args[0] == 'reports': + # bcfg2-admin reporting interface for Snapshots + if '-a' in args[1:]: + # Query all hosts for Name, Status, Revision, Timestamp + q = self.session.query(Client.name, + Snapshot.correct, + Snapshot.revision, + Snapshot.timestamp)\ + .filter(Client.id==Snapshot.client_id)\ + .group_by(Client.id) + rows = [] + labels = ('Client', 'Correct', 'Revision', 'Time') + for item in q.all(): + cli, cor, time, rev = item + rows.append([cli, cor, time, rev]) + self.print_table([labels]+rows, + justify='left', + hdr=True, vdelim=" ", + padding=1) + elif '-b' in args[1:]: + # Query a single host for bad entries + if len(args) < 3: + print("Usage: bcfg2-admin snapshots -b ") + return + client = args[2] + snap = Snapshot.get_current(self.session, u_str(client)) + if not snap: + print("Current snapshot for %s not found" % client) + sys.exit(1) + print("Bad entries:") + bad_pkgs = [self.session.query(Package) + .filter(Package.id==p.start_id).one().name \ + for p in snap.packages if p.correct == False] + for p in bad_pkgs: + print(" Package:%s" % p) + bad_files = [self.session.query(File) + .filter(File.id==f.start_id).one().name \ + for f in snap.files if f.correct == False] + for filename in bad_files: + print(" File:%s" % filename) + bad_svcs = [self.session.query(Service) + .filter(Service.id==s.start_id).one().name \ + for s in snap.services if s.correct == False] + for svc in bad_svcs: + print(" Service:%s" % svc) + elif '-e' in args[1:]: + # Query a single host for extra entries + client = args[2] + snap = Snapshot.get_current(self.session, u_str(client)) + if not snap: + print("Current snapshot for %s not found" % client) + sys.exit(1) + print("Extra entries:") + for pkg in snap.extra_packages: + print(" Package:%s" % pkg.name) + # FIXME: Do we know about extra files yet? + for f in snap.extra_files: + print(" File:%s" % f.name) + for svc in snap.extra_services: + print(" Service:%s" % svc.name) + elif '--date' in args[1:]: + year, month, day = args[2:] + timestamp = date(int(year), int(month), int(day)) + snaps = [] + for client in self.session.query(Client).filter(Client.active == True): + snaps.append(Snapshot.get_by_date(self.session, + client.name, + timestamp)) + rows = [] + labels = ('Client', 'Correct', 'Revision', 'Time') + for snap in snaps: + rows.append([snap.client.name, + snap.correct, + snap.revision, + snap.timestamp]) + self.print_table([labels]+rows, + justify='left', + hdr=True, + vdelim=" ", + padding=1) + else: + print("Unknown options: ", args[1:]) diff --git a/src/lib/Bcfg2/Server/Admin/Tidy.py b/src/lib/Bcfg2/Server/Admin/Tidy.py new file mode 100644 index 000000000..82319b93e --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Tidy.py @@ -0,0 +1,69 @@ +import os +import re +import socket + +import Bcfg2.Server.Admin + + +class Tidy(Bcfg2.Server.Admin.Mode): + __shorthelp__ = "Clean up useless files in the repo" + __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin tidy [-f] [-I]\n" + __usage__ = ("bcfg2-admin tidy [options]\n\n" + " %-25s%s\n" + " %-25s%s\n" % + ("-f", + "force", + "-I", + "interactive")) + + def __call__(self, args): + Bcfg2.Server.Admin.Mode.__call__(self, args) + badfiles = self.buildTidyList() + if '-f' in args or '-I' in args: + if '-I' in args: + for name in badfiles[:]: + # py3k compatibility + try: + answer = raw_input("Unlink file %s? [yN] " % name) + except NameError: + answer = input("Unlink file %s? [yN] " % name) + if answer not in ['y', 'Y']: + badfiles.remove(name) + for name in badfiles: + try: + os.unlink(name) + except IOError: + print("Failed to unlink %s" % name) + else: + for name in badfiles: + print(name) + + def buildTidyList(self): + """Clean up unused or unusable files from the repository.""" + hostmatcher = re.compile('.*\.H_(\S+)$') + to_remove = [] + good = [] + bad = [] + + # clean up unresolvable hosts in SSHbase + for name in os.listdir("%s/SSHbase" % self.setup['repo']): + if hostmatcher.match(name): + hostname = hostmatcher.match(name).group(1) + if hostname in good + bad: + continue + try: + socket.gethostbyname(hostname) + good.append(hostname) + except: + bad.append(hostname) + for name in os.listdir("%s/SSHbase" % self.setup['repo']): + if not hostmatcher.match(name): + to_remove.append("%s/SSHbase/%s" % (self.setup['repo'], + name)) + else: + if hostmatcher.match(name).group(1) in bad: + to_remove.append("%s/SSHbase/%s" % + (self.setup['repo'], name)) + # clean up file~ + # clean up files without parsable names in Cfg + return to_remove diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py new file mode 100644 index 000000000..2faa423c1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Viz.py @@ -0,0 +1,119 @@ +import getopt +from subprocess import Popen, PIPE +import sys +import pipes +import Bcfg2.Server.Admin + + +class Viz(Bcfg2.Server.Admin.MetadataCore): + __shorthelp__ = "Produce graphviz diagrams of metadata structures" + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin viz [--includehosts] " + "[--includebundles] [--includekey] " + "[--only-client clientname] " + "[-o output.]\n") + __usage__ = ("bcfg2-admin viz [options]\n\n" + " %-32s%s\n" + " %-32s%s\n" + " %-32s%s\n" + " %-32s%s\n" + " %-32s%s\n" % + ("-H, --includehosts", + "include hosts in the viz output", + "-b, --includebundles", + "include bundles in the viz output", + "-k, --includekey", + "show a key for different digraph shapes", + "-c, --only-client ", + "show only the groups, bundles for the named client", + "-o, --outfile ", + "write viz output to an output file")) + + colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', + 'indianred1', 'limegreen', 'orange1', 'lightblue2', + 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] + + __plugin_blacklist__ = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', 'Packages', + 'Rules', 'Account', 'Decisions', 'Deps', 'Git', + 'Svn', 'Fossil', 'Bzr', 'Bundler', 'TGenshi', + 'SGenshi', 'Base'] + + def __call__(self, args): + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + # First get options to the 'viz' subcommand + try: + opts, args = getopt.getopt(args, 'Hbkc:o:', + ['includehosts', 'includebundles', + 'includekey', 'only-client=', 'outfile=']) + except getopt.GetoptError: + msg = sys.exc_info()[1] + print(msg) + print(self.__longhelp__) + raise SystemExit(1) + + hset = False + bset = False + kset = False + only_client = None + outputfile = False + for opt, arg in opts: + if opt in ("-H", "--includehosts"): + hset = True + elif opt in ("-b", "--includebundles"): + bset = True + elif opt in ("-k", "--includekey"): + kset = True + elif opt in ("-c", "--only-client"): + only_client = arg + elif opt in ("-o", "--outfile"): + outputfile = arg + + data = self.Visualize(self.setup['repo'], hset, bset, + kset, only_client, outputfile) + if data: + print(data) + raise SystemExit(0) + + def Visualize(self, repopath, hosts=False, + bundles=False, key=False, only_client=None, output=False): + """Build visualization of groups file.""" + if output: + format = output.split('.')[-1] + else: + format = 'png' + + cmd = ["dot", "-T", format] + if output: + cmd.extend(["-o", output]) + try: + dotpipe = Popen(cmd, stdin=PIPE, stdout=PIPE, close_fds=True) + except OSError: + # on some systems (RHEL 6), you cannot run dot with + # shell=True. on others (Gentoo with Python 2.7), you + # must. In yet others (RHEL 5), either way works. I have + # no idea what the difference is, but it's kind of a PITA. + cmd = ["dot", "-T", pipes.quote(format)] + if output: + cmd.extend(["-o", pipes.quote(output)]) + dotpipe = Popen(cmd, shell=True, + stdin=PIPE, stdout=PIPE, close_fds=True) + try: + dotpipe.stdin.write("digraph groups {\n") + except: + print("write to dot process failed. Is graphviz installed?") + raise SystemExit(1) + dotpipe.stdin.write('\trankdir="LR";\n') + dotpipe.stdin.write(self.metadata.viz(hosts, bundles, + key, only_client, self.colors)) + if key: + dotpipe.stdin.write("\tsubgraph cluster_key {\n") + dotpipe.stdin.write('''\tstyle="filled";\n''') + dotpipe.stdin.write('''\tcolor="lightblue";\n''') + dotpipe.stdin.write('''\tBundle [ shape="septagon" ];\n''') + dotpipe.stdin.write('''\tGroup [shape="ellipse"];\n''') + dotpipe.stdin.write('''\tProfile [style="bold", shape="ellipse"];\n''') + dotpipe.stdin.write('''\tHblock [label="Host1|Host2|Host3", shape="record"];\n''') + dotpipe.stdin.write('''\tlabel="Key";\n''') + dotpipe.stdin.write("\t}\n") + dotpipe.stdin.write("}\n") + dotpipe.stdin.close() + return dotpipe.stdout.read() diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py new file mode 100644 index 000000000..140465468 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -0,0 +1,60 @@ +import sys + +import Bcfg2.Options +import Bcfg2.Proxy +import Bcfg2.Server.Admin + +# Compatibility import +from Bcfg2.Bcfg2Py3k import xmlrpclib + + +class Xcmd(Bcfg2.Server.Admin.Mode): + __shorthelp__ = ("XML-RPC Command Interface") + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin xcmd command\n") + __usage__ = ("bcfg2-admin xcmd ") + + def __call__(self, args): + optinfo = { + 'server': Bcfg2.Options.SERVER_LOCATION, + 'user': Bcfg2.Options.CLIENT_USER, + 'password': Bcfg2.Options.SERVER_PASSWORD, + 'key': Bcfg2.Options.SERVER_KEY, + 'certificate': Bcfg2.Options.CLIENT_CERT, + 'ca': Bcfg2.Options.CLIENT_CA, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, + } + setup = Bcfg2.Options.OptionParser(optinfo) + setup.parse(args) + Bcfg2.Proxy.RetryMethod.max_retries = 1 + proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], + setup['user'], + setup['password'], + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca'], + timeout=setup['timeout']) + if len(setup['args']) == 0: + print("Usage: xcmd ") + return + cmd = setup['args'][0] + args = () + if len(setup['args']) > 1: + args = tuple(setup['args'][1:]) + try: + data = getattr(proxy, cmd)(*args) + except xmlrpclib.Fault: + flt = sys.exc_info()[1] + if flt.faultCode == 7: + print("Unknown method %s" % cmd) + return + elif flt.faultCode == 20: + return + else: + raise + except Bcfg2.Proxy.ProxyError: + err = sys.exc_info()[1] + print("Proxy Error: %s" % err) + return + + if data != None: + print(data) diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py new file mode 100644 index 000000000..fdb9a0972 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -0,0 +1,136 @@ +__all__ = [ + 'Backup', + 'Bundle', + 'Client', + 'Compare', + 'Group', + 'Init', + 'Minestruct', + 'Perf', + 'Pull', + 'Query', + 'Reports', + 'Snapshots', + 'Tidy', + 'Viz', + 'Xcmd' + ] + +import logging +import lxml.etree +import sys + +import Bcfg2.Server.Core +import Bcfg2.Options +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser + + +class ModeOperationError(Exception): + pass + + +class Mode(object): + """Help message has not yet been added for mode.""" + __shorthelp__ = 'Shorthelp not defined yet' + __longhelp__ = 'Longhelp not defined yet' + __usage__ = None + __args__ = [] + + def __init__(self, setup): + self.setup = setup + self.configfile = setup['configfile'] + self.__cfp = False + self.log = logging.getLogger('Bcfg2.Server.Admin.Mode') + if self.__usage__ is not None: + setup.hm = self.__usage__ + + def getCFP(self): + if not self.__cfp: + self.__cfp = ConfigParser.ConfigParser() + self.__cfp.read(self.configfile) + return self.__cfp + + cfp = property(getCFP) + + def __call__(self, args): + pass + + def errExit(self, emsg): + print(emsg) + raise SystemExit(1) + + def load_stats(self, client): + stats = lxml.etree.parse("%s/etc/statistics.xml" % self.setup['repo']) + hostent = stats.xpath('//Node[@name="%s"]' % client) + if not hostent: + self.errExit("Could not find stats for client %s" % (client)) + return hostent[0] + + def print_table(self, rows, justify='left', hdr=True, vdelim=" ", padding=1): + """Pretty print a table + + rows - list of rows ([[row 1], [row 2], ..., [row n]]) + hdr - if True the first row is treated as a table header + vdelim - vertical delimiter between columns + padding - # of spaces around the longest element in the column + justify - may be left,center,right + + """ + hdelim = "=" + justify = {'left': str.ljust, + 'center': str.center, + 'right': str.rjust}[justify.lower()] + + """ + Calculate column widths (longest item in each column + plus padding on both sides) + + """ + cols = list(zip(*rows)) + colWidths = [max([len(str(item)) + 2 * padding for \ + item in col]) for col in cols] + borderline = vdelim.join([w * hdelim for w in colWidths]) + + # Print out the table + print(borderline) + for row in rows: + print(vdelim.join([justify(str(item), width) for \ + (item, width) in zip(row, colWidths)])) + if hdr: + print(borderline) + hdr = False + + +class MetadataCore(Mode): + """Base class for admin-modes that handle metadata.""" + __plugin_whitelist__ = None + __plugin_blacklist__ = None + + def __init__(self, setup): + Mode.__init__(self, setup) + if self.__plugin_whitelist__ is not None: + setup['plugins'] = [p for p in setup['plugins'] + if p in self.__plugin_whitelist__] + elif self.__plugin_blacklist__ is not None: + setup['plugins'] = [p for p in setup['plugins'] + if p not in self.__plugin_blacklist__] + + try: + self.bcore = \ + Bcfg2.Server.Core.Core(setup['repo'], + setup['plugins'], + setup['password'], + setup['encoding'], + filemonitor=setup['filemonitor']) + if setup['event debug']: + self.bcore.fam.debug = True + except Bcfg2.Server.Core.CoreInitError: + msg = sys.exc_info()[1] + self.errExit("Core load failed: %s" % msg) + self.bcore.fam.handle_events_in_interval(5) + self.metadata = self.bcore.metadata + + +class StructureMode(MetadataCore): + pass -- cgit v1.2.3-1-g7c22