diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
79 files changed, 4814 insertions, 2902 deletions
diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py new file mode 100644 index 000000000..7c2241f58 --- /dev/null +++ b/src/lib/Bcfg2/Server/Admin.py @@ -0,0 +1,1155 @@ +""" Subcommands and helpers for bcfg2-admin """ + +import os +import sys +import time +import glob +import stat +import random +import socket +import string +import getpass +import difflib +import tarfile +import argparse +import lxml.etree +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Client.Proxy +from Bcfg2.Server.Plugin import PullSource, Generator, MetadataConsistencyError +from Bcfg2.Utils import hostnames2ranges, Executor, safe_input +import Bcfg2.Server.Plugins.Metadata + +try: + import Bcfg2.settings + os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.settings' + from django.core.exceptions import ImproperlyConfigured + from django.core import management + import Bcfg2.Server.models + + HAS_DJANGO = True + try: + import south # pylint: disable=W0611 + HAS_REPORTS = True + except ImportError: + HAS_REPORTS = False +except ImportError: + HAS_DJANGO = False + HAS_REPORTS = False + + +class ccolors: + # pylint: disable=W1401 + ADDED = '\033[92m' + CHANGED = '\033[93m' + REMOVED = '\033[91m' + ENDC = '\033[0m' + # pylint: enable=W1401 + + @staticmethod + def disable(cls): + cls.ADDED = '' + cls.CHANGED = '' + cls.REMOVED = '' + cls.ENDC = '' + + +def gen_password(length): + """Generates a random alphanumeric password with length characters.""" + chars = string.letters + string.digits + return "".join(random.choice(chars) for i in range(length)) + + +def print_table(rows, justify='left', hdr=True, vdelim=" ", padding=1): + """Pretty print a table + + rows - list of rows ([[row 1], [row 2], ..., [row n]]) + hdr - if True the first row is treated as a table header + vdelim - vertical delimiter between columns + padding - # of spaces around the longest element in the column + justify - may be left,center,right + + """ + hdelim = "=" + justify = {'left': str.ljust, + 'center': str.center, + 'right': str.rjust}[justify.lower()] + + # Calculate column widths (longest item in each column + # plus padding on both sides) + cols = list(zip(*rows)) + col_widths = [max([len(str(item)) + 2 * padding + for item in col]) for col in cols] + borderline = vdelim.join([w * hdelim for w in col_widths]) + + # Print out the table + print(borderline) + for row in rows: + print(vdelim.join([justify(str(item), width) + for (item, width) in zip(row, col_widths)])) + if hdr: + print(borderline) + hdr = False + + +class AdminCmd(Bcfg2.Options.Subcommand): + def setup(self): + """ Perform post-init (post-options parsing), pre-run setup + tasks """ + pass + + def errExit(self, emsg): + """ exit with an error """ + print(emsg) + raise SystemExit(1) + + +class _ServerAdminCmd(AdminCmd): + """Base class for admin modes that run a Bcfg2 server.""" + __plugin_whitelist__ = None + __plugin_blacklist__ = None + + options = AdminCmd.options + Bcfg2.Server.Core.Core.options + + def setup(self): + if self.__plugin_whitelist__ is not None: + Bcfg2.Options.setup.plugins = [ + p for p in Bcfg2.Options.setup.plugins + if p.name in self.__plugin_whitelist__] + elif self.__plugin_blacklist__ is not None: + Bcfg2.Options.setup.plugins = [ + p for p in Bcfg2.Options.setup.plugins + if p.name not in self.__plugin_blacklist__] + + try: + self.core = Bcfg2.Server.Core.Core() + except Bcfg2.Server.Core.CoreInitError: + msg = sys.exc_info()[1] + self.errExit("Core load failed: %s" % msg) + self.core.load_plugins() + self.core.fam.handle_event_set() + self.metadata = self.core.metadata + + def shutdown(self): + self.core.shutdown() + + +class _ProxyAdminCmd(AdminCmd): + """ Base class for admin modes that proxy to a running Bcfg2 server """ + + options = AdminCmd.options + Bcfg2.Client.Proxy.ComponentProxy.options + + def __init__(self): + AdminCmd.__init__(self) + self.proxy = None + + def setup(self): + self.proxy = Bcfg2.Client.Proxy.ComponentProxy() + + +class Backup(AdminCmd): + """ Make a backup of the Bcfg2 repository """ + + options = AdminCmd.options + [Bcfg2.Options.Common.repository] + + def run(self, setup): + timestamp = time.strftime('%Y%m%d%H%M%S') + datastore = setup.repository + fmt = 'gz' + mode = 'w:' + fmt + filename = timestamp + '.tar' + '.' + fmt + out = tarfile.open(os.path.join(datastore, filename), mode=mode) + out.add(datastore, os.path.basename(datastore)) + out.close() + print("Archive %s was stored under %s" % (filename, datastore)) + + +class Client(_ServerAdminCmd): + """ Create, delete, or list client entries """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.PositionalArgument( + "mode", + choices=["add", "del", "list"]), + Bcfg2.Options.PositionalArgument("hostname", nargs='?')] + + __plugin_whitelist__ = ["Metadata"] + + def run(self, setup): + if setup.mode != 'list' and not setup.hostname: + self.parser.error("<hostname> is required in %s mode" % setup.mode) + elif setup.mode == 'list' and setup.hostname: + self.logger.warning("<hostname> is not honored in list mode") + + if setup.mode == 'add': + try: + self.metadata.add_client(setup.hostname) + except MetadataConsistencyError: + err = sys.exc_info()[1] + self.errExit("Error adding client %s: %s" % (setup.hostname, + err)) + elif setup.mode == 'del': + try: + self.metadata.remove_client(setup.hostname) + except MetadataConsistencyError: + err = sys.exc_info()[1] + self.errExit("Error deleting client %s: %s" % (setup.hostname, + err)) + elif setup.mode == 'list': + for client in self.metadata.list_clients(): + print(client) + + +class Compare(AdminCmd): + """ Compare two hosts or two versions of a host specification """ + + help = "Given two XML files (as produced by bcfg2-info build or bcfg2 " + \ + "-qnc) or two directories containing XML files (as produced by " + \ + "bcfg2-info buildall or bcfg2-info builddir), output a detailed, " + \ + "Bcfg2-centric diff." + + options = AdminCmd.options + [ + Bcfg2.Options.Option( + "-d", "--diff-lines", type=int, + help="Show only N lines of a diff"), + Bcfg2.Options.BooleanOption( + "-c", "--color", help="Use colors even if not run from a TTY"), + Bcfg2.Options.BooleanOption( + "-q", "--quiet", + help="Only show that entries differ, not how they differ"), + Bcfg2.Options.PathOption("path1", metavar="<file-or-dir>"), + Bcfg2.Options.PathOption("path2", metavar="<file-or-dir>")] + + changes = dict() + + def removed(self, msg, host): + self.record("%sRemoved: %s%s" % (ccolors.REMOVED, msg, ccolors.ENDC), + host) + + def added(self, msg, host): + self.record("%sAdded: %s%s" % (ccolors.ADDED, msg, ccolors.ENDC), host) + + def changed(self, msg, host): + self.record("%sChanged: %s%s" % (ccolors.CHANGED, msg, ccolors.ENDC), + host) + + def record(self, msg, host): + if msg not in self.changes: + self.changes[msg] = [host] + else: + self.changes[msg].append(host) + + def udiff(self, l1, l2, **kwargs): + """ get a unified diff with control lines stripped """ + lines = None + if "lines" in kwargs: + if kwargs['lines'] is not None: + lines = int(kwargs['lines']) + del kwargs['lines'] + if lines == 0: + return [] + kwargs['n'] = 0 + diff = [] + for line in difflib.unified_diff(l1, l2, **kwargs): + if (line.startswith("--- ") or line.startswith("+++ ") or + line.startswith("@@ ")): + continue + if lines is not None and len(diff) > lines: + diff.append(" ...") + break + if line.startswith("+"): + for l in line.splitlines(): + diff.append(" %s%s%s" % (ccolors.ADDED, l, ccolors.ENDC)) + elif line.startswith("-"): + for l in line.splitlines(): + diff.append(" %s%s%s" % (ccolors.REMOVED, l, + ccolors.ENDC)) + return diff + + def _bundletype(self, el): + if el.get("tag") == "Independent": + return "Independent bundle" + else: + return "Bundle" + + def run(self, setup): + if not sys.stdout.isatty() and not setup.color: + ccolors.disable(ccolors) + + files = [] + if os.path.isdir(setup.path1) and os.path.isdir(setup.path1): + for fpath in glob.glob(os.path.join(setup.path1, '*')): + fname = os.path.basename(fpath) + if os.path.exists(os.path.join(setup.path2, fname)): + files.append((os.path.join(setup.path1, fname), + os.path.join(setup.path2, fname))) + else: + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + self.removed(host, '') + for fpath in glob.glob(os.path.join(setup.path2, '*')): + fname = os.path.basename(fpath) + if not os.path.exists(os.path.join(setup.path1, fname)): + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + self.added(host, '') + elif os.path.isfile(setup.path1) and os.path.isfile(setup.path2): + files.append((setup.path1, setup.path2)) + else: + self.errExit("Cannot diff a file and a directory") + + for file1, file2 in files: + host = None + if os.path.basename(file1) == os.path.basename(file2): + fname = os.path.basename(file1) + if fname.endswith(".xml"): + host = fname[0:-4] + else: + host = fname + + xdata1 = lxml.etree.parse(file1).getroot() + xdata2 = lxml.etree.parse(file2).getroot() + + elements1 = dict() + elements2 = dict() + bundles1 = [el.get("name") for el in xdata1.iterchildren()] + bundles2 = [el.get("name") for el in xdata2.iterchildren()] + for el in xdata1.iterchildren(): + if el.get("name") not in bundles2: + self.removed("%s %s" % (self._bundletype(el), + el.get("name")), + host) + for el in xdata2.iterchildren(): + if el.get("name") not in bundles1: + self.added("%s %s" % (self._bundletype(el), + el.get("name")), + host) + + for bname in bundles1: + bundle = xdata1.find("*[@name='%s']" % bname) + for el in bundle.getchildren(): + elements1["%s:%s" % (el.tag, el.get("name"))] = el + for bname in bundles2: + bundle = xdata2.find("*[@name='%s']" % bname) + for el in bundle.getchildren(): + elements2["%s:%s" % (el.tag, el.get("name"))] = el + + for el in elements1.values(): + elid = "%s:%s" % (el.tag, el.get("name")) + if elid not in elements2: + self.removed("Element %s" % elid, host) + else: + el2 = elements2[elid] + if (el.getparent().get("name") != + el2.getparent().get("name")): + self.changed( + "Element %s was in bundle %s, " + "now in bundle %s" % (elid, + el.getparent().get("name"), + el2.getparent().get("name")), + host) + attr1 = sorted(["%s=\"%s\"" % (attr, el.get(attr)) + for attr in el.attrib]) + attr2 = sorted(["%s=\"%s\"" % (attr, el.get(attr)) + for attr in el2.attrib]) + if attr1 != attr2: + err = ["Element %s has different attributes" % elid] + if not setup.quiet: + err.extend(self.udiff(attr1, attr2)) + self.changed("\n".join(err), host) + + if el.text != el2.text: + if el.text is None: + self.changed("Element %s content was added" % elid, + host) + elif el2.text is None: + self.changed("Element %s content was removed" % + elid, host) + else: + err = ["Element %s has different content" % + elid] + if not setup.quiet: + err.extend( + self.udiff(el.text.splitlines(), + el2.text.splitlines(), + lines=setup.diff_lines)) + self.changed("\n".join(err), host) + + for el in elements2.values(): + elid = "%s:%s" % (el.tag, el.get("name")) + if elid not in elements2: + self.removed("Element %s" % elid, host) + + for change, hosts in self.changes.items(): + hlist = [h for h in hosts if h is not None] + if len(files) > 1 and len(hlist): + print("===== %s =====" % + "\n ".join(hostnames2ranges(hlist))) + print(change) + if len(files) > 1 and len(hlist): + print("") + + +class Help(AdminCmd, Bcfg2.Options.HelpCommand): + """ Get help on a specific subcommand """ + def command_registry(self): + return CLI.commands + + +class Init(AdminCmd): + """Interactively initialize a new repository.""" + + options = AdminCmd.options + [ + Bcfg2.Options.Common.repository, Bcfg2.Options.Common.plugins] + + # default config file + config = '''[server] +repository = %s +plugins = %s +# Uncomment the following to listen on all interfaces +#listen_all = true + +[database] +#engine = sqlite3 +# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. +#name = +# Or path to database file if using sqlite3. +#<repository>/etc/bcfg2.sqlite is default path if left empty +#user = +# Not used with sqlite3. +#password = +# Not used with sqlite3. +#host = +# Not used with sqlite3. +#port = + +[reporting] +transport = LocalFilesystem + +[communication] +password = %s +certificate = %s +key = %s +ca = %s + +[components] +bcfg2 = %s +''' + + # Default groups + groups = '''<Groups> + <Group profile='true' public='true' default='true' name='basic'/> +</Groups> +''' + + # Default contents of clients.xml + clients = '''<Clients> + <Client profile="basic" name="%s"/> +</Clients> +''' + + def __init__(self): + AdminCmd.__init__(self) + self.data = dict() + + def _set_defaults(self, setup): + """Set default parameters.""" + self.data['plugins'] = setup.plugins + self.data['configfile'] = setup.config + self.data['repopath'] = setup.repository + self.data['password'] = gen_password(8) + self.data['shostname'] = socket.getfqdn() + self.data['server_uri'] = "https://%s:6789" % self.data['shostname'] + self.data['country'] = 'US' + self.data['state'] = 'Illinois' + self.data['location'] = 'Argonne' + if os.path.exists("/etc/pki/tls"): + self.data['keypath'] = "/etc/pki/tls/private/bcfg2.key" + self.data['certpath'] = "/etc/pki/tls/certs/bcfg2.crt" + elif os.path.exists("/etc/ssl"): + self.data['keypath'] = "/etc/ssl/bcfg2.key" + self.data['certpath'] = "/etc/ssl/bcfg2.crt" + else: + basepath = os.path.dirname(self.data['configfile']) + self.data['keypath'] = os.path.join(basepath, "bcfg2.key") + self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') + + def input_with_default(self, msg, default_name): + val = safe_input("%s [%s]: " % (msg, self.data[default_name])) + if val: + self.data[default_name] = val + + def run(self, setup): + self._set_defaults(setup) + + # Prompt the user for input + self._prompt_server() + self._prompt_config() + self._prompt_repopath() + self._prompt_password() + self._prompt_keypath() + self._prompt_certificate() + + # Initialize the repository + self.init_repo() + + def _prompt_server(self): + """Ask for the server name and URI.""" + self.input_with_default("What is the server's hostname", 'shostname') + # reset default server URI + self.data['server_uri'] = "https://%s:6789" % self.data['shostname'] + self.input_with_default("Server location", 'server_uri') + + def _prompt_config(self): + """Ask for the configuration file path.""" + self.input_with_default("Path to Bcfg2 configuration", 'configfile') + + def _prompt_repopath(self): + """Ask for the repository path.""" + while True: + self.input_with_default("Location of Bcfg2 repository", 'repopath') + if os.path.isdir(self.data['repopath']): + response = safe_input("Directory %s exists. Overwrite? [y/N]:" + % self.data['repopath']) + if response.lower().strip() == 'y': + break + else: + break + + def _prompt_password(self): + """Ask for a password or generate one if none is provided.""" + newpassword = getpass.getpass( + "Input password used for communication verification " + "(without echoing; leave blank for random): ").strip() + if len(newpassword) != 0: + self.data['password'] = newpassword + + def _prompt_certificate(self): + """Ask for the key details (country, state, and location).""" + print("The following questions affect SSL certificate generation.") + print("If no data is provided, the default values are used.") + self.input_with_default("Country code for certificate", 'country') + self.input_with_default("State or Province Name (full name) for " + "certificate", 'state') + self.input_with_default("Locality Name (e.g., city) for certificate", + 'location') + + def _prompt_keypath(self): + """ Ask for the key pair location. Try to use sensible + defaults depending on the OS """ + self.input_with_default("Path where Bcfg2 server private key will be " + "created", 'keypath') + self.input_with_default("Path where Bcfg2 server cert will be created", + 'certpath') + + def _init_plugins(self): + """Initialize each plugin-specific portion of the repository.""" + for plugin in self.data['plugins']: + kwargs = dict() + if issubclass(plugin, Bcfg2.Server.Plugins.Metadata.Metadata): + kwargs.update( + dict(groups_xml=self.groups, + clients_xml=self.clients % self.data['shostname'])) + plugin.init_repo(self.data['repopath'], **kwargs) + + def create_conf(self): + """ create the config file """ + confdata = self.config % ( + self.data['repopath'], + ','.join(p.__name__ for p in self.data['plugins']), + self.data['password'], + self.data['certpath'], + self.data['keypath'], + self.data['certpath'], + self.data['server_uri']) + + # Don't overwrite existing bcfg2.conf file + if os.path.exists(self.data['configfile']): + result = safe_input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % self.data['configfile']) + if result not in ['Y', 'y']: + print("Leaving %s unchanged" % self.data['configfile']) + return + try: + open(self.data['configfile'], "w").write(confdata) + os.chmod(self.data['configfile'], + stat.S_IRUSR | stat.S_IWUSR) # 0600 + except: # pylint: disable=W0702 + self.errExit("Error trying to write configuration file '%s': %s" % + (self.data['configfile'], sys.exc_info()[1])) + + def init_repo(self): + """Setup a new repo and create the content of the + configuration file.""" + # Create the repository + path = os.path.join(self.data['repopath'], 'etc') + try: + os.makedirs(path) + self._init_plugins() + print("Repository created successfuly in %s" % + self.data['repopath']) + except OSError: + print("Failed to create %s." % path) + + # Create the configuration file and SSL key + self.create_conf() + self.create_key() + + def create_key(self): + """Creates a bcfg2.key at the directory specifed by keypath.""" + cmd = Executor(timeout=120) + subject = "/C=%s/ST=%s/L=%s/CN=%s'" % ( + self.data['country'], self.data['state'], self.data['location'], + self.data['shostname']) + key = cmd.run(["openssl", "req", "-batch", "-x509", "-nodes", + "-subj", subject, "-days", "1000", + "-newkey", "rsa:2048", + "-keyout", self.data['keypath'], "-noout"]) + if not key.success: + print("Error generating key: %s" % key.error) + return + os.chmod(self.data['keypath'], stat.S_IRUSR | stat.S_IWUSR) # 0600 + csr = cmd.run(["openssl", "req", "-batch", "-new", "-subj", subject, + "-key", self.data['keypath']]) + if not csr.success: + print("Error generating certificate signing request: %s" % + csr.error) + return + cert = cmd.run(["openssl", "x509", "-req", "-days", "1000", + "-signkey", self.data['keypath'], + "-out", self.data['certpath']], + inputdata=csr.stdout) + if not cert.success: + print("Error signing certificate: %s" % cert.error) + return + + +class Minestruct(_ServerAdminCmd): + """ Extract extra entry lists from statistics """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.PathOption( + "-f", "--outfile", type=argparse.FileType('w'), default=sys.stdout, + help="Write to the given file"), + Bcfg2.Options.Option( + "-g", "--groups", help="Only build config for groups", + type=Bcfg2.Options.Types.colon_list, default=[]), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + extra = set() + for source in self.core.plugins_by_type(PullSource): + for item in source.GetExtra(setup.hostname): + extra.add(item) + except: # pylint: disable=W0702 + self.errExit("Failed to find extra entry info for client %s: %s" % + (setup.hostname, sys.exc_info()[1])) + root = lxml.etree.Element("Base") + self.logger.info("Found %d extra entries" % len(extra)) + add_point = root + for grp in setup.groups: + add_point = lxml.etree.SubElement(add_point, "Group", name=grp) + for tag, name in extra: + self.logger.info("%s: %s" % (tag, name)) + lxml.etree.SubElement(add_point, tag, name=name) + + lxml.etree.ElementTree(root).write(setup.outfile, pretty_print=True) + + +class Perf(_ProxyAdminCmd): + """ Get performance data from server """ + + def run(self, setup): + output = [('Name', 'Min', 'Max', 'Mean', 'Count')] + data = self.proxy.get_statistics() + for key in sorted(data.keys()): + output.append( + (key, ) + + tuple(["%.06f" % item + for item in data[key][:-1]] + [data[key][-1]])) + print_table(output) + + +class Pull(_ServerAdminCmd): + """ Retrieves entries from clients and integrates the information + into the repository """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.Common.interactive, + Bcfg2.Options.BooleanOption( + "-s", "--stdin", + help="Read lists of <hostname> <entrytype> <entryname> from stdin " + "instead of the command line"), + Bcfg2.Options.PositionalArgument("hostname", nargs='?'), + Bcfg2.Options.PositionalArgument("entrytype", nargs='?'), + Bcfg2.Options.PositionalArgument("entryname", nargs='?')] + + def __init__(self): + _ServerAdminCmd.__init__(self) + self.interactive = False + + def setup(self): + if (not Bcfg2.Options.setup.stdin and + not (Bcfg2.Options.setup.hostname and + Bcfg2.Options.setup.entrytype and + Bcfg2.Options.setup.entryname)): + print("You must specify either --stdin or a hostname, entry type, " + "and entry name on the command line.") + self.errExit(self.usage()) + _ServerAdminCmd.setup(self) + + def run(self, setup): + self.interactive = setup.interactive + if setup.stdin: + for line in sys.stdin: + try: + self.PullEntry(*line.split(None, 3)) + except SystemExit: + print(" for %s" % line) + except: + print("Bad entry: %s" % line.strip()) + else: + self.PullEntry(setup.hostname, setup.entrytype, setup.entryname) + + def BuildNewEntry(self, client, etype, ename): + """Construct a new full entry for + given client/entry from statistics. + """ + new_entry = {'type': etype, 'name': ename} + pull_sources = self.core.plugins_by_type(PullSource) + for plugin in pull_sources: + try: + (owner, group, mode, contents) = \ + plugin.GetCurrentEntry(client, etype, ename) + break + except Bcfg2.Server.Plugin.PluginExecutionError: + if plugin == pull_sources[-1]: + self.errExit("Pull Source failure; could not fetch " + "current state") + + try: + data = {'owner': owner, + 'group': group, + 'mode': mode, + 'text': contents} + except UnboundLocalError: + self.errExit("Unable to build entry") + for key, val in list(data.items()): + if val: + new_entry[key] = val + return new_entry + + def Choose(self, choices): + """Determine where to put pull data.""" + if self.interactive: + for choice in choices: + print("Plugin returned choice:") + if id(choice) == id(choices[0]): + print("(current entry) ") + if choice.all: + print(" => global entry") + elif choice.group: + print(" => group entry: %s (prio %d)" % + (choice.group, choice.prio)) + else: + print(" => host entry: %s" % (choice.hostname)) + + # flush input buffer + ans = safe_input("Use this entry? [yN]: ") in ['y', 'Y'] + if ans: + return choice + return False + else: + if not choices: + return False + return choices[0] + + def PullEntry(self, client, etype, ename): + """Make currently recorded client state correct for entry.""" + new_entry = self.BuildNewEntry(client, etype, ename) + + meta = self.core.build_metadata(client) + # Find appropriate plugin in core + glist = [gen for gen in self.core.plugins_by_type(Generator) + if ename in gen.Entries.get(etype, {})] + if len(glist) != 1: + self.errExit("Got wrong numbers of matching generators for entry:" + "%s" % ([g.name for g in glist])) + plugin = glist[0] + if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) + try: + choices = plugin.AcceptChoices(new_entry, meta) + specific = self.Choose(choices) + if specific: + plugin.AcceptPullData(specific, new_entry, self.logger) + except Bcfg2.Server.Plugin.PluginExecutionError: + self.errExit("Configuration upload not supported by plugin %s" % + plugin.name) + + # Commit if running under a VCS + for vcsplugin in list(self.core.plugins.values()): + if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): + files = "%s/%s" % (plugin.data, ename) + comment = 'file "%s" pulled from host %s' % (files, client) + vcsplugin.commit_data([files], comment) + + +class _ReportsCmd(AdminCmd): + def __init__(self): + AdminCmd.__init__(self) + self.reports_entries = () + self.reports_classes = () + + def setup(self): + # this has to be imported after options are parsed, + # because Django finalizes its settings as soon as it's + # loaded, which means that if we import this before + # Bcfg2.settings has been populated, Django gets a null + # configuration, and subsequent updates to Bcfg2.settings + # won't help. + import Bcfg2.Reporting.models + self.reports_entries = (Bcfg2.Reporting.models.Group, + Bcfg2.Reporting.models.Bundle, + Bcfg2.Reporting.models.FailureEntry, + Bcfg2.Reporting.models.ActionEntry, + Bcfg2.Reporting.models.PathEntry, + Bcfg2.Reporting.models.PackageEntry, + Bcfg2.Reporting.models.PathEntry, + Bcfg2.Reporting.models.ServiceEntry) + self.reports_classes = self.reports_entries + ( + Bcfg2.Reporting.models.Client, + Bcfg2.Reporting.models.Interaction, + Bcfg2.Reporting.models.Performance) + + +if HAS_DJANGO: + class _DjangoProxyCmd(AdminCmd): + command = None + args = [] + + def run(self, _): + '''Call a django command''' + if self.command is not None: + command = self.command + else: + command = self.__class__.__name__.lower() + args = [command] + self.args + management.call_command(*args) + + class DBShell(_DjangoProxyCmd): + """ Call the Django 'dbshell' command on the database """ + + class Shell(_DjangoProxyCmd): + """ Call the Django 'shell' command on the database """ + + class ValidateDB(_DjangoProxyCmd): + """ Call the Django 'validate' command on the database """ + command = "validate" + + class Syncdb(AdminCmd): + """ Sync the Django ORM with the configured database """ + + def run(self, setup): + management.setup_environ(Bcfg2.settings) + Bcfg2.Server.models.load_models() + try: + management.call_command("syncdb", interactive=False, + verbosity=setup.verbose + setup.debug) + except ImproperlyConfigured: + err = sys.exc_info()[1] + self.logger.error("Django configuration problem: %s" % err) + raise SystemExit(1) + except: + err = sys.exc_info()[1] + self.logger.error("Database update failed: %s" % err) + raise SystemExit(1) + + +if HAS_REPORTS: + import datetime + + class ScrubReports(_ReportsCmd): + """ Perform a thorough scrub and cleanup of the Reporting + database """ + + def setup(self): + _ReportsCmd.setup(self) + # this has to be imported after options are parsed, + # because Django finalizes its settings as soon as it's + # loaded, which means that if we import this before + # Bcfg2.settings has been populated, Django gets a null + # configuration, and subsequent updates to Bcfg2.settings + # won't help. + from django.db.transaction import commit_on_success + self.run = commit_on_success(self.run) + + def run(self, _): + # Cleanup unused entries + for cls in self.reports_entries: + try: + start_count = cls.objects.count() + cls.prune_orphans() + self.logger.info("Pruned %d %s records" % + (start_count - cls.objects.count(), + cls.__name__)) + except: # pylint: disable=W0702 + print("Failed to prune %s: %s" % + (cls.__name__, sys.exc_info()[1])) + + class InitReports(AdminCmd): + """ Initialize the Reporting database """ + def run(self, setup): + verbose = setup.verbose + setup.debug + try: + management.call_command("syncdb", interactive=False, + verbosity=verbose) + management.call_command("migrate", interactive=False, + verbosity=verbose) + except: # pylint: disable=W0702 + self.errExit("%s failed: %s" % + (self.__class__.__name__.title(), + sys.exc_info()[1])) + + class UpdateReports(InitReports): + """ Apply updates to the reporting database """ + + class ReportsStats(_ReportsCmd): + """ Print Reporting database statistics """ + def run(self, _): + for cls in self.reports_classes: + print("%s has %s records" % (cls.__name__, + cls.objects.count())) + + class PurgeReports(_ReportsCmd): + """ Purge records from the Reporting database """ + + options = AdminCmd.options + [ + Bcfg2.Options.Option("--client", help="Client to operate on"), + Bcfg2.Options.Option("--days", type=int, metavar='N', + help="Records older than N days"), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption("--expired", + help="Expired clients only"), + Bcfg2.Options.Option("--state", help="Purge entries in state", + choices=['dirty', 'clean', 'modified']), + required=False)] + + def run(self, setup): + if setup.days: + maxdate = datetime.datetime.now() - \ + datetime.timedelta(days=setup.days) + else: + maxdate = None + + starts = {} + for cls in self.reports_classes: + starts[cls] = cls.objects.count() + if setup.expired: + self.purge_expired(maxdate) + else: + self.purge(setup.client, maxdate, setup.state) + for cls in self.reports_classes: + self.logger.info("Purged %s %s records" % + (starts[cls] - cls.objects.count(), + cls.__name__)) + + def purge(self, client=None, maxdate=None, state=None): + '''Purge historical data from the database''' + # indicates whether or not a client should be deleted + filtered = False + + if not client and not maxdate and not state: + self.errExit("Refusing to prune all data. Specify an option " + "to %s" % self.__class__.__name__.lower()) + + ipurge = Bcfg2.Reporting.models.Interaction.objects + if client: + try: + cobj = Bcfg2.Reporting.models.Client.objects.get( + name=client) + ipurge = ipurge.filter(client=cobj) + except Bcfg2.Reporting.models.Client.DoesNotExist: + self.errExit("Client %s not in database" % client) + self.logger.debug("Filtering by client: %s" % client) + + if maxdate: + filtered = True + self.logger.debug("Filtering by maxdate: %s" % maxdate) + ipurge = ipurge.filter(timestamp__lt=maxdate) + + if Bcfg2.settings.DATABASES['default']['ENGINE'] == \ + 'django.db.backends.sqlite3': + grp_limit = 100 + else: + grp_limit = 1000 + if state: + filtered = True + self.logger.debug("Filtering by state: %s" % state) + ipurge = ipurge.filter(state=state) + + count = ipurge.count() + rnum = 0 + try: + while rnum < count: + grp = list(ipurge[:grp_limit].values("id")) + # just in case... + if not grp: + break + Bcfg2.Reporting.models.Interaction.objects.filter( + id__in=[x['id'] for x in grp]).delete() + rnum += len(grp) + self.logger.debug("Deleted %s of %s" % (rnum, count)) + except: # pylint: disable=W0702 + self.logger.error("Failed to remove interactions: %s" % + sys.exc_info()[1]) + + # Prune any orphaned ManyToMany relations + for m2m in self.reports_entries: + self.logger.debug("Pruning any orphaned %s objects" % + m2m.__name__) + m2m.prune_orphans() + + if client and not filtered: + # Delete the client, ping data is automatic + try: + self.logger.debug("Purging client %s" % client) + cobj.delete() + except: # pylint: disable=W0702 + self.logger.error("Failed to delete client %s: %s" % + (client, sys.exc_info()[1])) + + def purge_expired(self, maxdate=None): + """ Purge expired clients from the Reporting database """ + + if maxdate: + if not isinstance(maxdate, datetime.datetime): + raise TypeError("maxdate is not a DateTime object") + self.logger.debug("Filtering by maxdate: %s" % maxdate) + clients = Bcfg2.Reporting.models.Client.objects.filter( + expiration__lt=maxdate) + else: + clients = Bcfg2.Reporting.models.Client.objects.filter( + expiration__isnull=False) + + for client in clients: + self.logger.debug("Purging client %s" % client) + Bcfg2.Reporting.models.Interaction.objects.filter( + client=client).delete() + client.delete() + + class ReportsSQLAll(_DjangoProxyCmd): + """ Call the Django 'sqlall' command on the Reporting database """ + args = ["Reporting"] + + +class Viz(_ServerAdminCmd): + """ Produce graphviz diagrams of metadata structures """ + + options = _ServerAdminCmd.options + [ + Bcfg2.Options.BooleanOption( + "-H", "--includehosts", + help="Include hosts in the viz output"), + Bcfg2.Options.BooleanOption( + "-b", "--includebundles", + help="Include bundles in the viz output"), + Bcfg2.Options.BooleanOption( + "-k", "--includekey", + help="Show a key for different digraph shapes"), + Bcfg2.Options.Option( + "-c", "--only-client", metavar="<hostname>", + help="Show only the groups, bundles for the named client"), + Bcfg2.Options.PathOption( + "-o", "--outfile", + help="Write viz output to an output file")] + + colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', + 'indianred1', 'limegreen', 'orange1', 'lightblue2', + 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] + + __plugin_blacklist__ = ['DBStats', 'Cfg', 'Pkgmgr', 'Packages', 'Rules', + 'Decisions', 'Deps', 'Git', 'Svn', 'Fossil', 'Bzr', + 'Bundler'] + + def run(self, setup): + if setup.outfile: + fmt = setup.outfile.split('.')[-1] + else: + fmt = 'png' + + exc = Executor() + cmd = ["dot", "-T", fmt] + if setup.outfile: + cmd.extend(["-o", setup.outfile]) + inputlist = ["digraph groups {", + '\trankdir="LR";', + self.metadata.viz(setup.includehosts, + setup.includebundles, + setup.includekey, + setup.only_client, + self.colors)] + if setup.includekey: + inputlist.extend( + ["\tsubgraph cluster_key {", + '\tstyle="filled";', + '\tcolor="lightblue";', + '\tBundle [ shape="septagon" ];', + '\tGroup [shape="ellipse"];', + '\tProfile [style="bold", shape="ellipse"];', + '\tHblock [label="Host1|Host2|Host3",shape="record"];', + '\tlabel="Key";', + "\t}"]) + inputlist.append("}") + idata = "\n".join(inputlist) + try: + result = exc.run(cmd, inputdata=idata) + except OSError: + # on some systems (RHEL 6), you cannot run dot with + # shell=True. on others (Gentoo with Python 2.7), you + # must. In yet others (RHEL 5), either way works. I have + # no idea what the difference is, but it's kind of a PITA. + result = exc.run(cmd, shell=True, inputdata=idata) + if not result.success: + self.errExit("Error running %s: %s" % (cmd, result.error)) + if not setup.outfile: + print(result.stdout) + + +class Xcmd(_ProxyAdminCmd): + """ XML-RPC Command Interface """ + + options = _ProxyAdminCmd.options + [ + Bcfg2.Options.PositionalArgument("command"), + Bcfg2.Options.PositionalArgument("arguments", nargs='*')] + + def run(self, setup): + try: + data = getattr(self.proxy, setup.command)(*setup.arguments) + except Bcfg2.Client.Proxy.ProxyError: + self.errExit("Proxy Error: %s" % sys.exc_info()[1]) + + if data is not None: + print(data) + + +class CLI(Bcfg2.Options.CommandRegistry): + """ CLI class for bcfg2-admin """ + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=AdminCmd) + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + + def run(self): + self.commands[Bcfg2.Options.setup.subcommand].setup() + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Admin/Backup.py b/src/lib/Bcfg2/Server/Admin/Backup.py deleted file mode 100644 index 0a04df98b..000000000 --- a/src/lib/Bcfg2/Server/Admin/Backup.py +++ /dev/null @@ -1,22 +0,0 @@ -""" Make a backup of the Bcfg2 repository """ - -import os -import time -import tarfile -import Bcfg2.Server.Admin -import Bcfg2.Options - - -class Backup(Bcfg2.Server.Admin.MetadataCore): - """ Make a backup of the Bcfg2 repository """ - - def __call__(self, args): - datastore = self.setup['repo'] - timestamp = time.strftime('%Y%m%d%H%M%S') - fmt = 'gz' - mode = 'w:' + fmt - filename = timestamp + '.tar' + '.' + fmt - out = tarfile.open(os.path.join(datastore, filename), mode=mode) - out.add(datastore, os.path.basename(datastore)) - out.close() - print("Archive %s was stored under %s" % (filename, datastore)) diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py deleted file mode 100644 index 187ccfd71..000000000 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ /dev/null @@ -1,32 +0,0 @@ -""" Create, delete, or list client entries """ - -import sys -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import MetadataConsistencyError - - -class Client(Bcfg2.Server.Admin.MetadataCore): - """ Create, delete, or list client entries """ - __usage__ = "[options] [add|del|list] [attr=val]" - __plugin_whitelist__ = ["Metadata"] - - def __call__(self, args): - if len(args) == 0: - self.errExit("No argument specified.\n" - "Usage: %s" % self.__usage__) - if args[0] == 'add': - try: - self.metadata.add_client(args[1]) - except MetadataConsistencyError: - self.errExit("Error in adding client: %s" % sys.exc_info()[1]) - elif args[0] in ['delete', 'remove', 'del', 'rm']: - try: - self.metadata.remove_client(args[1]) - except MetadataConsistencyError: - self.errExit("Error in deleting client: %s" % - sys.exc_info()[1]) - elif args[0] in ['list', 'ls']: - for client in self.metadata.list_clients(): - print(client) - else: - self.errExit("No command specified") diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py deleted file mode 100644 index 6bb15cafd..000000000 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ /dev/null @@ -1,147 +0,0 @@ -import lxml.etree -import os -import Bcfg2.Server.Admin - - -class Compare(Bcfg2.Server.Admin.Mode): - """ Determine differences between files or directories of client - specification instances """ - __usage__ = ("<old> <new>\n\n" - " -r\trecursive") - - def __init__(self): - Bcfg2.Server.Admin.Mode.__init__(self) - self.important = {'Path': ['name', 'type', 'owner', 'group', 'mode', - 'important', 'paranoid', 'sensitive', - 'dev_type', 'major', 'minor', 'prune', - 'encoding', 'empty', 'to', 'recursive', - 'vcstype', 'sourceurl', 'revision', - 'secontext'], - 'Package': ['name', 'type', 'version', 'simplefile', - 'verify'], - 'Service': ['name', 'type', 'status', 'mode', - 'target', 'sequence', 'parameters'], - 'Action': ['name', 'timing', 'when', 'status', - 'command'] - } - - def compareStructures(self, new, old): - if new.get("name"): - bundle = new.get('name') - else: - bundle = 'Independent' - - identical = True - - for child in new.getchildren(): - if child.tag not in self.important: - print(" %s in (new) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - continue - equiv = old.xpath('%s[@name="%s"]' % - (child.tag, child.get('name'))) - if len(equiv) == 0: - print(" %s %s in bundle %s:\n only in new configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - continue - diff = [] - if child.tag == 'Path' and child.get('type') == 'file' and \ - child.text != equiv[0].text: - diff.append('contents') - attrdiff = [field for field in self.important[child.tag] if \ - child.get(field) != equiv[0].get(field)] - if attrdiff: - diff.append('attributes (%s)' % ', '.join(attrdiff)) - if diff: - print(" %s %s in bundle %s:\n %s differ" % (child.tag, \ - child.get('name'), bundle, ' and '.join(diff))) - identical = False - - for child in old.getchildren(): - if child.tag not in self.important: - print(" %s in (old) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - elif len(new.xpath('%s[@name="%s"]' % - (child.tag, child.get('name')))) == 0: - print(" %s %s in bundle %s:\n only in old configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - - return identical - - def compareSpecifications(self, path1, path2): - try: - new = lxml.etree.parse(path1).getroot() - except IOError: - print("Failed to read %s" % (path1)) - raise SystemExit(1) - - try: - old = lxml.etree.parse(path2).getroot() - except IOError: - print("Failed to read %s" % (path2)) - raise SystemExit(1) - - for src in [new, old]: - for bundle in src.findall('./Bundle'): - if bundle.get('name')[-4:] == '.xml': - bundle.set('name', bundle.get('name')[:-4]) - - identical = True - - for bundle in old.findall('./Bundle'): - if len(new.xpath('Bundle[@name="%s"]' % (bundle.get('name')))) == 0: - print(" Bundle %s only in old configuration" % - bundle.get('name')) - identical = False - for bundle in new.findall('./Bundle'): - equiv = old.xpath('Bundle[@name="%s"]' % (bundle.get('name'))) - if len(equiv) == 0: - print(" Bundle %s only in new configuration" % - bundle.get('name')) - identical = False - elif not self.compareStructures(bundle, equiv[0]): - identical = False - - i1 = lxml.etree.Element('Independent') - i2 = lxml.etree.Element('Independent') - i1.extend(new.findall('./Independent/*')) - i2.extend(old.findall('./Independent/*')) - if not self.compareStructures(i1, i2): - identical = False - - return identical - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin compare help for usage.") - if '-r' in args: - args = list(args) - args.remove('-r') - (oldd, newd) = args - (old, new) = [os.listdir(spot) for spot in args] - old_extra = [] - for item in old: - if item not in new: - old_extra.append(item) - continue - print("File: %s" % item) - state = self.__call__([oldd + '/' + item, newd + '/' + item]) - new.remove(item) - if state: - print("File %s is good" % item) - else: - print("File %s is bad" % item) - if new: - print("%s has extra files: %s" % (newd, ', '.join(new))) - if old_extra: - print("%s has extra files: %s" % (oldd, ', '.join(old_extra))) - return - try: - (old, new) = args - return self.compareSpecifications(new, old) - except IndexError: - self.errExit(self.__call__.__doc__) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py deleted file mode 100644 index ba553c7ef..000000000 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ /dev/null @@ -1,353 +0,0 @@ -""" Interactively initialize a new repository. """ - -import os -import sys -import stat -import select -import random -import socket -import string -import getpass -from Bcfg2.Utils import Executor -import Bcfg2.Server.Admin -import Bcfg2.Server.Plugin -import Bcfg2.Options -import Bcfg2.Server.Plugins.Metadata -from Bcfg2.Compat import input # pylint: disable=W0622 - -# default config file -CONFIG = '''[server] -repository = %s -plugins = %s -# Uncomment the following to listen on all interfaces -#listen_all = true - -[statistics] -sendmailpath = %s -#web_debug = False -#time_zone = - -[database] -#engine = sqlite3 -# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. -#name = -# Or path to database file if using sqlite3. -#<repository>/etc/bcfg2.sqlite is default path if left empty -#user = -# Not used with sqlite3. -#password = -# Not used with sqlite3. -#host = -# Not used with sqlite3. -#port = - -[reporting] -transport = LocalFilesystem - -[communication] -protocol = %s -password = %s -certificate = %s -key = %s -ca = %s - -[components] -bcfg2 = %s -''' - -# Default groups -GROUPS = '''<Groups version='3.0'> - <Group profile='true' public='true' default='true' name='basic'> - <Group name='%s'/> - </Group> - <Group name='ubuntu'/> - <Group name='debian'/> - <Group name='freebsd'/> - <Group name='gentoo'/> - <Group name='redhat'/> - <Group name='suse'/> - <Group name='mandrake'/> - <Group name='solaris'/> - <Group name='arch'/> -</Groups> -''' - -# Default contents of clients.xml -CLIENTS = '''<Clients version="3.0"> - <Client profile="basic" name="%s"/> -</Clients> -''' - -# Mapping of operating system names to groups -OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/CentOS', 'redhat'), - ('SUSE/SLES', 'suse'), - ('Mandrake', 'mandrake'), - ('Debian', 'debian'), - ('Ubuntu', 'ubuntu'), - ('Gentoo', 'gentoo'), - ('FreeBSD', 'freebsd'), - ('Arch', 'arch')] - - -def safe_input(prompt): - """ input() that flushes the input buffer before accepting input """ - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - return input(prompt) - - -def gen_password(length): - """Generates a random alphanumeric password with length characters.""" - chars = string.letters + string.digits - return "".join(random.choice(chars) for i in range(length)) - - -def create_key(hostname, keypath, certpath, country, state, location): - """Creates a bcfg2.key at the directory specifed by keypath.""" - cmd = Executor(timeout=120) - subject = "/C=%s/ST=%s/L=%s/CN=%s'" % (country, state, location, hostname) - key = cmd.run(["openssl", "req", "-batch", "-x509", "-nodes", - "-subj", subject, "-days", "1000", "-newkey", "rsa:2048", - "-keyout", keypath, "-noout"]) - if not key.success: - print("Error generating key: %s" % key.error) - return - os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - csr = cmd.run(["openssl", "req", "-batch", "-new", "-subj", subject, - "-key", keypath]) - if not csr.success: - print("Error generating certificate signing request: %s" % csr.error) - return - cert = cmd.run(["openssl", "x509", "-req", "-days", "1000", - "-signkey", keypath, "-out", certpath], - inputdata=csr.stdout) - if not cert.success: - print("Error signing certificate: %s" % cert.error) - return - - -def create_conf(confpath, confdata): - """ create the config file """ - # Don't overwrite existing bcfg2.conf file - if os.path.exists(confpath): - result = safe_input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) - if result not in ['Y', 'y']: - print("Leaving %s unchanged" % confpath) - return - try: - open(confpath, "w").write(confdata) - os.chmod(confpath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - except Exception: - err = sys.exc_info()[1] - print("Error trying to write configuration file '%s': %s" % - (confpath, err)) - raise SystemExit(1) - - -class Init(Bcfg2.Server.Admin.Mode): - """Interactively initialize a new repository.""" - - def __init__(self): - Bcfg2.Server.Admin.Mode.__init__(self) - self.data = dict() - self.plugins = Bcfg2.Options.SERVER_PLUGINS.default - - def _set_defaults(self, opts): - """Set default parameters.""" - self.data['configfile'] = opts['configfile'] - self.data['repopath'] = opts['repo'] - self.data['password'] = gen_password(8) - self.data['server_uri'] = "https://%s:6789" % socket.getfqdn() - self.data['sendmail'] = opts['sendmail'] - self.data['proto'] = opts['proto'] - if os.path.exists("/etc/pki/tls"): - self.data['keypath'] = "/etc/pki/tls/private/bcfg2.key" - self.data['certpath'] = "/etc/pki/tls/certs/bcfg2.crt" - elif os.path.exists("/etc/ssl"): - self.data['keypath'] = "/etc/ssl/bcfg2.key" - self.data['certpath'] = "/etc/ssl/bcfg2.crt" - else: - basepath = os.path.dirname(self.configfile) - self.data['keypath'] = os.path.join(basepath, "bcfg2.key") - self.data['certpath'] = os.path.join(basepath, 'bcfg2.crt') - - def __call__(self, args): - # Parse options - setup = Bcfg2.Options.get_option_parser() - setup.add_options(dict(configfile=Bcfg2.Options.CFILE, - plugins=Bcfg2.Options.SERVER_PLUGINS, - proto=Bcfg2.Options.SERVER_PROTOCOL, - repo=Bcfg2.Options.SERVER_REPOSITORY, - sendmail=Bcfg2.Options.SENDMAIL_PATH)) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - self._set_defaults(setup) - - # Prompt the user for input - self._prompt_config() - self._prompt_repopath() - self._prompt_password() - self._prompt_hostname() - self._prompt_server() - self._prompt_groups() - self._prompt_keypath() - self._prompt_certificate() - - # Initialize the repository - self.init_repo() - - def _prompt_hostname(self): - """Ask for the server hostname.""" - data = safe_input("What is the server's hostname [%s]: " % - socket.getfqdn()) - if data != '': - self.data['shostname'] = data - else: - self.data['shostname'] = socket.getfqdn() - - def _prompt_config(self): - """Ask for the configuration file path.""" - newconfig = safe_input("Store Bcfg2 configuration in [%s]: " % - self.configfile) - if newconfig != '': - self.data['configfile'] = os.path.abspath(newconfig) - - def _prompt_repopath(self): - """Ask for the repository path.""" - while True: - newrepo = safe_input("Location of Bcfg2 repository [%s]: " % - self.data['repopath']) - if newrepo != '': - self.data['repopath'] = os.path.abspath(newrepo) - if os.path.isdir(self.data['repopath']): - response = safe_input("Directory %s exists. Overwrite? [y/N]:" - % self.data['repopath']) - if response.lower().strip() == 'y': - break - else: - break - - def _prompt_password(self): - """Ask for a password or generate one if none is provided.""" - newpassword = getpass.getpass( - "Input password used for communication verification " - "(without echoing; leave blank for a random): ").strip() - if len(newpassword) != 0: - self.data['password'] = newpassword - - def _prompt_server(self): - """Ask for the server name.""" - newserver = safe_input( - "Input the server location (the server listens on a single " - "interface by default) [%s]: " % self.data['server_uri']) - if newserver != '': - self.data['server_uri'] = newserver - - def _prompt_groups(self): - """Create the groups.xml file.""" - prompt = '''Input base Operating System for clients:\n''' - for entry in OS_LIST: - prompt += "%d: %s\n" % (OS_LIST.index(entry) + 1, entry[0]) - prompt += ': ' - while True: - try: - osidx = int(safe_input(prompt)) - self.data['os_sel'] = OS_LIST[osidx - 1][1] - break - except ValueError: - continue - - def _prompt_certificate(self): - """Ask for the key details (country, state, and location).""" - print("The following questions affect SSL certificate generation.") - print("If no data is provided, the default values are used.") - newcountry = safe_input("Country name (2 letter code) for " - "certificate: ") - if newcountry != '': - if len(newcountry) == 2: - self.data['country'] = newcountry - else: - while len(newcountry) != 2: - newcountry = safe_input("2 letter country code (eg. US): ") - if len(newcountry) == 2: - self.data['country'] = newcountry - break - else: - self.data['country'] = 'US' - - newstate = safe_input("State or Province Name (full name) for " - "certificate: ") - if newstate != '': - self.data['state'] = newstate - else: - self.data['state'] = 'Illinois' - - newlocation = safe_input("Locality Name (eg, city) for certificate: ") - if newlocation != '': - self.data['location'] = newlocation - else: - self.data['location'] = 'Argonne' - - def _prompt_keypath(self): - """ Ask for the key pair location. Try to use sensible - defaults depending on the OS """ - keypath = safe_input("Path where Bcfg2 server private key will be " - "created [%s]: " % self.data['keypath']) - if keypath: - self.data['keypath'] = keypath - certpath = safe_input("Path where Bcfg2 server cert will be created " - "[%s]: " % self.data['certpath']) - if certpath: - self.data['certpath'] = certpath - - def _init_plugins(self): - """Initialize each plugin-specific portion of the repository.""" - for plugin in self.plugins: - if plugin == 'Metadata': - Bcfg2.Server.Plugins.Metadata.Metadata.init_repo( - self.data['repopath'], - groups_xml=GROUPS % self.data['os_sel'], - clients_xml=CLIENTS % socket.getfqdn()) - else: - try: - module = __import__("Bcfg2.Server.Plugins.%s" % plugin, '', - '', ["Bcfg2.Server.Plugins"]) - cls = getattr(module, plugin) - cls.init_repo(self.data['repopath']) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - print("Plugin setup for %s failed: %s\n" - "Check that dependencies are installed" % (plugin, - err)) - - def init_repo(self): - """Setup a new repo and create the content of the - configuration file.""" - # Create the repository - path = os.path.join(self.data['repopath'], 'etc') - try: - os.makedirs(path) - self._init_plugins() - print("Repository created successfuly in %s" % - self.data['repopath']) - except OSError: - print("Failed to create %s." % path) - - confdata = CONFIG % (self.data['repopath'], - ','.join(self.plugins), - self.data['sendmail'], - self.data['proto'], - self.data['password'], - self.data['certpath'], - self.data['keypath'], - self.data['certpath'], - self.data['server_uri']) - - # Create the configuration file and SSL key - create_conf(self.data['configfile'], confdata) - create_key(self.data['shostname'], self.data['keypath'], - self.data['certpath'], self.data['country'], - self.data['state'], self.data['location']) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py deleted file mode 100644 index 37ca74894..000000000 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ /dev/null @@ -1,56 +0,0 @@ -""" Extract extra entry lists from statistics """ -import getopt -import lxml.etree -import sys -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import PullSource - - -class Minestruct(Bcfg2.Server.Admin.StructureMode): - """ Extract extra entry lists from statistics """ - __usage__ = ("[options] <client>\n\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-f <filename>", "build a particular file", - "-g <groups>", "only build config for groups")) - - def __call__(self, args): - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin minestruct help for usage.") - try: - (opts, args) = getopt.getopt(args, 'f:g:h') - except getopt.GetoptError: - self.errExit(self.__doc__) - - client = args[0] - output = sys.stdout - groups = [] - - for (opt, optarg) in opts: - if opt == '-f': - try: - output = open(optarg, 'w') - except IOError: - self.errExit("Failed to open file: %s" % (optarg)) - elif opt == '-g': - groups = optarg.split(':') - - try: - extra = set() - for source in self.bcore.plugins_by_type(PullSource): - for item in source.GetExtra(client): - extra.add(item) - except: # pylint: disable=W0702 - self.errExit("Failed to find extra entry info for client %s" % - client) - root = lxml.etree.Element("Base") - self.log.info("Found %d extra entries" % (len(extra))) - add_point = root - for grp in groups: - add_point = lxml.etree.SubElement(add_point, "Group", name=grp) - for tag, name in extra: - self.log.info("%s: %s" % (tag, name)) - lxml.etree.SubElement(add_point, tag, name=name) - - lxml.etree.ElementTree(root).write(output, pretty_print=True) diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py deleted file mode 100644 index 1a772e6fc..000000000 --- a/src/lib/Bcfg2/Server/Admin/Perf.py +++ /dev/null @@ -1,38 +0,0 @@ -""" Get performance data from server """ - -import sys -import Bcfg2.Options -import Bcfg2.Client.Proxy -import Bcfg2.Server.Admin - - -class Perf(Bcfg2.Server.Admin.Mode): - """ Get performance data from server """ - - def __call__(self, args): - output = [('Name', 'Min', 'Max', 'Mean', 'Count')] - setup = Bcfg2.Options.get_option_parser() - setup.add_options(dict(ca=Bcfg2.Options.CLIENT_CA, - certificate=Bcfg2.Options.CLIENT_CERT, - key=Bcfg2.Options.SERVER_KEY, - password=Bcfg2.Options.SERVER_PASSWORD, - server=Bcfg2.Options.SERVER_LOCATION, - user=Bcfg2.Options.CLIENT_USER, - timeout=Bcfg2.Options.CLIENT_TIMEOUT)) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - data = proxy.get_statistics() - for key in sorted(data.keys()): - output.append( - (key, ) + - tuple(["%.06f" % item - for item in data[key][:-1]] + [data[key][-1]])) - self.print_table(output) diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py deleted file mode 100644 index 8f84cd87d..000000000 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ /dev/null @@ -1,147 +0,0 @@ -""" Retrieves entries from clients and integrates the information into -the repository """ - -import os -import sys -import getopt -import select -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugin import PullSource, Generator -from Bcfg2.Compat import input # pylint: disable=W0622 - - -class Pull(Bcfg2.Server.Admin.MetadataCore): - """ Retrieves entries from clients and integrates the information - into the repository """ - __usage__ = ("[options] <client> <entry type> <entry name>\n\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-v", "be verbose", - "-f", "force", - "-I", "interactive", - "-s", "stdin")) - - def __init__(self): - Bcfg2.Server.Admin.MetadataCore.__init__(self) - self.log = False - self.mode = 'interactive' - - def __call__(self, args): - use_stdin = False - try: - opts, gargs = getopt.getopt(args, 'vfIs') - except getopt.GetoptError: - self.errExit(self.__doc__) - for opt in opts: - if opt[0] == '-v': - self.log = True - elif opt[0] == '-f': - self.mode = 'force' - elif opt[0] == '-I': - self.mode = 'interactive' - elif opt[0] == '-s': - use_stdin = True - - if use_stdin: - for line in sys.stdin: - try: - self.PullEntry(*line.split(None, 3)) - except SystemExit: - print(" for %s" % line) - except: - print("Bad entry: %s" % line.strip()) - elif len(gargs) < 3: - self.usage() - else: - self.PullEntry(gargs[0], gargs[1], gargs[2]) - - def BuildNewEntry(self, client, etype, ename): - """Construct a new full entry for - given client/entry from statistics. - """ - new_entry = {'type': etype, 'name': ename} - pull_sources = self.bcore.plugins_by_type(PullSource) - for plugin in pull_sources: - try: - (owner, group, mode, contents) = \ - plugin.GetCurrentEntry(client, etype, ename) - break - except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == pull_sources[-1]: - print("Pull Source failure; could not fetch current state") - raise SystemExit(1) - - try: - data = {'owner': owner, - 'group': group, - 'mode': mode, - 'text': contents} - except UnboundLocalError: - print("Unable to build entry. " - "Do you have a statistics plugin enabled?") - raise SystemExit(1) - for key, val in list(data.items()): - if val: - new_entry[key] = val - return new_entry - - def Choose(self, choices): - """Determine where to put pull data.""" - if self.mode == 'interactive': - for choice in choices: - print("Plugin returned choice:") - if id(choice) == id(choices[0]): - print("(current entry) ") - if choice.all: - print(" => global entry") - elif choice.group: - print(" => group entry: %s (prio %d)" % - (choice.group, choice.prio)) - else: - print(" => host entry: %s" % (choice.hostname)) - - # flush input buffer - while len(select.select([sys.stdin.fileno()], [], [], - 0.0)[0]) > 0: - os.read(sys.stdin.fileno(), 4096) - ans = input("Use this entry? [yN]: ") in ['y', 'Y'] - if ans: - return choice - return False - else: - # mode == 'force' - if not choices: - return False - return choices[0] - - def PullEntry(self, client, etype, ename): - """Make currently recorded client state correct for entry.""" - new_entry = self.BuildNewEntry(client, etype, ename) - - meta = self.bcore.build_metadata(client) - # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.plugins_by_type(Generator) - if ename in gen.Entries.get(etype, {})] - if len(glist) != 1: - self.errExit("Got wrong numbers of matching generators for entry:" - "%s" % ([g.name for g in glist])) - plugin = glist[0] - if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): - self.errExit("Configuration upload not supported by plugin %s" % - plugin.name) - try: - choices = plugin.AcceptChoices(new_entry, meta) - specific = self.Choose(choices) - if specific: - plugin.AcceptPullData(specific, new_entry, self.log) - except Bcfg2.Server.Plugin.PluginExecutionError: - self.errExit("Configuration upload not supported by plugin %s" % - plugin.name) - # Commit if running under a VCS - for vcsplugin in list(self.bcore.plugins.values()): - if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): - files = "%s/%s" % (plugin.data, ename) - comment = 'file "%s" pulled from host %s' % (files, client) - vcsplugin.commit_data([files], comment) diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py deleted file mode 100644 index d21d66a22..000000000 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ /dev/null @@ -1,262 +0,0 @@ -'''Admin interface for dynamic reports''' -import Bcfg2.Logger -import Bcfg2.Server.Admin -import datetime -import os -import sys -import traceback -from Bcfg2 import settings - -# Load django and reports stuff _after_ we know we can load settings -from django.core import management -from Bcfg2.Reporting.utils import * - -project_directory = os.path.dirname(settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() - -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name -from django.db import transaction - -from Bcfg2.Reporting.models import Client, Interaction, \ - Performance, Bundle, Group, FailureEntry, PathEntry, \ - PackageEntry, ServiceEntry, ActionEntry - - -def printStats(fn): - """ - Print db stats. - - Decorator for purging. Prints database statistics after a run. - """ - def print_stats(self, *data): - classes = (Client, Interaction, Performance, \ - FailureEntry, ActionEntry, PathEntry, PackageEntry, \ - ServiceEntry, Group, Bundle) - - starts = {} - for cls in classes: - starts[cls] = cls.objects.count() - - fn(self, *data) - - for cls in classes: - print("%s removed: %s" % (cls().__class__.__name__, - starts[cls] - cls.objects.count())) - - return print_stats - - -class Reports(Bcfg2.Server.Admin.Mode): - """ Manage dynamic reports """ - django_commands = ['dbshell', 'shell', 'sqlall', 'validate'] - __usage__ = ("[command] [options]\n" - " Commands:\n" - " init Initialize the database\n" - " purge Purge records\n" - " --client [n] Client to operate on\n" - " --days [n] Records older then n days\n" - " --expired Expired clients only\n" - " scrub Scrub the database for duplicate " - "reasons and orphaned entries\n" - " stats print database statistics\n" - " update Apply any updates to the reporting " - "database\n" - "\n" - " Django commands:\n " \ - + "\n ".join(django_commands)) - - def __init__(self): - Bcfg2.Server.Admin.Mode.__init__(self) - try: - import south - except ImportError: - print("Django south is required for Reporting") - raise SystemExit(-3) - - def __call__(self, args): - if len(args) == 0 or args[0] == '-h': - self.errExit(self.__usage__) - - # FIXME - dry run - - if args[0] in self.django_commands: - self.django_command_proxy(args[0]) - elif args[0] == 'scrub': - self.scrub() - elif args[0] == 'stats': - self.stats() - elif args[0] in ['init', 'update', 'syncdb']: - if self.setup['debug']: - vrb = 2 - elif self.setup['verbose']: - vrb = 1 - else: - vrb = 0 - try: - management.call_command("syncdb", verbosity=vrb) - management.call_command("migrate", verbosity=vrb) - except: - self.errExit("Update failed: %s" % sys.exc_info()[1]) - elif args[0] == 'purge': - expired = False - client = None - maxdate = None - state = None - i = 1 - while i < len(args): - if args[i] == '-c' or args[i] == '--client': - if client: - self.errExit("Only one client per run") - client = args[i + 1] - print(client) - i = i + 1 - elif args[i] == '--days': - if maxdate: - self.errExit("Max date specified multiple times") - try: - maxdate = datetime.datetime.now() - \ - datetime.timedelta(days=int(args[i + 1])) - except: - self.errExit("Invalid number of days: %s" % - args[i + 1]) - i = i + 1 - elif args[i] == '--expired': - expired = True - i = i + 1 - if expired: - if state: - self.errExit("--state is not valid with --expired") - self.purge_expired(maxdate) - else: - self.purge(client, maxdate, state) - else: - self.errExit("Unknown command: %s" % args[0]) - - @transaction.commit_on_success - def scrub(self): - ''' Perform a thorough scrub and cleanup of the database ''' - - # Cleanup unused entries - for cls in (Group, Bundle, FailureEntry, ActionEntry, PathEntry, - PackageEntry, PathEntry): - try: - start_count = cls.objects.count() - cls.prune_orphans() - self.log.info("Pruned %d %s records" % \ - (start_count - cls.objects.count(), cls.__class__.__name__)) - except: - print("Failed to prune %s: %s" % - (cls.__class__.__name__, sys.exc_info()[1])) - - def django_command_proxy(self, command): - '''Call a django command''' - if command == 'sqlall': - management.call_command(command, 'Reporting') - else: - management.call_command(command) - - @printStats - def purge(self, client=None, maxdate=None, state=None): - '''Purge historical data from the database''' - - filtered = False # indicates whether or not a client should be deleted - - if not client and not maxdate and not state: - self.errExit("Reports.prune: Refusing to prune all data") - - ipurge = Interaction.objects - if client: - try: - cobj = Client.objects.get(name=client) - ipurge = ipurge.filter(client=cobj) - except Client.DoesNotExist: - self.errExit("Client %s not in database" % client) - self.log.debug("Filtering by client: %s" % client) - - if maxdate: - filtered = True - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - ipurge = ipurge.filter(timestamp__lt=maxdate) - - if settings.DATABASES['default']['ENGINE'] == \ - 'django.db.backends.sqlite3': - grp_limit = 100 - else: - grp_limit = 1000 - if state: - filtered = True - if state not in ('dirty', 'clean', 'modified'): - raise TypeError("state is not one of the following values: " - "dirty, clean, modified") - self.log.debug("Filtering by state: %s" % state) - ipurge = ipurge.filter(state=state) - - count = ipurge.count() - rnum = 0 - try: - while rnum < count: - grp = list(ipurge[:grp_limit].values("id")) - # just in case... - if not grp: - break - Interaction.objects.filter(id__in=[x['id'] - for x in grp]).delete() - rnum += len(grp) - self.log.debug("Deleted %s of %s" % (rnum, count)) - except: - self.log.error("Failed to remove interactions") - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - # Prune any orphaned ManyToMany relations - for m2m in (ActionEntry, PackageEntry, PathEntry, ServiceEntry, \ - FailureEntry, Group, Bundle): - self.log.debug("Pruning any orphaned %s objects" % \ - m2m().__class__.__name__) - m2m.prune_orphans() - - if client and not filtered: - # Delete the client, ping data is automatic - try: - self.log.debug("Purging client %s" % client) - cobj.delete() - except: - self.log.error("Failed to delete client %s" % client) - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - @printStats - def purge_expired(self, maxdate=None): - '''Purge expired clients from the database''' - - if maxdate: - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - clients = Client.objects.filter(expiration__lt=maxdate) - else: - clients = Client.objects.filter(expiration__isnull=False) - - for client in clients: - self.log.debug("Purging client %s" % client) - Interaction.objects.filter(client=client).delete() - client.delete() - - def stats(self): - classes = (Client, Interaction, Performance, \ - FailureEntry, ActionEntry, PathEntry, PackageEntry, \ - ServiceEntry, Group, Bundle) - - for cls in classes: - print("%s has %s records" % (cls().__class__.__name__, - cls.objects.count())) diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py deleted file mode 100644 index 2722364f7..000000000 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import Bcfg2.settings -import Bcfg2.Options -import Bcfg2.Server.Admin -import Bcfg2.Server.models -from django.core.exceptions import ImproperlyConfigured -from django.core.management import setup_environ, call_command - - -class Syncdb(Bcfg2.Server.Admin.Mode): - """ Sync the Django ORM with the configured database """ - - def __call__(self, args): - # Parse options - setup = Bcfg2.Options.get_option_parser() - setup.add_option("web_configfile", Bcfg2.Options.WEB_CFILE) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - - setup_environ(Bcfg2.settings) - Bcfg2.Server.models.load_models(cfile=setup['web_configfile']) - - try: - call_command("syncdb", interactive=False, verbosity=0) - self._database_available = True - except ImproperlyConfigured: - self.errExit("Django configuration problem: %s" % - sys.exc_info()[1]) - except: - self.errExit("Database update failed: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py deleted file mode 100644 index a29fdaceb..000000000 --- a/src/lib/Bcfg2/Server/Admin/Viz.py +++ /dev/null @@ -1,104 +0,0 @@ -""" Produce graphviz diagrams of metadata structures """ - -import getopt -import Bcfg2.Server.Admin -from Bcfg2.Utils import Executor - - -class Viz(Bcfg2.Server.Admin.MetadataCore): - """ Produce graphviz diagrams of metadata structures """ - __usage__ = ("[options]\n\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" % - ("-H, --includehosts", - "include hosts in the viz output", - "-b, --includebundles", - "include bundles in the viz output", - "-k, --includekey", - "show a key for different digraph shapes", - "-c, --only-client <clientname>", - "show only the groups, bundles for the named client", - "-o, --outfile <file>", - "write viz output to an output file")) - - colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', - 'indianred1', 'limegreen', 'orange1', 'lightblue2', - 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] - - __plugin_blacklist__ = ['DBStats', 'Cfg', 'Pkgmgr', - 'Packages', 'Rules', 'Decisions', - 'Deps', 'Git', 'Svn', 'Fossil', 'Bzr', 'Bundler'] - - def __call__(self, args): - # First get options to the 'viz' subcommand - try: - opts, args = getopt.getopt(args, 'Hbkc:o:', - ['includehosts', 'includebundles', - 'includekey', 'only-client=', - 'outfile=']) - except getopt.GetoptError: - self.usage() - - hset = False - bset = False - kset = False - only_client = None - outputfile = False - for opt, arg in opts: - if opt in ("-H", "--includehosts"): - hset = True - elif opt in ("-b", "--includebundles"): - bset = True - elif opt in ("-k", "--includekey"): - kset = True - elif opt in ("-c", "--only-client"): - only_client = arg - elif opt in ("-o", "--outfile"): - outputfile = arg - - data = self.Visualize(hset, bset, kset, only_client, outputfile) - if data: - print(data) - - def Visualize(self, hosts=False, bundles=False, key=False, - only_client=None, output=None): - """Build visualization of groups file.""" - if output: - fmt = output.split('.')[-1] - else: - fmt = 'png' - - exc = Executor() - cmd = ["dot", "-T", fmt] - if output: - cmd.extend(["-o", output]) - idata = ["digraph groups {", - '\trankdir="LR";', - self.metadata.viz(hosts, bundles, - key, only_client, self.colors)] - if key: - idata.extend( - ["\tsubgraph cluster_key {", - '\tstyle="filled";', - '\tcolor="lightblue";', - '\tBundle [ shape="septagon" ];', - '\tGroup [shape="ellipse"];', - '\tProfile [style="bold", shape="ellipse"];', - '\tHblock [label="Host1|Host2|Host3",shape="record"];', - '\tlabel="Key";', - "\t}"]) - idata.append("}") - try: - result = exc.run(cmd, inputdata=idata) - except OSError: - # on some systems (RHEL 6), you cannot run dot with - # shell=True. on others (Gentoo with Python 2.7), you - # must. In yet others (RHEL 5), either way works. I have - # no idea what the difference is, but it's kind of a PITA. - result = exc.run(cmd, shell=True, inputdata=idata) - if not result.success: - print("Error running %s: %s" % (cmd, result.error)) - raise SystemExit(result.retval) diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py deleted file mode 100644 index f7f30fd80..000000000 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ /dev/null @@ -1,54 +0,0 @@ -""" XML-RPC Command Interface for bcfg2-admin""" - -import sys -import Bcfg2.Options -import Bcfg2.Client.Proxy -import Bcfg2.Server.Admin -from Bcfg2.Compat import xmlrpclib - - -class Xcmd(Bcfg2.Server.Admin.Mode): - """ XML-RPC Command Interface """ - __usage__ = "<command>" - - def __call__(self, args): - setup = Bcfg2.Options.get_option_parser() - setup.add_options(dict(ca=Bcfg2.Options.CLIENT_CA, - certificate=Bcfg2.Options.CLIENT_CERT, - key=Bcfg2.Options.SERVER_KEY, - password=Bcfg2.Options.SERVER_PASSWORD, - server=Bcfg2.Options.SERVER_LOCATION, - user=Bcfg2.Options.CLIENT_USER, - timeout=Bcfg2.Options.CLIENT_TIMEOUT)) - opts = sys.argv[1:] - opts.remove(self.__class__.__name__.lower()) - setup.reparse(argv=opts) - Bcfg2.Client.Proxy.RetryMethod.max_retries = 1 - proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - if len(setup['args']) == 0 or len(args) == 0: - self.errExit("Usage: xcmd <xmlrpc method> <optional arguments>") - cmd = args[0] - try: - data = getattr(proxy, cmd)(*args[1:]) - except xmlrpclib.Fault: - flt = sys.exc_info()[1] - if flt.faultCode == 7: - print("Unknown method %s" % cmd) - return - elif flt.faultCode == 20: - return - else: - raise - except Bcfg2.Client.Proxy.ProxyError: - err = sys.exc_info()[1] - print("Proxy Error: %s" % err) - return - - if data is not None: - print(data) diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py deleted file mode 100644 index 06a419354..000000000 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ /dev/null @@ -1,142 +0,0 @@ -""" Base classes for admin modes """ - -import re -import sys -import logging -import lxml.etree -import Bcfg2.Server.Core -import Bcfg2.Options -from Bcfg2.Compat import ConfigParser, walk_packages - -__all__ = [m[1] for m in walk_packages(path=__path__)] - - -class Mode(object): - """ Base object for admin modes. Docstrings are used as help - messages, so if you are seeing this, a help message has not yet - been added for this mode. """ - __usage__ = None - __args__ = [] - - def __init__(self): - self.setup = Bcfg2.Options.get_option_parser() - self.configfile = self.setup['configfile'] - self.__cfp = False - self.log = logging.getLogger('Bcfg2.Server.Admin.Mode') - usage = "bcfg2-admin %s" % self.__class__.__name__.lower() - if self.__usage__ is not None: - usage += " " + self.__usage__ - self.setup.hm = usage - - def getCFP(self): - """ get a config parser for the Bcfg2 config file """ - if not self.__cfp: - self.__cfp = ConfigParser.ConfigParser() - self.__cfp.read(self.configfile) - return self.__cfp - - cfp = property(getCFP) - - def __call__(self, args): - raise NotImplementedError - - @classmethod - def usage(cls, rv=1): - """ Exit with a long usage message """ - print(re.sub(r'\s{2,}', ' ', cls.__doc__.strip())) - print("") - print("Usage:") - usage = "bcfg2-admin %s" % cls.__name__.lower() - if cls.__usage__ is not None: - usage += " " + cls.__usage__ - print(" %s" % usage) - raise SystemExit(rv) - - def shutdown(self): - """ Perform any necessary shtudown tasks for this mode """ - pass - - def errExit(self, emsg): - """ exit with an error """ - print(emsg) - raise SystemExit(1) - - def load_stats(self, client): - """ Load static statistics from the repository """ - stats = lxml.etree.parse("%s/etc/statistics.xml" % self.setup['repo']) - hostent = stats.xpath('//Node[@name="%s"]' % client) - if not hostent: - self.errExit("Could not find stats for client %s" % (client)) - return hostent[0] - - def print_table(self, rows, justify='left', hdr=True, vdelim=" ", - padding=1): - """Pretty print a table - - rows - list of rows ([[row 1], [row 2], ..., [row n]]) - hdr - if True the first row is treated as a table header - vdelim - vertical delimiter between columns - padding - # of spaces around the longest element in the column - justify - may be left,center,right - - """ - hdelim = "=" - justify = {'left': str.ljust, - 'center': str.center, - 'right': str.rjust}[justify.lower()] - - # Calculate column widths (longest item in each column - # plus padding on both sides) - cols = list(zip(*rows)) - col_widths = [max([len(str(item)) + 2 * padding - for item in col]) for col in cols] - borderline = vdelim.join([w * hdelim for w in col_widths]) - - # Print out the table - print(borderline) - for row in rows: - print(vdelim.join([justify(str(item), width) - for (item, width) in zip(row, col_widths)])) - if hdr: - print(borderline) - hdr = False - - -# pylint wants MetadataCore and StructureMode to be concrete classes -# and implement __call__, but they aren't and they don't, so we -# disable that warning -# pylint: disable=W0223 - -class MetadataCore(Mode): - """Base class for admin-modes that handle metadata.""" - __plugin_whitelist__ = None - __plugin_blacklist__ = None - - def __init__(self): - Mode.__init__(self) - if self.__plugin_whitelist__ is not None: - self.setup['plugins'] = [p for p in self.setup['plugins'] - if p in self.__plugin_whitelist__] - elif self.__plugin_blacklist__ is not None: - self.setup['plugins'] = [p for p in self.setup['plugins'] - if p not in self.__plugin_blacklist__] - - # admin modes don't need to watch for changes. one shot is fine here. - self.setup['filemonitor'] = 'pseudo' - try: - self.bcore = Bcfg2.Server.Core.BaseCore() - except Bcfg2.Server.Core.CoreInitError: - msg = sys.exc_info()[1] - self.errExit("Core load failed: %s" % msg) - self.bcore.load_plugins() - self.bcore.fam.handle_event_set() - self.metadata = self.bcore.metadata - - def shutdown(self): - if hasattr(self, 'bcore'): - self.bcore.shutdown() - - -class StructureMode(MetadataCore): # pylint: disable=W0223 - """ Base class for admin modes that handle structure plugins """ - pass diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index ea1d97e83..179a6aa9f 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -4,8 +4,9 @@ import sys import time import socket import daemon +import Bcfg2.Options import Bcfg2.Server.Statistics -from Bcfg2.Server.Core import BaseCore, NoExposedMethod +from Bcfg2.Server.Core import NetworkCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.Server.SSLServer import XMLRPCServer @@ -18,29 +19,29 @@ except ImportError: # pylint: enable=E0611 -class Core(BaseCore): +class BuiltinCore(NetworkCore): """ The built-in server core """ name = 'bcfg2-server' def __init__(self): - BaseCore.__init__(self) + NetworkCore.__init__(self) #: The :class:`Bcfg2.Server.SSLServer.XMLRPCServer` instance #: powering this server core self.server = None - daemon_args = dict(uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8), + daemon_args = dict(uid=Bcfg2.Options.setup.daemon_uid, + gid=Bcfg2.Options.setup.daemon_gid, + umask=int(Bcfg2.Options.setup.umask, 8), detach_process=True) - if self.setup['daemon']: - daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], - acquire_timeout=5) + if Bcfg2.Options.setup.daemon: + daemon_args['pidfile'] = TimeoutPIDLockFile( + Bcfg2.Options.setup.daemon, acquire_timeout=5) #: The :class:`daemon.DaemonContext` used to drop #: privileges, write the PID file (with :class:`PidFile`), #: and daemonize this core. self.context = daemon.DaemonContext(**daemon_args) - __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] + __init__.__doc__ = NetworkCore.__init__.__doc__.split('.. -----')[0] def _dispatch(self, method, args, dispatch_dict): """ Dispatch XML-RPC method calls @@ -95,25 +96,25 @@ class Core(BaseCore): except LockTimeout: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, self.setup['daemon'])) + "on %s" % (self.name, + Bcfg2.Options.setup.daemon)) return False def _run(self): """ Create :attr:`server` to start the server listening. """ - hostname, port = urlparse(self.setup['location'])[1].split(':') + hostname, port = urlparse(Bcfg2.Options.setup.server)[1].split(':') server_address = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)[0][4] try: - self.server = XMLRPCServer(self.setup['listen_all'], + self.server = XMLRPCServer(Bcfg2.Options.setup.listen_all, server_address, - keyfile=self.setup['key'], - certfile=self.setup['cert'], + keyfile=Bcfg2.Options.setup.key, + certfile=Bcfg2.Options.setup.cert, register=False, timeout=1, - ca=self.setup['ca'], - protocol=self.setup['protocol']) + ca=Bcfg2.Options.setup.ca) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.logger.error("Server startup failed: %s" % err) diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherrypyCore.py index bf3be72f9..dbfe260f7 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherrypyCore.py @@ -5,7 +5,7 @@ import sys import time import Bcfg2.Server.Statistics from Bcfg2.Compat import urlparse, xmlrpclib, b64decode -from Bcfg2.Server.Core import BaseCore +from Bcfg2.Server.Core import NetworkCore import cherrypy from cherrypy.lib import xmlrpcutil from cherrypy._cptools import ErrorTool @@ -27,7 +27,7 @@ def on_error(*args, **kwargs): # pylint: disable=W0613 cherrypy.tools.xmlrpc_error = ErrorTool(on_error) -class Core(BaseCore): +class CherrypyCore(NetworkCore): """ The CherryPy-based server core. """ #: Base CherryPy config for this class. We enable the @@ -37,7 +37,7 @@ class Core(BaseCore): 'tools.bcfg2_authn.on': True} def __init__(self): - BaseCore.__init__(self) + NetworkCore.__init__(self) cherrypy.tools.bcfg2_authn = cherrypy.Tool('on_start_resource', self.do_authn) @@ -45,11 +45,11 @@ class Core(BaseCore): #: List of exposed plugin RMI self.rmi = self._get_rmi() cherrypy.engine.subscribe('stop', self.shutdown) - __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] + __init__.__doc__ = NetworkCore.__init__.__doc__.split('.. -----')[0] def do_authn(self): """ Perform authentication by calling - :func:`Bcfg2.Server.Core.BaseCore.authenticate`. This is + :func:`Bcfg2.Server.Core.NetworkCore.authenticate`. This is implemented as a CherryPy tool.""" try: header = cherrypy.request.headers['Authorization'] @@ -115,36 +115,36 @@ class Core(BaseCore): with :class:`cherrypy.process.plugins.Daemonizer`, and write a PID file with :class:`cherrypy.process.plugins.PIDFile`. """ DropPrivileges(cherrypy.engine, - uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8)).subscribe() + uid=Bcfg2.Options.setup.daemon_uid, + gid=Bcfg2.Options.setup.daemon_gid, + umask=int(Bcfg2.Options.setup.umask, 8)).subscribe() Daemonizer(cherrypy.engine).subscribe() - PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() + PIDFile(cherrypy.engine, Bcfg2.Options.setup.daemon).subscribe() return True def _run(self): """ Start the server listening. """ - hostname, port = urlparse(self.setup['location'])[1].split(':') - if self.setup['listen_all']: + hostname, port = urlparse(Bcfg2.Options.setup.server)[1].split(':') + if Bcfg2.Options.setup.listen_all: hostname = '0.0.0.0' config = {'engine.autoreload.on': False, 'server.socket_port': int(port), 'server.socket_host': hostname} - if self.setup['cert'] and self.setup['key']: + if Bcfg2.Options.setup.cert and Bcfg2.Options.setup.key: config.update({'server.ssl_module': 'pyopenssl', - 'server.ssl_certificate': self.setup['cert'], - 'server.ssl_private_key': self.setup['key']}) - if self.setup['debug']: + 'server.ssl_certificate': Bcfg2.Options.setup.cert, + 'server.ssl_private_key': Bcfg2.Options.setup.key}) + if Bcfg2.Options.setup.debug: config['log.screen'] = True cherrypy.config.update(config) - cherrypy.tree.mount(self, '/', {'/': self.setup}) + cherrypy.tree.mount(self, '/', {'/': Bcfg2.Options.setup}) cherrypy.engine.start() return True def _block(self): """ Enter the blocking infinite server - loop. :func:`Bcfg2.Server.Core.BaseCore.shutdown` is called on + loop. :func:`Bcfg2.Server.Core.NetworkCore.shutdown` is called on exit by a :meth:`subscription <cherrypy.process.wspbus.Bus.subscribe>` on the top-level CherryPy engine.""" diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 698703457..360b7868d 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -13,12 +13,12 @@ import inspect import lxml.etree import Bcfg2.Server import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.settings import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain from Bcfg2.Server.Cache import Cache -from Bcfg2.Options import get_option_parser, SERVER_FAM_IGNORE from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 @@ -82,43 +82,40 @@ class NoExposedMethod (Exception): # in core we frequently want to catch all exceptions, regardless of # type, so disable the pylint rule that catches that. - -class BaseCore(object): +class Core(object): """ The server core is the container for all Bcfg2 server logic and modules. All core implementations must inherit from - ``BaseCore``. """ + ``Core``. """ + + options = [ + Bcfg2.Options.Common.plugins, + Bcfg2.Options.Common.repository, + Bcfg2.Options.Common.filemonitor, + Bcfg2.Options.BooleanOption( + cf=('server', 'fam_blocking'), default=False, + help='FAM blocks on startup until all events are processed'), + Bcfg2.Options.BooleanOption( + cf=('logging', 'performance'), dest="perflog", + help="Periodically log performance statistics"), + Bcfg2.Options.Option( + cf=('logging', 'performance_interval'), default=300.0, + type=Bcfg2.Options.Types.timeout, + help="Performance statistics logging interval in seconds"), + Bcfg2.Options.Option( + cf=('caching', 'client_metadata'), dest='client_metadata_cache', + default='off', + choices=['off', 'on', 'initial', 'cautious', 'aggressive'])] def __init__(self): # pylint: disable=R0912,R0915 """ - .. automethod:: _daemonize .. automethod:: _run .. automethod:: _block .. ----- .. automethod:: _file_monitor_thread .. automethod:: _perflog_thread """ - #: The Bcfg2 options dict - self.setup = get_option_parser() - #: The Bcfg2 repository directory - self.datastore = self.setup['repo'] - - if self.setup['debug']: - level = logging.DEBUG - elif self.setup['verbose']: - level = logging.INFO - else: - level = logging.WARNING - # we set a higher log level for the console by default. we - # assume that if someone is running bcfg2-server in such a way - # that it _can_ log to console, they want more output. if - # level is set to DEBUG, that will get handled by - # setup_logging and the console will get DEBUG output. - Bcfg2.Logger.setup_logging('bcfg2-server', - to_console=logging.INFO, - to_syslog=self.setup['syslog'], - to_file=self.setup['logging'], - level=level) + self.datastore = Bcfg2.Options.setup.repository #: A :class:`logging.Logger` object for use by the core self.logger = logging.getLogger('bcfg2-server') @@ -130,43 +127,32 @@ class BaseCore(object): #: special, and will be used for any log handlers whose name #: does not appear elsewhere in the dict. At a minimum, #: ``default`` must be provided. - self._loglevels = {True: dict(default=logging.DEBUG), - False: dict(console=logging.INFO, - default=level)} + self._loglevels = { + True: dict(default=logging.DEBUG), + False: dict(console=logging.INFO, + default=Bcfg2.Logger.default_log_level())} #: Used to keep track of the current debug state of the core. self.debug_flag = False # enable debugging on the core now. debugging is enabled on # everything else later - if self.setup['debug']: - self.set_core_debug(None, self.setup['debug']) - - if 'ignore' not in self.setup: - self.setup.add_option('ignore', SERVER_FAM_IGNORE) - self.setup.reparse() - - famargs = dict(filemonitor=self.setup['filemonitor'], - debug=self.setup['debug'], - ignore=self.setup['ignore']) - if self.setup['filemonitor'] not in Bcfg2.Server.FileMonitor.available: - self.logger.error("File monitor driver %s not available; " - "forcing to default" % self.setup['filemonitor']) - famargs['filemonitor'] = 'default' + if Bcfg2.Options.setup.debug: + self.set_core_debug(None, Bcfg2.Options.setup.debug) try: #: The :class:`Bcfg2.Server.FileMonitor.FileMonitor` #: object used by the core to monitor for Bcfg2 data #: changes. - self.fam = Bcfg2.Server.FileMonitor.load_fam(**famargs) + self.fam = Bcfg2.Server.FileMonitor.get_fam() except IOError: msg = "Failed to instantiate fam driver %s" % \ - self.setup['filemonitor'] + Bcfg2.Options.setup.filemonitor self.logger.error(msg, exc_info=1) raise CoreInitError(msg) #: Path to bcfg2.conf - self.cfile = self.setup['configfile'] + self.cfile = Bcfg2.Options.setup.config #: Dict of plugins that are enabled. Keys are the plugin #: names (just the plugin name, in the correct case; e.g., @@ -198,59 +184,19 @@ class BaseCore(object): # generate Django ORM settings. this must be done _before_ we # load plugins - Bcfg2.settings.read_config(repo=self.datastore) - - #: Whether or not it's possible to use the Django database - #: backend for plugins that have that capability - self._database_available = False - if Bcfg2.settings.HAS_DJANGO: - db_settings = Bcfg2.settings.DATABASES['default'] - if ('daemon' in self.setup and 'daemon_uid' in self.setup and - self.setup['daemon'] and self.setup['daemon_uid'] and - db_settings['ENGINE'].endswith(".sqlite3") and - not os.path.exists(db_settings['NAME'])): - # syncdb will create the sqlite database, and we're - # going to daemonize, dropping privs to a non-root - # user, so we need to chown the database after - # creating it - do_chown = True - else: - do_chown = False - - from django.core.exceptions import ImproperlyConfigured - from django.core import management - try: - management.call_command("syncdb", interactive=False, - verbosity=0) - self._database_available = True - except ImproperlyConfigured: - err = sys.exc_info()[1] - self.logger.error("Django configuration problem: %s" % err) - except: - err = sys.exc_info()[1] - self.logger.error("Database update failed: %s" % err) - - if do_chown and self._database_available: - try: - os.chown(db_settings['NAME'], - self.setup['daemon_uid'], - self.setup['daemon_gid']) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Failed to set ownership of database " - "at %s: %s" % (db_settings['NAME'], err)) - - #: The CA that signed the server cert - self.ca = self.setup['ca'] + Bcfg2.settings.read_config() #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ - threading.Thread(name="%sFAMThread" % self.setup['filemonitor'], + threading.Thread(name="%sFAMThread" % + Bcfg2.Options.setup.filemonitor.__name__, target=self._file_monitor_thread) + #: The :class:`threading.Thread` that reports performance + #: statistics to syslog. self.perflog_thread = None - if self.setup['perflog']: + if Bcfg2.Options.setup.perflog: self.perflog_thread = \ threading.Thread(name="PerformanceLoggingThread", target=self._perflog_thread) @@ -263,6 +209,24 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + #: Whether or not it's possible to use the Django database + #: backend for plugins that have that capability + self._database_available = False + if Bcfg2.settings.HAS_DJANGO: + from django.core.exceptions import ImproperlyConfigured + from django.core import management + try: + management.call_command("syncdb", interactive=False, + verbosity=0) + self._database_available = True + except ImproperlyConfigured: + err = sys.exc_info()[1] + self.logger.error("Django configuration problem: %s" % err) + except: + err = sys.exc_info()[1] + self.logger.error("Updating database %s failed: %s" % + (Bcfg2.Options.setup.db_name, err)) + def expire_caches_by_type(self, base_cls, key=None): """ Expire caches for all :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that @@ -302,7 +266,7 @@ class BaseCore(object): to syslog. """ self.logger.debug("Performance logging thread starting") while not self.terminate.isSet(): - self.terminate.wait(self.setup['perflog_interval']) + self.terminate.wait(Bcfg2.Options.setup.performance_interval) if not self.terminate.isSet(): for name, stats in self.get_statistics(None).items(): self.logger.info("Performance statistics: " @@ -354,10 +318,7 @@ class BaseCore(object): :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects. This does not start plugin threads; that is done later, in :func:`Bcfg2.Server.Core.BaseCore.run` """ - while '' in self.setup['plugins']: - self.setup['plugins'].remove('') - - for plugin in self.setup['plugins']: + for plugin in Bcfg2.Options.setup.plugins: if not plugin in self.plugins: self.init_plugin(plugin) @@ -397,10 +358,6 @@ class BaseCore(object): "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") - if self.debug_flag: - # enable debugging on plugins - self.plugins[plugin].set_debug(self.debug_flag) - def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. @@ -411,29 +368,13 @@ class BaseCore(object): :type plugin: string :returns: None """ - self.logger.debug("Loading plugin %s" % plugin) - try: - mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - (plugin)).Server.Plugins, plugin) - except ImportError: - try: - mod = __import__(plugin, globals(), locals(), - [plugin.split('.')[-1]]) - except: - self.logger.error("Failed to load plugin %s" % plugin) - return - try: - plug = getattr(mod, plugin.split('.')[-1]) - except AttributeError: - self.logger.error("Failed to load plugin %s: %s" % - (plugin, sys.exc_info()[1])) - return + self.logger.debug("Loading plugin %s" % plugin.name) # Blacklist conflicting plugins - cplugs = [conflict for conflict in plug.conflicts + cplugs = [conflict for conflict in plugin.conflicts if conflict in self.plugins] - self.plugin_blacklist[plug.name] = cplugs + self.plugin_blacklist[plugin.name] = cplugs try: - self.plugins[plugin] = plug(self, self.datastore) + self.plugins[plugin.name] = plugin(self, self.datastore) except PluginInitError: self.logger.error("Failed to instantiate plugin %s" % plugin, exc_info=1) @@ -461,10 +402,7 @@ class BaseCore(object): """ Get the client :attr:`metadata_cache` mode. Options are off, initial, cautious, aggressive, on (synonym for cautious). See :ref:`server-caching` for more details. """ - # pylint: disable=E1103 - mode = self.setup.cfp.get("caching", "client_metadata", - default="off").lower() - # pylint: enable=E1103 + mode = Bcfg2.Options.setup.client_metadata_cache if mode == "on": return "cautious" else: @@ -648,10 +586,9 @@ class BaseCore(object): del entry.attrib['realname'] return ret except: - entry.set('name', oldname) self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('name'), - entry.get('altsrc'))) + (entry.tag, oldname, entry.get('name'))) + entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -745,7 +682,7 @@ class BaseCore(object): return if event.code2str() == 'deleted': return - self.setup.reparse() + Bcfg2.Options.get_parser().reparse() self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def block_for_fam_events(self, handle_events=False): @@ -758,7 +695,7 @@ class BaseCore(object): if handle_events: self.fam.handle_events_in_interval(1) slept += 1 - if self.setup['fam_blocking']: + if Bcfg2.Options.setup.fam_blocking: time.sleep(1) slept += 1 while self.fam.pending() != 0: @@ -769,35 +706,12 @@ class BaseCore(object): self.logger.debug("Slept %s seconds while handling FAM events" % slept) def run(self): - """ Run the server core. This calls :func:`_daemonize`, - :func:`_run`, starts the :attr:`fam_thread`, and calls - :func:`_block`, but note that it is the responsibility of the - server core implementation to call :func:`shutdown` under - normal operation. This also handles creation of the directory - containing the pidfile, if necessary. """ - if self.setup['daemon']: - # if we're dropping privs, then the pidfile is likely - # /var/run/bcfg2-server/bcfg2-server.pid or similar. - # since some OSes clean directories out of /var/run on - # reboot, we need to ensure that the directory containing - # the pidfile exists and has the appropriate permissions - piddir = os.path.dirname(self.setup['daemon']) - if not os.path.exists(piddir): - os.makedirs(piddir) - os.chown(piddir, - self.setup['daemon_uid'], - self.setup['daemon_gid']) - os.chmod(piddir, 493) # 0775 - if not self._daemonize(): - return False - - # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so - # this is necessary to make that work when privileges are - # dropped - os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] - else: - os.umask(int(self.setup['umask'], 8)) - + """ Run the server core. This calls :func:`_run`, starts the + :attr:`fam_thread`, and calls :func:`_block`, but note that it + is the responsibility of the server core implementation to + call :func:`shutdown` under normal operation. This also + handles creation of the directory containing the pidfile, if + necessary.""" if not self._run(): self.shutdown() return False @@ -817,16 +731,9 @@ class BaseCore(object): self.shutdown() raise - if self.debug_flag: - self.set_debug(None, self.debug_flag) self.block_for_fam_events() self._block() - def _daemonize(self): - """ Daemonize the server and write the pidfile. This must be - overridden by a core implementation. """ - raise NotImplementedError - def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core @@ -884,9 +791,13 @@ class BaseCore(object): if all(ip_checks): # if all ACL plugins return True (allow), then allow + self.logger.debug("Client %s passed IP-based ACL checks for %s" % + (address[0], rmi)) return True elif False in ip_checks: # if any ACL plugin returned False (deny), then deny + self.logger.warning("Client %s failed IP-based ACL checks for %s" % + (address[0], rmi)) return False # else, no plugins returned False, but not all plugins # returned True, so some plugin returned None (defer), so @@ -894,7 +805,16 @@ class BaseCore(object): client, metadata = self.resolve_client(address) try: - return all(p.check_acl_metadata(metadata, rmi) for p in plugins) + rv = all(p.check_acl_metadata(metadata, rmi) for p in plugins) + if rv: + self.logger.debug( + "Client %s passed metadata ACL checks for %s" % + (metadata.hostname, rmi)) + else: + self.logger.warning( + "Client %s failed metadata ACL checks for %s" % + (metadata.hostname, rmi)) + return rv except: self.logger.error("Unexpected error checking ACLs for %s for %s: " "%s" % (client, rmi, sys.exc_info()[1])) @@ -1226,28 +1146,6 @@ class BaseCore(object): self.process_statistics(client, sdata) return True - def authenticate(self, cert, user, password, address): - """ Authenticate a client connection with - :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`. - - :param cert: an x509 certificate - :type cert: dict - :param user: The username of the user trying to authenticate - :type user: string - :param password: The password supplied by the client - :type password: string - :param address: An address pair of ``(<ip address>, <port>)`` - :type address: tuple - :return: bool - True if the authenticate succeeds, False otherwise - """ - if self.ca: - acert = cert - else: - # No ca, so no cert validation can be done - acert = None - return self.metadata.AuthenticateConnection(acert, user, password, - address) - @exposed def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. @@ -1369,3 +1267,110 @@ class BaseCore(object): address[0]) return "This method is deprecated and will be removed in a future " + \ "release\n%s" % self.fam.set_debug(debug) + + +class NetworkCore(Core): + """ A server core that actually listens on the network, can be + daemonized, etc.""" + options = Core.options + [ + Bcfg2.Options.Common.daemon, Bcfg2.Options.Common.syslog, + Bcfg2.Options.Common.location, Bcfg2.Options.Common.ssl_key, + Bcfg2.Options.Common.ssl_cert, Bcfg2.Options.Common.ssl_ca, + Bcfg2.Options.BooleanOption( + '--listen-all', cf=('server', 'listen_all'), default=False, + help="Listen on all interfaces"), + Bcfg2.Options.Option( + cf=('server', 'umask'), default='0077', help='Server umask', + type=Bcfg2.Options.Types.octal), + Bcfg2.Options.Option( + cf=('server', 'user'), default=0, dest='daemon_uid', + type=Bcfg2.Options.Types.username, + help="User to run the server daemon as"), + Bcfg2.Options.Option( + cf=('server', 'group'), default=0, dest='daemon_gid', + type=Bcfg2.Options.Types.groupname, + help="Group to run the server daemon as")] + + def __init__(self): + Core.__init__(self) + + #: The CA that signed the server cert + self.ca = Bcfg2.Options.setup.ca + + if self._database_available: + db_settings = Bcfg2.settings.DATABASES['default'] + if (Bcfg2.Options.setup.daemon and + Bcfg2.Options.setup.daemon_uid and + db_settings['ENGINE'].endswith(".sqlite3") and + not os.path.exists(db_settings['NAME'])): + # syncdb will create the sqlite database, and we're + # going to daemonize, dropping privs to a non-root + # user, so we need to chown the database after + # creating it + try: + os.chown(db_settings['NAME'], + Bcfg2.Options.setup.daemon_uid, + Bcfg2.Options.setup.daemon_gid) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Failed to set ownership of database " + "at %s: %s" % (db_settings['NAME'], err)) + __init__.__doc__ = Core.__init__.__doc__.split(".. -----")[0] + \ + "\n.. automethod:: _daemonize\n" + + def run(self): + """ Run the server core. This calls :func:`_daemonize` before + calling :func:`Bcfg2.Server.Core.Core.run` to run the server + core. """ + if Bcfg2.Options.setup.daemon: + # if we're dropping privs, then the pidfile is likely + # /var/run/bcfg2-server/bcfg2-server.pid or similar. + # since some OSes clean directories out of /var/run on + # reboot, we need to ensure that the directory containing + # the pidfile exists and has the appropriate permissions + piddir = os.path.dirname(Bcfg2.Options.setup.daemon) + if not os.path.exists(piddir): + os.makedirs(piddir) + os.chown(piddir, + Bcfg2.Options.setup.daemon_uid, + Bcfg2.Options.setup.daemon_gid) + os.chmod(piddir, 493) # 0775 + if not self._daemonize(): + return False + + # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so + # this is necessary to make that work when privileges are + # dropped + os.environ['HOME'] = \ + pwd.getpwuid(Bcfg2.Options.setup.daemon_uid)[5] + else: + os.umask(int(Bcfg2.Options.setup.umask, 8)) + + Core.run(self) + + def authenticate(self, cert, user, password, address): + """ Authenticate a client connection with + :func:`Bcfg2.Server.Plugin.interfaces.Metadata.AuthenticateConnection`. + + :param cert: an x509 certificate + :type cert: dict + :param user: The username of the user trying to authenticate + :type user: string + :param password: The password supplied by the client + :type password: string + :param address: An address pair of ``(<ip address>, <port>)`` + :type address: tuple + :return: bool - True if the authenticate succeeds, False otherwise + """ + if self.ca: + acert = cert + else: + # No ca, so no cert validation can be done + acert = None + return self.metadata.AuthenticateConnection(acert, user, password, + address) + + def _daemonize(self): + """ Daemonize the server and write the pidfile. This must be + overridden by a core implementation. """ + raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index 797b44ab9..7e1294587 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -3,10 +3,17 @@ for handling encryption in Bcfg2. See :ref:`server-encryption` for more details. """ import os +import sys +import copy +import logging +import lxml.etree +import Bcfg2.Logger import Bcfg2.Options from M2Crypto import Rand from M2Crypto.EVP import Cipher, EVPError -from Bcfg2.Compat import StringIO, md5, b64encode, b64decode +from Bcfg2.Utils import safe_input +from Bcfg2.Server import XMLParser +from Bcfg2.Compat import md5, b64encode, b64decode, StringIO #: Constant representing the encryption operation for #: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This @@ -23,26 +30,22 @@ DECRYPT = 0 #: automated fashion. IV = r'\0' * 16 -#: The config file section encryption options and passphrases are -#: stored in -CFG_SECTION = "encryption" -#: The config option used to store the algorithm -CFG_ALGORITHM = "algorithm" +class _OptionContainer(object): + options = [ + Bcfg2.Options.BooleanOption( + cf=("encryption", "lax_decryption"), + help="Decryption failures should cause warnings, not errors"), + Bcfg2.Options.Option( + cf=("encryption", "algorithm"), default="aes_256_cbc", + type=lambda v: v.lower().replace("-", "_"), + help="The encryption algorithm to use"), + Bcfg2.Options.Option( + cf=("encryption", "*"), dest='passphrases', default=dict(), + help="Encryption passphrases")] -#: The config option used to store the decryption strictness -CFG_DECRYPT = "decrypt" - -#: Default cipher algorithm. To get a full list of valid algorithms, -#: you can run:: -#: -#: openssl list-cipher-algorithms | grep -v ' => ' | \ -#: tr 'A-Z-' 'a-z_' | sort -u -ALGORITHM = Bcfg2.Options.get_option_parser().cfp.get( # pylint: disable=E1103 - CFG_SECTION, - CFG_ALGORITHM, - default="aes_256_cbc").lower().replace("-", "_") +Bcfg2.Options.get_parser().add_component(_OptionContainer) Rand.rand_seed(os.urandom(1024)) @@ -64,7 +67,7 @@ def _cipher_filter(cipher, instr): return rv -def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): +def str_encrypt(plaintext, key, iv=IV, algorithm=None, salt=None): """ Encrypt a string with a key. For a higher-level encryption interface, see :func:`ssl_encrypt`. @@ -80,11 +83,13 @@ def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): :type salt: string :returns: string - The decrypted data """ + if algorithm is None: + algorithm = Bcfg2.Options.setup.algorithm cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) return _cipher_filter(cipher, plaintext) -def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): +def str_decrypt(crypted, key, iv=IV, algorithm=None): """ Decrypt a string with a key. For a higher-level decryption interface, see :func:`ssl_decrypt`. @@ -98,11 +103,13 @@ def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): :type algorithm: string :returns: string - The decrypted data """ + if algorithm is None: + algorithm = Bcfg2.Options.setup.algorithm cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) return _cipher_filter(cipher, crypted) -def ssl_decrypt(data, passwd, algorithm=ALGORITHM): +def ssl_decrypt(data, passwd, algorithm=None): """ Decrypt openssl-encrypted data. This can decrypt data encrypted by :func:`ssl_encrypt`, or ``openssl enc``. It performs a base64 decode first if the data is base64 encoded, and @@ -132,7 +139,7 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): return str_decrypt(data[16:], key=key, iv=iv, algorithm=algorithm) -def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): +def ssl_encrypt(plaintext, passwd, algorithm=None, salt=None): """ Encrypt data in a format that is openssl compatible. :param plaintext: The plaintext data to encrypt @@ -164,25 +171,10 @@ def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): return b64encode("Salted__" + salt + crypted) + "\n" -def get_passphrases(): - """ Get all candidate encryption passphrases from the config file. - - :returns: dict - a dict of ``<passphrase name>``: ``<passphrase>`` - """ - setup = Bcfg2.Options.get_option_parser() - if setup.cfp.has_section(CFG_SECTION): - return dict([(o, setup.cfp.get(CFG_SECTION, o)) - for o in setup.cfp.options(CFG_SECTION) - if o not in [CFG_ALGORITHM, CFG_DECRYPT]]) - else: - return dict() - - -def bruteforce_decrypt(crypted, passphrases=None, algorithm=ALGORITHM): +def bruteforce_decrypt(crypted, passphrases=None, algorithm=None): """ Convenience method to decrypt the given encrypted string by - trying the given passphrases or all passphrases (as returned by - :func:`get_passphrases`) sequentially until one is found that - works. + trying the given passphrases or all passphrases sequentially until + one is found that works. :param crypted: The data to decrypt :type crypted: string @@ -194,10 +186,413 @@ def bruteforce_decrypt(crypted, passphrases=None, algorithm=ALGORITHM): :raises: :class:`M2Crypto.EVP.EVPError`, if the data cannot be decrypted """ if passphrases is None: - passphrases = get_passphrases().values() + passphrases = Bcfg2.Options.setup.passphrases.values() for passwd in passphrases: try: return ssl_decrypt(crypted, passwd, algorithm=algorithm) except EVPError: pass raise EVPError("Failed to decrypt") + + +class PassphraseError(Exception): + """ Exception raised when there's a problem determining the + passphrase to encrypt or decrypt with """ + + +class CryptoTool(object): + """ Generic decryption/encryption interface base object """ + + def __init__(self, filename): + self.logger = logging.getLogger(self.__class__.__name__) + self.filename = filename + self.data = open(self.filename).read() + self.pname, self.passphrase = self._get_passphrase() + + def _get_passphrase(self): + """ get the passphrase for the current file """ + if not Bcfg2.Options.setup.passphrases: + raise PassphraseError("No passphrases available in %s" % + Bcfg2.Options.setup.configfile) + + pname = None + if Bcfg2.Options.setup.passphrase: + pname = Bcfg2.Options.setup.passphrase + + if pname: + try: + passphrase = Bcfg2.Options.setup.passphrases[pname] + self.logger.debug("Using passphrase %s specified on command " + "line" % pname) + return (pname, passphrase) + except KeyError: + raise PassphraseError("Could not find passphrase %s in %s" % + (pname, Bcfg2.Options.setup.configfile)) + else: + if len(Bcfg2.Options.setup.passphrases) == 1: + pname, passphrase = Bcfg2.Options.setup.passphrases.items()[0] + self.logger.info("Using passphrase %s" % pname) + return (pname, passphrase) + elif len(Bcfg2.Options.setup.passphrases) > 1: + return (None, None) + raise PassphraseError("No passphrase could be determined") + + def get_destination_filename(self, original_filename): + """ Get the filename where data should be written """ + return original_filename + + def write(self, data): + """ write data to disk """ + new_fname = self.get_destination_filename(self.filename) + try: + self._write(new_fname, data) + self.logger.info("Wrote data to %s" % new_fname) + return True + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error writing data from %s to %s: %s" % + (self.filename, new_fname, err)) + return False + + def _write(self, filename, data): + """ Perform the actual write of data. This is separate from + :func:`CryptoTool.write` so it can be easily + overridden. """ + open(filename, "wb").write(data) + + +class Decryptor(CryptoTool): + """ Decryptor interface """ + def decrypt(self): + """ decrypt the file, returning the encrypted data """ + raise NotImplementedError + + +class Encryptor(CryptoTool): + """ encryptor interface """ + def encrypt(self): + """ encrypt the file, returning the encrypted data """ + raise NotImplementedError + + +class CfgEncryptor(Encryptor): + """ encryptor class for Cfg files """ + + def __init__(self, filename): + Encryptor.__init__(self, filename) + if self.passphrase is None: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + Bcfg2.Options.setup.configfile) + + def encrypt(self): + return ssl_encrypt(self.data, self.passphrase) + + def get_destination_filename(self, original_filename): + return original_filename + ".crypt" + + +class CfgDecryptor(Decryptor): + """ Decrypt Cfg files """ + + def decrypt(self): + """ decrypt the given file, returning the plaintext data """ + if self.passphrase: + try: + return ssl_decrypt(self.data, self.passphrase) + except EVPError: + self.logger.info("Could not decrypt %s with the " + "specified passphrase" % self.filename) + return False + except: + err = sys.exc_info()[1] + self.logger.error("Error decrypting %s: %s" % + (self.filename, err)) + return False + else: # no passphrase given, brute force + try: + return bruteforce_decrypt(self.data) + except EVPError: + self.logger.info("Could not decrypt %s with any passphrase" % + self.filename) + return False + + def get_destination_filename(self, original_filename): + if original_filename.endswith(".crypt"): + return original_filename[:-6] + else: + return Decryptor.get_destination_filename(self, original_filename) + + +class PropertiesCryptoMixin(object): + """ Mixin to provide some common methods for Properties crypto """ + default_xpath = '//*' + + def _get_elements(self, xdata): + """ Get the list of elements to encrypt or decrypt """ + if Bcfg2.Options.setup.xpath: + elements = xdata.xpath(Bcfg2.Options.setup.xpath) + if not elements: + self.logger.warning("XPath expression %s matched no elements" % + Bcfg2.Options.setup.xpath) + else: + elements = xdata.xpath(self.default_xpath) + if not elements: + elements = list(xdata.getiterator(tag=lxml.etree.Element)) + + # filter out elements without text data + for el in elements[:]: + if not el.text: + elements.remove(el) + + if Bcfg2.Options.setup.interactive: + for element in elements[:]: + if len(element): + elt = copy.copy(element) + for child in elt.iterchildren(): + elt.remove(child) + else: + elt = element + print(lxml.etree.tostring( + elt, + xml_declaration=False).decode("UTF-8").strip()) + ans = safe_input("Encrypt this element? [y/N] ") + if not ans.lower().startswith("y"): + elements.remove(element) + return elements + + def _get_element_passphrase(self, element): + """ Get the passphrase to use to encrypt or decrypt a given + element """ + pname = element.get("encrypted") + if pname in self.passphrases: + passphrase = self.passphrases[pname] + elif self.passphrase: + if pname: + self.logger.warning("Passphrase %s not found in %s, " + "using passphrase given on command line" % + (pname, Bcfg2.Option.setup.configfile)) + passphrase = self.passphrase + pname = self.pname + else: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + Bcfg2.Options.setup.configfile) + return (pname, passphrase) + + def _write(self, filename, data): + """ Write the data """ + data.getroottree().write(filename, + xml_declaration=False, + pretty_print=True) + + +class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): + """ encryptor class for Properties files """ + + def encrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + elt.text = ssl_encrypt(elt.text, passphrase).strip() + elt.set("encrypted", pname) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): + """ decryptor class for Properties files """ + default_xpath = '//*[@encrypted]' + + def decrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + decrypted = ssl_decrypt(elt.text, passphrase).strip() + try: + elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') + elt.set("encrypted", pname) + except UnicodeDecodeError: + # we managed to decrypt the value, but it contains + # content that can't even be encoded into xml + # entities. what probably happened here is that we + # coincidentally could decrypt a value encrypted with + # a different key, and wound up with gibberish. + self.logger.warning("Decrypted %s to gibberish, skipping" % + elt.tag) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class CLI(object): + """ The bcfg2-crypt CLI """ + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "--encrypt", help='Encrypt the specified file'), + Bcfg2.Options.BooleanOption( + "--decrypt", help='Decrypt the specified file')), + Bcfg2.Options.BooleanOption( + "--stdout", + help='Decrypt or encrypt the specified file to stdout'), + Bcfg2.Options.Option( + "-p", "--passphrase", metavar="NAME", + help='Encryption passphrase name'), + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.BooleanOption( + "--properties", + help='Encrypt the specified file as a Properties file'), + Bcfg2.Options.BooleanOption( + "--cfg", help='Encrypt the specified file as a Cfg file')), + Bcfg2.Options.OptionGroup( + Bcfg2.Options.Common.interactive, + Bcfg2.Options.Option( + "--xpath", + help='XPath expression to select elements to encrypt'), + title="Options for handling Properties files"), + Bcfg2.Options.OptionGroup( + Bcfg2.Options.BooleanOption( + "--remove", help='Remove the plaintext file after encrypting'), + title="Options for handling Cfg files"), + Bcfg2.Options.PathOption( + "files", help="File(s) to encrypt or decrypt", nargs='+')] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Encrypt and decrypt Bcfg2 data", + components=[self, _OptionContainer]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + if Bcfg2.Options.setup.decrypt: + if Bcfg2.Options.setup.remove: + self.logger.error("--remove cannot be used with --decrypt, " + "ignoring --remove") + Bcfg2.Options.setup.remove = False + elif Bcfg2.Options.setup.interactive: + self.logger.error("Cannot decrypt interactively") + Bcfg2.Options.setup.interactive = False + + def _is_properties(self, filename): + """ Determine if a given file is a Properties file or not """ + if Bcfg2.Options.setup.properties: + return True + elif Bcfg2.Options.setup.cfg: + return False + elif filename.endswith(".xml"): + try: + xroot = lxml.etree.parse(filename).getroot() + return xroot.tag == "Properties" + except lxml.etree.XMLSyntaxError: + return False + else: + return False + + def run(self): # pylint: disable=R0912,R0915 + for fname in Bcfg2.Options.setup.files: + if not os.path.exists(fname): + self.logger.error("%s does not exist, skipping" % fname) + continue + + # figure out if we need to encrypt this as a Properties file + # or as a Cfg file + try: + props = self._is_properties(fname) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + + if props: + if Bcfg2.Options.setup.remove: + self.logger.info("Cannot use --remove with Properties " + "file %s, ignoring for this file" % fname) + try: + tools = (PropertiesEncryptor(fname), + PropertiesDecryptor(fname)) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + else: + if Bcfg2.Options.setup.xpath: + self.logger.error("Specifying --xpath with --cfg is " + "nonsensical, ignoring --xpath") + Bcfg2.Options.setup.xpath = None + if Bcfg2.Options.setup.interactive: + self.logger.error("Cannot use interactive mode with " + "--cfg, ignoring --interactive") + Bcfg2.Options.setup.interactive = False + try: + tools = (CfgEncryptor(fname), CfgDecryptor(fname)) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + continue + except IOError: + self.logger.error("Error reading %s, skipping: %s" % + (fname, err)) + continue + + data = None + mode = None + if Bcfg2.Options.setup.encrypt: + tool = tools[0] + mode = "encrypt" + elif Bcfg2.Options.setup.decrypt: + tool = tools[1] + mode = "decrypt" + else: + self.logger.info("Neither --encrypt nor --decrypt specified, " + "determining mode") + tool = tools[1] + try: + data = tool.decrypt() + mode = "decrypt" + except: # pylint: disable=W0702 + pass + if data is False: + data = None + self.logger.info("Failed to decrypt %s, trying encryption" + % fname) + tool = tools[0] + mode = "encrypt" + + if data is None: + data = getattr(tool, mode)() + if not data: + self.logger.error("Failed to %s %s, skipping" % (mode, fname)) + continue + if Bcfg2.Options.setup.stdout: + if len(Bcfg2.Options.setup.files) > 1: + print("----- %s -----" % fname) + print(data) + if len(Bcfg2.Options.setup.files) > 1: + print("") + else: + tool.write(data) + + if (Bcfg2.Options.setup.remove and + tool.get_destination_filename(fname) != fname): + try: + os.unlink(fname) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error removing %s: %s" % (fname, err)) + continue diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py index 9134758b8..69463ab4c 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -33,8 +33,8 @@ class Gamin(FileMonitor): #: releases, so it has a fairly high priority. __priority__ = 90 - def __init__(self, ignore=None, debug=False): - FileMonitor.__init__(self, ignore=ignore, debug=debug) + def __init__(self): + FileMonitor.__init__(self) #: The :class:`Gamin.WatchMonitor` object for this monitor. self.mon = None diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 2cdf27ed8..39d062604 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -34,8 +34,8 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): #: listed in :attr:`action_map` mask = reduce(lambda x, y: x | y, action_map.keys()) - def __init__(self, ignore=None, debug=False): - Pseudo.__init__(self, ignore=ignore, debug=debug) + def __init__(self): + Pseudo.__init__(self) pyinotify.ProcessEvent.__init__(self) #: inotify can't set useful monitors directly on files, only diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 522ddb705..ae42a3429 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -48,11 +48,9 @@ Base Classes import os import sys import fnmatch -import logging +import Bcfg2.Options from time import sleep, time -from Bcfg2.Server.Plugin import Debuggable - -LOGGER = logging.getLogger(__name__) +from Bcfg2.Logger import Debuggable class Event(object): @@ -112,6 +110,14 @@ class FileMonitor(Debuggable): monitor objects to :attr:`handles` and received events to :attr:`events`; the basic interface will handle the rest. """ + options = [ + Bcfg2.Options.Option( + cf=('server', 'ignore_files'), + help='File globs to ignore', + type=Bcfg2.Options.Types.comma_list, + default=['*~', '*#', '.#*', '*.swp', '*.swpx', '.*.swx', + 'SCCS', '.svn', '4913', '.gitignore'])] + #: The relative priority of this FAM backend. Better backends #: should have higher priorities. __priority__ = -1 @@ -119,7 +125,7 @@ class FileMonitor(Debuggable): #: List of names of methods to be exposed as XML-RPC functions __rmi__ = Debuggable.__rmi__ + ["list_event_handlers"] - def __init__(self, ignore=None, debug=False): + def __init__(self): """ :param ignore: A list of filename globs describing events that should be ignored (i.e., not processed by any @@ -133,7 +139,6 @@ class FileMonitor(Debuggable): .. autoattribute:: __priority__ """ Debuggable.__init__(self) - self.debug_flag = debug #: A dict that records which objects handle which events. #: Keys are monitor handle IDs and values are objects whose @@ -143,12 +148,10 @@ class FileMonitor(Debuggable): #: Queue of events to handle self.events = [] - if ignore is None: - ignore = [] #: List of filename globs to ignore events for. For events #: that include the full path, both the full path and the bare #: filename will be checked against ``ignore``. - self.ignore = ignore + self.ignore = Bcfg2.Options.setup.ignore_files #: Whether or not the FAM has been started. See :func:`start`. self.started = False @@ -226,8 +229,8 @@ class FileMonitor(Debuggable): if self.should_ignore(event): return if event.requestID not in self.handles: - LOGGER.info("Got event for unexpected id %s, file %s" % - (event.requestID, event.filename)) + self.logger.info("Got event for unexpected id %s, file %s" % + (event.requestID, event.filename)) return self.debug_log("Dispatching event %s %s to obj %s" % (event.code2str(), event.filename, @@ -236,8 +239,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID].HandleEvent(event) except: # pylint: disable=W0702 err = sys.exc_info()[1] - LOGGER.error("Error in handling of event %s for %s: %s" % - (event.code2str(), event.filename, err)) + self.logger.error("Error in handling of event %s for %s: %s" % + (event.code2str(), event.filename, err)) def handle_event_set(self, lock=None): """ Handle all pending events. @@ -263,7 +266,8 @@ class FileMonitor(Debuggable): lock.release() end = time() if count > 0: - LOGGER.info("Handled %d events in %.03fs" % (count, (end - start))) + self.logger.info("Handled %d events in %.03fs" % (count, + (end - start))) def handle_events_in_interval(self, interval): """ Handle events for the specified period of time (in @@ -330,37 +334,17 @@ class FileMonitor(Debuggable): _FAM = None -def load_fam(filemonitor='default', ignore=None, debug=False): - """ Load a new :class:`Bcfg2.Server.FileMonitor.FileMonitor` - object, caching it in :attr:`_FAM` for later retrieval via - :func:`get_fam`. - - :param filemonitor: Which filemonitor backend to use - :type filemonitor: string - :param ignore: A list of filenames to ignore - :type ignore: list of strings (filename globs) - :param debug: Produce debugging information - :type debug: bool - :returns: :class:`Bcfg2.Server.FileMonitor.FileMonitor` - """ - global _FAM # pylint: disable=W0603 - if _FAM is None: - if ignore is None: - ignore = [] - _FAM = available[filemonitor](ignore=ignore, debug=debug) - return _FAM - - def get_fam(): - """ Get an already-created + """ Get a :class:`Bcfg2.Server.FileMonitor.FileMonitor` object. If :attr:`_FAM` has not been populated, then a new default FileMonitor will be created. :returns: :class:`Bcfg2.Server.FileMonitor.FileMonitor` """ + global _FAM # pylint: disable=W0603 if _FAM is None: - return load_fam('default') + _FAM = Bcfg2.Options.setup.filemonitor() return _FAM diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py new file mode 100644 index 000000000..24d7cc637 --- /dev/null +++ b/src/lib/Bcfg2/Server/Info.py @@ -0,0 +1,870 @@ +""" Subcommands and helpers for bcfg2-info """ +# -*- coding: utf-8 -*- + +import os +import sys +import cmd +import math +import time +import copy +import pipes +import fnmatch +import argparse +import operator +import lxml.etree +from code import InteractiveConsole +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Server.Plugin +import Bcfg2.Client.Tools.POSIX +from Bcfg2.Compat import any # pylint: disable=W0622 + +try: + try: + import cProfile as profile + except ImportError: + import profile + import pstats + HAS_PROFILE = True +except ImportError: + HAS_PROFILE = False + + +def print_tabular(rows): + """Print data in tabular format.""" + cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 + for index in range(len(rows[0]))]) + fstring = (" %%-%ss |" * len(cmax)) % cmax + fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax + print(fstring % rows[0]) + print((sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=') + for row in rows[1:]: + print(fstring % row) + + +def display_trace(trace): + """ display statistics from a profile trace """ + stats = pstats.Stats(trace) + stats.sort_stats('cumulative', 'calls', 'time') + stats.print_stats(200) + + +def load_interpreters(): + """ Load a dict of available Python interpreters """ + interpreters = dict(python=lambda v: InteractiveConsole(v).interact()) + default = "python" + try: + import bpython.cli + interpreters["bpython"] = lambda v: bpython.cli.main(args=[], + locals_=v) + default = "bpython" + except ImportError: + pass + + try: + # whether ipython is actually better than bpython is + # up for debate, but this is the behavior that existed + # before --interpreter was added, so we call IPython + # better + import IPython + # pylint: disable=E1101 + if hasattr(IPython, "Shell"): + interpreters["ipython"] = lambda v: \ + IPython.Shell.IPShell(argv=[], user_ns=v).mainloop() + default = "ipython" + elif hasattr(IPython, "embed"): + interpreters["ipython"] = lambda v: IPython.embed(user_ns=v) + default = "ipython" + else: + print("Unknown IPython API version") + # pylint: enable=E1101 + except ImportError: + pass + + return (interpreters, default) + + +class InfoCmd(Bcfg2.Options.Subcommand): + """ Base class for bcfg2-info subcommands """ + + def _expand_globs(self, globs, candidates): + # special cases to speed things up: + if globs is None or '*' in globs: + return candidates + has_wildcards = False + for glob in globs: + # check if any wildcard characters are in the string + if set('*?[]') & set(glob): + has_wildcards = True + break + if not has_wildcards: + return globs + + rv = set() + cset = set(candidates) + for glob in globs: + rv.update(c for c in cset if fnmatch.fnmatch(c, glob)) + cset.difference_update(rv) + return list(rv) + + def get_client_list(self, globs): + """ given a list of host globs, get a list of clients that + match them """ + return self._expand_globs(globs, self.core.metadata.clients) + + def get_group_list(self, globs): + """ given a list of group glob, get a list of groups that + match them""" + # special cases to speed things up: + return self._expand_globs(globs, + list(self.core.metadata.groups.keys())) + + +class Help(InfoCmd, Bcfg2.Options.HelpCommand): + """ Get help on a specific subcommand """ + def command_registry(self): + return self.core.commands + + +class Debug(InfoCmd): + """ Shell out to a Python interpreter """ + interpreters, default_interpreter = load_interpreters() + options = [ + Bcfg2.Options.BooleanOption( + "-n", "--non-interactive", + help="Do not enter the interactive debugger"), + Bcfg2.Options.PathOption( + "-f", dest="cmd_list", type=argparse.FileType('r'), + help="File containing commands to run"), + Bcfg2.Options.Option( + "--interpreter", cf=("bcfg2-info", "interpreter"), + env="BCFG2_INFO_INTERPRETER", + choices=interpreters.keys(), default=default_interpreter)] + + def run(self, setup): + if setup.cmd_list: + console = InteractiveConsole(locals()) + for command in setup.cmd_list.readlines(): + command = command.strip() + if command: + console.push(command) + if not setup.non_interactive: + print("Dropping to interpreter; press ^D to resume") + self.interpreters[setup.interpreter](self.core.get_locals()) + + +class Build(InfoCmd): + """ Build config for hostname, writing to filename """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("filename", nargs='?', + default=sys.stdout, + type=argparse.FileType('w'))] + + def run(self, setup): + etree = lxml.etree.ElementTree( + self.core.BuildConfiguration(setup.hostname)) + try: + etree.write( + setup.filename, + encoding='UTF-8', xml_declaration=True, + pretty_print=True) + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.filename, err)) + + +class Builddir(InfoCmd): + """ Build config for hostname, writing separate files to directory + """ + + # don't try to isntall these types of entries + blacklisted_types = ["nonexistent", "permissions"] + + options = Bcfg2.Client.Tools.POSIX.POSIX.options + [ + Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PathOption("directory")] + + help = """Generates a config for client <hostname> and writes the +individual configuration files out separately in a tree under <output +dir>. This only handles file entries, and does not respect 'owner' or +'group' attributes unless run as root. """ + + def run(self, setup): + setup.paranoid = False + client_config = self.core.BuildConfiguration(setup.hostname) + if client_config.tag == 'error': + print("Building client configuration failed.") + return 1 + + entries = [] + for struct in client_config: + for entry in struct: + if (entry.tag == 'Path' and + entry.get("type") not in self.blacklisted_types): + failure = entry.get("failure") + if failure is not None: + print("Skipping entry %s:%s with bind failure: %s" % + (entry.tag, entry.get("name"), failure)) + continue + entry.set('name', + os.path.join(setup.directory, + entry.get('name').lstrip("/"))) + entries.append(entry) + + Bcfg2.Client.Tools.POSIX.POSIX(client_config).Install(entries) + + +class Buildfile(InfoCmd): + """ Build config file for hostname """ + + options = [ + Bcfg2.Options.Option("-f", "--outfile", metavar="<path>", + type=argparse.FileType('w'), default=sys.stdout), + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + entry = lxml.etree.Element('Path', name=setup.filename) + if setup.altsrc: + entry.set("altsrc", setup.altsrc) + try: + self.core.Bind(entry, self.core.build_metadata(setup.hostname)) + except: # pylint: disable=W0702 + print("Failed to build entry %s for host %s" % (setup.filename, + setup.hostname)) + raise + try: + setup.outfile.write( + lxml.etree.tostring(entry, + xml_declaration=False).decode('UTF-8')) + setup.outfile.write("\n") + except IOError: + err = sys.exc_info()[1] + print("Failed to write %s: %s" % (setup.outfile.name, err)) + + +class BuildAllMixin(object): + """ InfoCmd mixin to make a version of an existing command that + applies to multiple hosts""" + + directory_arg = Bcfg2.Options.PathOption("directory") + hostname_arg = Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[]) + options = [directory_arg, hostname_arg] + + @property + def _parent(self): + """ the parent command """ + for cls in self.__class__.__mro__: + if (cls != InfoCmd and cls != self.__class__ and + issubclass(cls, InfoCmd)): + return cls + + def run(self, setup): + try: + os.makedirs(setup.directory) + except OSError: + err = sys.exc_info()[1] + if err.errno != 17: + print("Could not create %s: %s" % (setup.directory, err)) + return 1 + clients = self.get_client_list(setup.hostname) + for client in clients: + csetup = self._get_setup(client, copy.copy(setup)) + csetup.hostname = client + self._parent.run(self, csetup) # pylint: disable=E1101 + + def _get_setup(self, client, setup): + raise NotImplementedError + + +class Buildallfile(Buildfile, BuildAllMixin): + """ Build config file for all clients in directory """ + + options = [BuildAllMixin.directory_arg, + Bcfg2.Options.PathOption("--altsrc"), + Bcfg2.Options.PathOption("filename"), + BuildAllMixin.hostname_arg] + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.outfile = open(os.path.join(setup.directory, client), 'w') + return setup + + +class Buildall(Build, BuildAllMixin): + """ Build configs for all clients in directory """ + + options = BuildAllMixin.options + + def run(self, setup): + BuildAllMixin.run(self, setup) + + def _get_setup(self, client, setup): + setup.filename = os.path.join(setup.directory, client + ".xml") + return setup + + +class Buildbundle(InfoCmd): + """ Render a templated bundle for hostname """ + + options = [Bcfg2.Options.PositionalArgument("bundle"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + bundler = self.core.plugins['Bundler'] + bundle = None + if setup.bundle in bundler.entries: + bundle = bundler.entries[setup.bundle] + elif not setup.bundle.endswith(".xml"): + fname = setup.bundle + ".xml" + if fname in bundler.entries: + bundle = bundler.entries[bundle] + if not bundle: + print("No such bundle %s" % setup.bundle) + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(bundle.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + except: # pylint: disable=W0702 + print("Failed to render bundle %s for host %s: %s" % + (setup.bundle, client, sys.exc_info()[1])) + raise + + +class Automatch(InfoCmd): + """ Perform automatch on a Properties file """ + + options = [ + Bcfg2.Options.BooleanOption( + "-f", "--force", + help="Force automatch even if it's disabled"), + Bcfg2.Options.PositionalArgument("propertyfile"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + props = self.core.plugins['Properties'] + except KeyError: + print("Properties plugin not enabled") + return 1 + + pfile = props.entries[setup.propertyfile] + if (not Bcfg2.Options.setup.force and + not Bcfg2.Options.setup.automatch and + pfile.xdata.get("automatch", "false").lower() != "true"): + print("Automatch not enabled on %s" % setup.propertyfile) + else: + metadata = self.core.build_metadata(setup.hostname) + print(lxml.etree.tostring(pfile.XMLMatch(metadata), + xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class ExpireCache(InfoCmd): + """ Expire the metadata cache """ + + options = [ + Bcfg2.Options.PositionalArgument( + "hostname", nargs="*", default=[], + help="Expire cache for the given host(s)")] + + def run(self, setup): + if setup.clients: + for client in self.get_client_list(setup.clients): + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) + else: + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + + +class Bundles(InfoCmd): + """ Print out group/bundle info """ + + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def run(self, setup): + data = [('Group', 'Bundles')] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + ','.join(self.core.metadata.groups[group][0]))) + print_tabular(data) + + +class Clients(InfoCmd): + """ Print out client/profile info """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*', + default=[])] + + def run(self, setup): + data = [('Client', 'Profile')] + for client in sorted(self.get_client_list(setup.hostname)): + imd = self.core.metadata.get_initial_metadata(client) + data.append((client, imd.profile)) + print_tabular(data) + + +class Config(InfoCmd): + """ Print out the current configuration of Bcfg2""" + + options = [ + Bcfg2.Options.BooleanOption( + "--raw", + help="Produce more accurate but less readable raw output")] + + def run(self, setup): + parser = Bcfg2.Options.get_parser() + data = [('Description', 'Value')] + for option in parser.option_list: + if hasattr(setup, option.dest): + value = getattr(setup, option.dest) + if any(issubclass(a.__class__, + Bcfg2.Options.ComponentAction) + for a in option.actions.values()): + if not setup.raw: + try: + if option.action.islist: + value = [v.__name__ for v in value] + else: + value = value.__name__ + except AttributeError: + # just use the value as-is + pass + if setup.raw: + value = repr(value) + data.append((getattr(option, "help", option.dest), value)) + print_tabular(data) + + +class Probes(InfoCmd): + """ Get probes for the given host """ + + options = [ + Bcfg2.Options.BooleanOption("-p", "--pretty", + help="Human-readable output"), + Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + if setup.pretty: + probes = [] + else: + probes = lxml.etree.Element('probes') + metadata = self.core.build_metadata(setup.hostname) + for plugin in self.core.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for probe in plugin.GetProbes(metadata): + probes.append(probe) + if setup.pretty: + for probe in probes: + pname = probe.get("name") + print("=" * (len(pname) + 2)) + print(" %s" % pname) + print("=" * (len(pname) + 2)) + print("") + print(probe.text) + print("") + else: + print(lxml.etree.tostring(probes, xml_declaration=False, + pretty_print=True).decode('UTF-8')) + + +class Showentries(InfoCmd): + """ Show abstract configuration entries for a given host """ + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("type", nargs='?')] + + def run(self, setup): + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + structures = self.core.GetStructures(metadata) + output = [('Entry Type', 'Name')] + etypes = None + if setup.type: + etypes = [setup.type, "Bound%s" % setup.type] + for item in structures: + output.extend((child.tag, child.get('name')) + for child in item.getchildren() + if not etypes or child.tag in etypes) + print_tabular(output) + + +class Groups(InfoCmd): + """ Print out group info """ + options = [Bcfg2.Options.PositionalArgument("group", nargs='*')] + + def _profile_flag(self, group): + if self.core.metadata.groups[group].is_profile: + return 'yes' + else: + return 'no' + + def run(self, setup): + data = [("Groups", "Profile", "Category")] + groups = self.get_group_list(setup.group) + groups.sort() + for group in groups: + data.append((group, + self._profile_flag(group), + self.core.metadata.groups[group].category)) + print_tabular(data) + + +class Showclient(InfoCmd): + """ Show metadata for the given hosts """ + + options = [Bcfg2.Options.PositionalArgument("hostname", nargs='*')] + + def run(self, setup): + for client in self.get_client_list(setup.hostname): + try: + metadata = self.core.build_metadata(client) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Could not build metadata for %s: %s" % + (client, sys.exc_info()[1])) + continue + fmt = "%-10s %s" + print(fmt % ("Hostname:", metadata.hostname)) + print(fmt % ("Profile:", metadata.profile)) + + group_fmt = "%-10s %-30s %s" + header = False + for group in list(metadata.groups): + category = "" + for cat, grp in metadata.categories.items(): + if grp == group: + category = "Category: %s" % cat + break + if not header: + print(group_fmt % ("Groups:", group, category)) + header = True + else: + print(group_fmt % ("", group, category)) + + if metadata.bundles: + print(fmt % ("Bundles:", list(metadata.bundles)[0])) + for bnd in list(metadata.bundles)[1:]: + print(fmt % ("", bnd)) + if metadata.connectors: + print("Connector data") + print("=" * 80) + for conn in metadata.connectors: + if getattr(metadata, conn): + print(fmt % (conn + ":", getattr(metadata, conn))) + print("=" * 80) + + +class Mappings(InfoCmd): + """ Print generator mappings for optional type and name """ + + options = [Bcfg2.Options.PositionalArgument("type", nargs='?'), + Bcfg2.Options.PositionalArgument("name", nargs='?')] + + def run(self, setup): + data = [('Plugin', 'Type', 'Name')] + for generator in self.core.plugins_by_type( + Bcfg2.Server.Plugin.Generator): + etypes = setup.type or list(generator.Entries.keys()) + if setup.name: + interested = [(etype, [setup.name]) for etype in etypes] + else: + interested = [(etype, generator.Entries[etype]) + for etype in etypes + if etype in generator.Entries] + for etype, names in interested: + data.extend((generator.name, etype, name) + for name in names + if name in generator.Entries.get(etype, {})) + print_tabular(data) + + +class PackageResolve(InfoCmd): + """ Resolve packages for the given host""" + + options = [Bcfg2.Options.PositionalArgument("hostname"), + Bcfg2.Options.PositionalArgument("package", nargs="*")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + + metadata = self.core.build_metadata(setup.hostname) + + indep = lxml.etree.Element("Independent", + name=self.__class__.__name__.lower()) + if setup.package: + structures = [lxml.etree.Element("Bundle", name="packages")] + for package in setup.package: + lxml.etree.SubElement(structures[0], "Package", name=package) + else: + structures = self.core.GetStructures(metadata) + + pkgs._build_packages(metadata, indep, # pylint: disable=W0212 + structures) + print("%d new packages added" % len(indep.getchildren())) + if len(indep.getchildren()): + print(" %s" % "\n ".join(lxml.etree.tostring(p) + for p in indep.getchildren())) + + +class Packagesources(InfoCmd): + """ Show package sources """ + + options = [Bcfg2.Options.PositionalArgument("hostname")] + + def run(self, setup): + try: + pkgs = self.core.plugins['Packages'] + except KeyError: + print("Packages plugin not enabled") + return 1 + try: + metadata = self.core.build_metadata(setup.hostname) + except Bcfg2.Server.Plugin.MetadataConsistencyError: + print("Unable to build metadata for %s: %s" % (setup.hostname, + sys.exc_info()[1])) + return 1 + print(pkgs.get_collection(metadata).sourcelist()) + + +class Query(InfoCmd): + """ Query clients """ + + options = [ + Bcfg2.Options.ExclusiveOptionGroup( + Bcfg2.Options.Option( + "-g", "--group", metavar="<group>", dest="querygroups", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-p", "--profile", metavar="<profile>", dest="queryprofiles", + type=Bcfg2.Options.Types.comma_list), + Bcfg2.Options.Option( + "-b", "--bundle", metavar="<bundle>", dest="querybundles", + type=Bcfg2.Options.Types.comma_list), + required=True)] + + def run(self, setup): + if setup.queryprofiles: + res = self.core.metadata.get_client_names_by_profiles( + setup.queryprofiles) + elif setup.querygroups: + res = self.core.metadata.get_client_names_by_groups( + setup.querygroups) + elif setup.querybundles: + res = self.core.metadata.get_client_names_by_bundles( + setup.querybundles) + print("\n".join(res)) + + +class Shell(InfoCmd): + """ Open an interactive shell to run multiple bcfg2-info commands """ + interactive = False + + def run(self, setup): + try: + self.core.cmdloop('Welcome to bcfg2-info\n' + 'Type "help" for more information') + except KeyboardInterrupt: + print("Ctrl-C pressed, exiting...") + loop = False + + +class ProfileTemplates(InfoCmd): + """ Benchmark template rendering times """ + + options = [ + Bcfg2.Options.Option( + "--clients", type=Bcfg2.Options.Types.comma_list, + help="Benchmark templates for the named clients"), + Bcfg2.Options.Option( + "--runs", help="Number of rendering passes per template", + default=5, type=int), + Bcfg2.Options.PositionalArgument( + "templates", nargs="*", default=[], + help="Profile the named templates instead of all templates")] + + def profile_entry(self, entry, metadata, runs=5): + times = [] + for i in range(runs): # pylint: disable=W0612 + start = time.time() + try: + self.core.Bind(entry, metadata) + times.append(time.time() - start) + except: # pylint: disable=W0702 + break + if times: + avg = sum(times) / len(times) + if avg: + self.logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) + return times + + def profile_struct(self, struct, metadata, templates=None, runs=5): + times = dict() + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates is None or entry.get("name") in templates: + self.logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, + len(entries))) + times.setdefault(entry.get("name"), + self.profile_entry(entry, metadata, + runs=runs)) + return times + + def profile_client(self, metadata, templates=None, runs=5): + structs = self.core.GetStructures(metadata) + struct_count = 0 + times = dict() + for struct in structs: + struct_count += 1 + self.logger.info("Rendering templates from structure %s:%s " + "(%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + times.update(self.profile_struct(struct, metadata, + templates=templates, runs=runs)) + return times + + def stdev(self, nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean) ** 2 for n in nums) / float(len(nums))) + + def run(self, setup): + clients = self.get_client_list(setup.clients) + + times = dict() + client_count = 0 + for client in clients: + client_count += 1 + self.logger.info("Rendering templates for client %s (%s/%s)" % + (client, client_count, len(clients))) + times.update(self.profile_client(self.core.build_metadata(client), + templates=setup.templates, + runs=setup.runs)) + + # print out per-file results + tmpltimes = [] + for tmpl, ptimes in times.items(): + try: + mean = float(sum(ptimes)) / len(ptimes) + except ZeroDivisionError: + continue + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = self.stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or setup.templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + + +if HAS_PROFILE: + class Profile(InfoCmd): + """ Profile a single bcfg2-info command """ + + options = [Bcfg2.Options.PositionalArgument("command"), + Bcfg2.Options.PositionalArgument("args", nargs="*")] + + def run(self, setup): + prof = profile.Profile() + cls = self.core.commands[setup.command] + prof.runcall(cls, " ".join(pipes.quote(a) for a in setup.args)) + display_trace(prof) + + +class InfoCore(cmd.Cmd, + Bcfg2.Server.Core.Core, + Bcfg2.Options.CommandRegistry): + """Main class for bcfg2-info.""" + + def __init__(self): + cmd.Cmd.__init__(self) + Bcfg2.Server.Core.Core.__init__(self) + Bcfg2.Options.CommandRegistry.__init__(self) + self.prompt = 'bcfg2-info> ' + + def get_locals(self): + return locals() + + def do_quit(self, _): + """ quit|exit - Exit program """ + raise SystemExit(0) + + do_EOF = do_quit + do_exit = do_quit + + def do_eventdebug(self, _): + """ eventdebug - Enable debugging output for FAM events """ + self.fam.set_debug(True) + + do_event_debug = do_eventdebug + + def do_update(self, _): + """ update - Process pending filesystem events """ + self.fam.handle_events_in_interval(0.1) + + def run(self): + self.load_plugins() + self.block_for_fam_events(handle_events=True) + + def _daemonize(self): + pass + + def _run(self): + pass + + def _block(self): + pass + + def shutdown(self): + Bcfg2.Options.CommandRegistry.shutdown(self) + Bcfg2.Server.Core.Core.shutdown(self) + + +class CLI(object): + options = [Bcfg2.Options.BooleanOption("-p", "--profile", help="Profile")] + + def __init__(self): + Bcfg2.Options.register_commands(InfoCore, globals().values(), + parent=InfoCmd) + parser = Bcfg2.Options.get_parser( + description="Inspect a running Bcfg2 server", + components=[self, InfoCore]) + parser.parse() + + if Bcfg2.Options.setup.profile and HAS_PROFILE: + prof = profile.Profile() + self.core = prof.runcall(InfoCore) + display_trace(prof) + else: + if Bcfg2.Options.setup.profile: + print("Profiling functionality not available.") + self.core = InfoCore() + + for command in self.core.commands.values(): + command.core = self.core + + def run(self): + if Bcfg2.Options.setup.subcommand != 'help': + self.core.run() + return self.core.runcommand() diff --git a/src/lib/Bcfg2/Server/Lint/Bundler.py b/src/lib/Bcfg2/Server/Lint/Bundler.py new file mode 100644 index 000000000..b41313349 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Bundler.py @@ -0,0 +1,55 @@ +from Bcfg2.Server.Lint import ServerPlugin + + +class Bundler(ServerPlugin): + """ Perform various :ref:`Bundler + <server-plugins-structures-bundler-index>` checks. """ + + def Run(self): + self.missing_bundles() + for bundle in self.core.plugins['Bundler'].entries.values(): + if self.HandlesFile(bundle.name): + self.bundle_names(bundle) + + @classmethod + def Errors(cls): + return {"bundle-not-found": "error", + "unused-bundle": "warning", + "explicit-bundle-name": "error", + "genshi-extension-bundle": "error"} + + def missing_bundles(self): + """ Find bundles listed in Metadata but not implemented in + Bundler. """ + if self.files is None: + # when given a list of files on stdin, this check is + # useless, so skip it + groupdata = self.metadata.groups_xml.xdata + ref_bundles = set([b.get("name") + for b in groupdata.findall("//Bundle")]) + + allbundles = self.core.plugins['Bundler'].bundles.keys() + for bundle in ref_bundles: + if bundle not in allbundles: + self.LintError("bundle-not-found", + "Bundle %s referenced, but does not exist" % + bundle) + + for bundle in allbundles: + if bundle not in ref_bundles: + self.LintError("unused-bundle", + "Bundle %s defined, but is not referenced " + "in Metadata" % bundle) + + def bundle_names(self, bundle): + """ Verify that deprecated bundle .genshi bundles and explicit + bundle names aren't used """ + if bundle.xdata.get('name'): + self.LintError("explicit-bundle-name", + "Deprecated explicit bundle name in %s" % + bundle.name) + + if bundle.name.endswith(".genshi"): + self.LintError("genshi-extension-bundle", + "Bundle %s uses deprecated .genshi extension" % + bundle.name) diff --git a/src/lib/Bcfg2/Server/Lint/Cfg.py b/src/lib/Bcfg2/Server/Lint/Cfg.py new file mode 100644 index 000000000..4cdf5c48a --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Cfg.py @@ -0,0 +1,95 @@ +import os +from Bcfg2.Server.Lint import ServerPlugin + + +class Cfg(ServerPlugin): + """ warn about Cfg issues """ + + def Run(self): + for basename, entry in list(self.core.plugins['Cfg'].entries.items()): + self.check_pubkey(basename, entry) + self.check_missing_files() + + @classmethod + def Errors(cls): + return {"no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error"} + + def check_pubkey(self, basename, entry): + """ check that privkey.xml files have corresponding pubkey.xml + files """ + if "privkey.xml" not in entry.entries: + return + privkey = entry.entries["privkey.xml"] + if not self.HandlesFile(privkey.name): + return + + pubkey = basename + ".pub" + if pubkey not in self.core.plugins['Cfg'].entries: + self.LintError("no-pubkey-xml", + "%s has no corresponding pubkey.xml at %s" % + (basename, pubkey)) + else: + pubset = self.core.plugins['Cfg'].entries[pubkey] + if "pubkey.xml" not in pubset.entries: + self.LintError("no-pubkey-xml", + "%s has no corresponding pubkey.xml at %s" % + (basename, pubkey)) + + def _list_path_components(self, path): + """ Get a list of all components of a path. E.g., + ``self._list_path_components("/foo/bar/foobaz")`` would return + ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed + to be in order.""" + rv = [] + remaining, component = os.path.split(path) + while component != '': + rv.append(component) + remaining, component = os.path.split(remaining) + return rv + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = set() + for hdlr in handlers(): + ignore.update(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + for fname in files: + fpath = os.path.join(root, fname) + # check against the handler ignore patterns and the + # global FAM ignore list + if (not any(fname.endswith("." + i) for i in ignore) and + not any(fnmatch(fpath, p) + for p in self.config['ignore']) and + not any(fnmatch(c, p) + for p in self.config['ignore'] + for c in self._list_path_components(fpath))): + all_files.add(fpath) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index f028e225e..e2d1ec597 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -2,6 +2,7 @@ import os import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Server import XI_NAMESPACE from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \ @@ -16,6 +17,81 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): give information about the files. For instance, you can require SVN keywords in a comment, or require the name of the maintainer of a Genshi template, and so on. """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Option( + cf=("Comments", "global_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for all file types"), + Bcfg2.Options.Option( + cf=("Comments", "global_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for all file types"), + Bcfg2.Options.Option( + cf=("Comments", "bundler_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for non-templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "bundler_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for non-templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "genshibundler_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "genshibundler_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for templated bundles"), + Bcfg2.Options.Option( + cf=("Comments", "properties_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Properties files"), + Bcfg2.Options.Option( + cf=("Comments", "properties_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Properties files"), + Bcfg2.Options.Option( + cf=("Comments", "cfg_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for non-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cfg_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for non-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "genshi_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Genshi-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "genshi_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Genshi-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cheetah_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for Cheetah-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "cheetah_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for Cheetah-templated Cfg files"), + Bcfg2.Options.Option( + cf=("Comments", "infoxml_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for info.xml files"), + Bcfg2.Options.Option( + cf=("Comments", "infoxml_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for info.xml files"), + Bcfg2.Options.Option( + cf=("Comments", "probe_keywords"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required keywords for probes"), + Bcfg2.Options.Option( + cf=("Comments", "probe_comments"), + type=Bcfg2.Options.Types.comma_list, default=[], + help="Required comments for probes")] + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.config_cache = {} @@ -73,17 +149,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): if rtype not in self.config_cache[itype]: rv = [] - global_item = "global_%ss" % itype - if global_item in self.config: - rv.extend(self.config[global_item].split(",")) - - item = "%s_%ss" % (rtype.lower(), itype) - if item in self.config: - if self.config[item]: - rv.extend(self.config[item].split(",")) - else: - # config explicitly specifies nothing - rv = [] + rv.extend(getattr(Bcfg2.Options.setup, "global_%ss" % itype)) + local_reqs = getattr(Bcfg2.Options.setup, + "%s_%ss" % (rtype.lower(), itype)) + if local_reqs == ['']: + # explicitly specified as empty + rv = [] + else: + rv.extend(local_reqs) self.config_cache[itype][rtype] = rv return self.config_cache[itype][rtype] diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index da8da1aa4..76e1986f9 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -18,7 +18,20 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"genshi-syntax-error": "error"} + return {"genshi-syntax-error": "error", + "unknown-genshi-error": "error"} + + def check_template(self, loader, fname, cls=None): + try: + loader.load(fname, cls=cls) + except TemplateSyntaxError: + err = sys.exc_info()[1] + self.LintError("genshi-syntax-error", + "Genshi syntax error in %s: %s" % (fname, err)) + except: + err = sys.exc_info()[1] + self.LintError("unknown-genshi-error", + "Unknown Genshi error in %s: %s" % (fname, err)) def check_cfg(self): """ Check genshi templates in Cfg for syntax errors. """ @@ -27,30 +40,13 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): if (self.HandlesFile(entry.name) and isinstance(entry, CfgGenshiGenerator) and not entry.template): - try: - entry.loader.load(entry.name, - cls=NewTextTemplate) - except TemplateSyntaxError: - err = sys.exc_info()[1] - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) - except: - etype, err = sys.exc_info()[:2] - self.LintError( - "genshi-syntax-error", - "Unexpected Genshi error on %s: %s: %s" % - (entry.name, etype.__name__, err)) + self.check_template(entry.loader, entry.name, + cls=NewTextTemplate) def check_bundler(self): """ Check templates in Bundler for syntax errors. """ loader = TemplateLoader() - for entry in self.core.plugins['Bundler'].entries.values(): if (self.HandlesFile(entry.name) and entry.template is not None): - try: - loader.load(entry.name, cls=MarkupTemplate) - except TemplateSyntaxError: - err = sys.exc_info()[1] - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) + self.check_template(loader, entry.name, cls=MarkupTemplate) diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py index 730f32750..e28080300 100644 --- a/src/lib/Bcfg2/Server/Lint/GroupNames.py +++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py @@ -39,7 +39,8 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): continue xdata = rules.pnode.data self.check_entries(xdata.xpath("//Group"), - os.path.join(self.config['repo'], rules.name)) + os.path.join(Bcfg2.Options.setup.repository, + rules.name)) def check_bundles(self): """ Check groups used in the Bundler plugin for validity. """ @@ -52,7 +53,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin): """ Check groups used or declared in the Metadata plugin for validity. """ self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"), - os.path.join(self.config['repo'], + os.path.join(Bcfg2.Options.setup.repository, self.metadata.groups_xml.name)) def check_grouppatterns(self): diff --git a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py new file mode 100644 index 000000000..8a0ab4f18 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py @@ -0,0 +1,40 @@ +import sys +from Bcfg2.Server.Lint import ServerPlugin +from Bcfg2.Server.Plugins.GroupPatterns import PatternMap + + +class GroupPatterns(ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns + <server-plugins-grouping-grouppatterns>` patterns for validity. + This is simply done by trying to create a + :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for + each pattern, and catching exceptions and presenting them as + ``bcfg2-lint`` errors.""" + + def Run(self): + cfg = self.core.plugins['GroupPatterns'].config + for entry in cfg.xdata.xpath('//GroupPattern'): + groups = [g.text for g in entry.findall('Group')] + self.check(entry, groups, ptype='NamePattern') + self.check(entry, groups, ptype='NameRange') + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, groups, ptype="NamePattern"): + """ Check a single pattern for validity """ + if ptype == "NamePattern": + pmap = lambda p: PatternMap(p, None, groups) + else: + pmap = lambda p: PatternMap(None, p, groups) + + for el in entry.findall(ptype): + pat = el.text + try: + pmap(pat) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + self.LintError("pattern-fails-to-initialize", + "Failed to initialize %s %s for %s: %s" % + (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 184f657b7..4b1513a11 100644 --- a/src/lib/Bcfg2/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -15,6 +15,15 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): * Paranoid mode disabled in an ``info.xml`` file; * Required attributes missing from ``info.xml`` """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Common.default_paranoid, + Bcfg2.Options.Option( + cf=("InfoXML", "required_attrs"), + type=Bcfg2.Options.Types.comma_list, + default=["owner", "group", "mode"], + help="Attributes to require on <Info> tags")] + def Run(self): if 'Cfg' not in self.core.plugins: return @@ -26,7 +35,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): for entry in entryset.entries.values(): if isinstance(entry, CfgInfoXML): self.check_infoxml(infoxml_fname, - entry.infoxml.pnode.data) + entry.infoxml.xdata) found = True if not found: self.LintError("no-infoxml", @@ -42,8 +51,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): """ Verify that info.xml contains everything it should. """ for info in xdata.getroottree().findall("//Info"): required = [] - if "required_attrs" in self.config: - required = self.config["required_attrs"].split(",") + required = Bcfg2.Options.setup.required_attrs missing = [attr for attr in required if info.get(attr) is None] if missing: @@ -52,10 +60,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): (",".join(missing), fname, self.RenderXML(info))) - if ((Bcfg2.Options.MDATA_PARANOID.value and + if ((Bcfg2.Options.setup.default_paranoid == "true" and info.get("paranoid") is not None and info.get("paranoid").lower() == "false") or - (not Bcfg2.Options.MDATA_PARANOID.value and + (Bcfg2.Options.setup.default_paranoid == "false" and (info.get("paranoid") is None or info.get("paranoid").lower() != "true"))): self.LintError("paranoid-false", diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 2419c3d43..dff95fbf3 100644 --- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -8,9 +8,24 @@ import Bcfg2.Server.Lint from Bcfg2.Server.Plugins.Cfg import CfgGenerator +def threshold(val): + """ Option type processor to accept either a percentage (e.g., + "threshold=75") or a ratio (e.g., "threshold=.75") """ + threshold = float(val) + if threshold > 1: + threshold /= 100 + return threshold + + class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ find Probes or Cfg files with multiple similar files that might be merged into one """ + + options = Bcfg2.Server.Lint.ServerPlugin.options + [ + Bcfg2.Options.Option( + cf=("MergeFiles", "threshold"), default="0.75", type=threshold, + help="The threshold at which to suggest merging files and probes")] + def Run(self): if 'Cfg' in self.core.plugins: self.check_cfg() @@ -48,19 +63,10 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ Get a list of similar files from the entry dict. Return value is a list of lists, each of which gives the filenames of similar files """ - if "threshold" in self.config: - # accept threshold either as a percent (e.g., "threshold=75") or - # as a ratio (e.g., "threshold=.75") - threshold = float(self.config['threshold']) - if threshold > 1: - threshold /= 100 - else: - threshold = 0.75 rv = [] elist = list(entries.items()) while elist: - result = self._find_similar(elist.pop(0), copy.copy(elist), - threshold) + result = self._find_similar(elist.pop(0), copy.copy(elist)) if len(result) > 1: elist = [(fname, fdata) for fname, fdata in elist @@ -68,7 +74,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): rv.append(result) return rv - def _find_similar(self, ftuple, others, threshold): + def _find_similar(self, ftuple, others): """ Find files similar to the one described by ftupe in the list of other files. ftuple is a tuple of (filename, data); others is a list of such tuples. threshold is a float between @@ -80,9 +86,9 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): cname, cdata = others.pop(0) seqmatch = SequenceMatcher(None, fdata.data, cdata.data) # perform progressively more expensive comparisons - if (seqmatch.real_quick_ratio() > threshold and - seqmatch.quick_ratio() > threshold and - seqmatch.ratio() > threshold): - rv.extend(self._find_similar((cname, cdata), copy.copy(others), - threshold)) + if (seqmatch.real_quick_ratio() > Bcfg2.Options.setup.threshold and + seqmatch.quick_ratio() > Bcfg2.Options.setup.threshold and + seqmatch.ratio() > Bcfg2.Options.setup.threshold): + rv.extend( + self._find_similar((cname, cdata), copy.copy(others))) return rv diff --git a/src/lib/Bcfg2/Server/Lint/Metadata.py b/src/lib/Bcfg2/Server/Lint/Metadata.py new file mode 100644 index 000000000..a349805fd --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Metadata.py @@ -0,0 +1,148 @@ +from Bcfg2.Server.Lint import ServerPlugin + + +class Metadata(ServerPlugin): + """ ``bcfg2-lint`` plugin for :ref:`Metadata + <server-plugins-grouping-metadata>`. This checks for several things: + + * ``<Client>`` tags nested inside other ``<Client>`` tags; + * Deprecated options (like ``location="floating"``); + * Profiles that don't exist, or that aren't profile groups; + * Groups or clients that are defined multiple times; + * Multiple default groups or a default group that isn't a profile + group. + """ + + def Run(self): + self.nested_clients() + self.deprecated_options() + self.bogus_profiles() + self.duplicate_groups() + self.duplicate_default_groups() + self.duplicate_clients() + self.default_is_profile() + + @classmethod + def Errors(cls): + return {"nested-client-tags": "warning", + "deprecated-clients-options": "warning", + "nonexistent-profile-group": "error", + "non-profile-set-as-profile": "error", + "duplicate-group": "error", + "duplicate-client": "error", + "multiple-default-groups": "error", + "default-is-not-profile": "error"} + + def deprecated_options(self): + """ Check for the ``location='floating'`` option, which has + been deprecated in favor of ``floating='true'``. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + clientdata = self.metadata.clients_xml.xdata + for el in clientdata.xpath("//Client"): + loc = el.get("location") + if loc: + if loc == "floating": + floating = True + else: + floating = False + self.LintError("deprecated-clients-options", + "The location='%s' option is deprecated. " + "Please use floating='%s' instead:\n%s" % + (loc, floating, self.RenderXML(el))) + + def nested_clients(self): + """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, + which is either redundant or will never match. """ + groupdata = self.metadata.groups_xml.xdata + for el in groupdata.xpath("//Client//Client"): + self.LintError("nested-client-tags", + "Client %s nested within Client tag: %s" % + (el.get("name"), self.RenderXML(el))) + + def bogus_profiles(self): + """ Check for clients that have profiles that are either not + flagged as profile groups in ``groups.xml``, or don't exist. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + for client in self.metadata.clients_xml.xdata.findall('.//Client'): + profile = client.get("profile") + if profile not in self.metadata.groups: + self.LintError("nonexistent-profile-group", + "%s has nonexistent profile group %s:\n%s" % + (client.get("name"), profile, + self.RenderXML(client))) + elif not self.metadata.groups[profile].is_profile: + self.LintError("non-profile-set-as-profile", + "%s is set as profile for %s, but %s is not a " + "profile group:\n%s" % + (profile, client.get("name"), profile, + self.RenderXML(client))) + + def duplicate_default_groups(self): + """ Check for multiple default groups. """ + defaults = [] + for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): + if grp.get("default", "false").lower() == "true": + defaults.append(self.RenderXML(grp)) + if len(defaults) > 1: + self.LintError("multiple-default-groups", + "Multiple default groups defined:\n%s" % + "\n".join(defaults)) + + def duplicate_clients(self): + """ Check for clients that are defined more than once. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return + self.duplicate_entries( + self.metadata.clients_xml.xdata.xpath("//Client"), + "client") + + def duplicate_groups(self): + """ Check for groups that are defined more than once. We + count a group tag as a definition if it a) has profile or + public set; or b) has any children.""" + allgroups = [ + g + for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") + if g.get("profile") or g.get("public") or g.getchildren()] + self.duplicate_entries(allgroups, "group") + + def duplicate_entries(self, allentries, etype): + """ Generic duplicate entry finder. + + :param allentries: A list of all entries to check for + duplicates. + :type allentries: list of lxml.etree._Element + :param etype: The entry type. This will be used to determine + the error name (``duplicate-<etype>``) and for + display to the end user. + :type etype: string + """ + entries = dict() + for el in allentries: + if el.get("name") in entries: + entries[el.get("name")].append(self.RenderXML(el)) + else: + entries[el.get("name")] = [self.RenderXML(el)] + for ename, els in entries.items(): + if len(els) > 1: + self.LintError("duplicate-%s" % etype, + "%s %s is defined multiple times:\n%s" % + (etype.title(), ename, "\n".join(els))) + + def default_is_profile(self): + """ Ensure that the default group is a profile group. """ + if (self.metadata.default and + not self.metadata.groups[self.metadata.default].is_profile): + xdata = \ + self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % + self.metadata.default)[0] + self.LintError("default-is-not-profile", + "Default group is not a profile group:\n%s" % + self.RenderXML(xdata)) diff --git a/src/lib/Bcfg2/Server/Lint/Pkgmgr.py b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py new file mode 100644 index 000000000..54f6f07d1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py @@ -0,0 +1,46 @@ +import os +import glob +import lxml.etree +import Bcfg2.Options +from Bcfg2.Server.Lint import ServerlessPlugin + + +class Pkgmgr(ServerlessPlugin): + """ Find duplicate :ref:`Pkgmgr + <server-plugins-generators-pkgmgr>` entries with the same + priority. """ + + def Run(self): + pset = set() + for pfile in glob.glob(os.path.join(Bcfg2.Options.setup.repository, + 'Pkgmgr', '*.xml')): + if self.HandlesFile(pfile): + xdata = lxml.etree.parse(pfile).getroot() + # get priority, type, group + priority = xdata.get('priority') + ptype = xdata.get('type') + for pkg in xdata.xpath("//Package"): + if pkg.getparent().tag == 'Group': + grp = pkg.getparent().get('name') + if (type(grp) is not str and + grp.getparent().tag == 'Group'): + pgrp = grp.getparent().get('name') + else: + pgrp = 'none' + else: + grp = 'none' + pgrp = 'none' + ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) + # check if package is already listed with same + # priority, type, grp + if ptuple in pset: + self.LintError( + "duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) + else: + pset.add(ptuple) + + @classmethod + def Errors(cls): + return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 3bf76765b..cf7b51ecc 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -167,8 +167,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): for rules in self.core.plugins['Rules'].entries.values(): xdata = rules.pnode.data for path in xdata.xpath("//Path"): - self.check_entry(path, os.path.join(self.config['repo'], - rules.name)) + self.check_entry(path, + os.path.join(Bcfg2.Options.setup.repository, + rules.name)) def check_bundles(self): """ Check bundles for BoundPath entries with missing diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py new file mode 100644 index 000000000..a24d70cab --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -0,0 +1,77 @@ +import sys +import imp +from Bcfg2.Server.Lint import ServerPlugin +from Bcfg2.Server.Plugins.TemplateHelper import HelperModule, MODULE_RE, \ + safe_module_name + + +class TemplateHelperLint(ServerPlugin): + """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper + <server-plugins-connectors-templatehelper>` modules are valid. + This can check for: + + * A TemplateHelper module that cannot be imported due to syntax or + other compile-time errors; + * A TemplateHelper module that does not have an ``__export__`` + attribute, or whose ``__export__`` is not a list; + * Bogus symbols listed in ``__export__``, including symbols that + don't exist, that are reserved, or that start with underscores. + """ + + def __init__(self, *args, **kwargs): + ServerPlugin.__init__(self, *args, **kwargs) + self.reserved_keywords = dir(HelperModule("foo.py")) + + def Run(self): + for helper in self.core.plugins['TemplateHelper'].entries.values(): + if self.HandlesFile(helper.name): + self.check_helper(helper.name) + + def check_helper(self, helper): + """ Check a single helper module. + + :param helper: The filename of the helper module + :type helper: string + """ + module_name = MODULE_RE.search(helper).group(1) + + try: + module = imp.load_source(safe_module_name(module_name), helper) + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + self.LintError("templatehelper-import-error", + "Failed to import %s: %s" % + (helper, err)) + return + + if not hasattr(module, "__export__"): + self.LintError("templatehelper-no-export", + "%s has no __export__ list" % helper) + return + elif not isinstance(module.__export__, list): + self.LintError("templatehelper-nonlist-export", + "__export__ is not a list in %s" % helper) + return + + for sym in module.__export__: + if not hasattr(module, sym): + self.LintError("templatehelper-nonexistent-export", + "%s: exported symbol %s does not exist" % + (helper, sym)) + elif sym in self.reserved_keywords: + self.LintError("templatehelper-reserved-export", + "%s: exported symbol %s is reserved" % + (helper, sym)) + elif sym.startswith("_"): + self.LintError("templatehelper-underscore-export", + "%s: exported symbol %s starts with underscore" + % (helper, sym)) + + @classmethod + def Errors(cls): + return {"templatehelper-import-error": "error", + "templatehelper-no-export": "error", + "templatehelper-nonlist-export": "error", + "templatehelper-nonexistent-export": "error", + "templatehelper-reserved-export": "error", + "templatehelper-underscore-export": "warning"} diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index ca9f138ef..2f245561b 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -6,6 +6,7 @@ import sys import glob import fnmatch import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Lint from Bcfg2.Utils import Executor @@ -14,6 +15,12 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that all XML files in the Bcfg2 repository validate according to their respective schemas. """ + options = Bcfg2.Server.Lint.ServerlessPlugin.options + [ + Bcfg2.Options.PathOption( + "--schema", cf=("Validate", "schema"), + default="/usr/share/bcfg2/schema", + help="The full path to the XML schema files")] + def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) @@ -58,7 +65,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.cmd = Executor() def Run(self): - schemadir = self.config['schema'] for path, schemaname in self.filesets.items(): try: @@ -68,7 +74,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): if filelist: # avoid loading schemas for empty file lists - schemafile = os.path.join(schemadir, schemaname) + schemafile = os.path.join(Bcfg2.Options.setup.schema, + schemaname) schema = self._load_schema(schemafile) if schema: for filename in filelist: @@ -165,8 +172,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): listfiles = lambda p: fnmatch.filter(self.files, os.path.join('*', p)) else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) + listfiles = lambda p: \ + glob.glob(os.path.join(Bcfg2.Options.setup.repository, p)) for path in self.filesets.keys(): if '/**/' in path: @@ -175,9 +182,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): else: # self.files is None fpath, fname = path.split('/**/') self.filelists[path] = [] - for root, _, files in \ - os.walk(os.path.join(self.config['repo'], - fpath)): + for root, _, files in os.walk( + os.path.join(Bcfg2.Options.setup.repository, fpath)): self.filelists[path].extend([os.path.join(root, f) for f in files if f == fname]) diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 28644263f..4f64fd006 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -2,16 +2,17 @@ import os import sys +import time +import copy +import fcntl +import struct +import termios import logging -from copy import copy import textwrap import lxml.etree -import fcntl -import termios -import struct -from Bcfg2.Compat import walk_packages - -plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 +import Bcfg2.Options +import Bcfg2.Server.Core +import Bcfg2.Server.Plugins def _ioctl_GWINSZ(fd): # pylint: disable=C0103 @@ -46,10 +47,10 @@ def get_termsize(): class Plugin(object): """ Base class for all bcfg2-lint plugins """ - def __init__(self, config, errorhandler=None, files=None): + options = [Bcfg2.Options.Common.repository] + + def __init__(self, errorhandler=None, files=None): """ - :param config: A :mod:`Bcfg2.Options` setup dict - :type config: dict :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` that will be used to handle lint errors. If one is not provided, a new one will be @@ -63,9 +64,6 @@ class Plugin(object): #: The list of files that bcfg2-lint should be run against self.files = files - #: The Bcfg2.Options setup dict - self.config = config - self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: #: The error handler @@ -96,9 +94,10 @@ class Plugin(object): False otherwise. """ return (self.files is None or fname in self.files or - os.path.join(self.config['repo'], fname) in self.files or + os.path.join(Bcfg2.Options.setup.repository, + fname) in self.files or os.path.abspath(fname) in self.files or - os.path.abspath(os.path.join(self.config['repo'], + os.path.abspath(os.path.join(Bcfg2.Options.setup.repository, fname)) in self.files) def LintError(self, err, msg): @@ -125,7 +124,7 @@ class Plugin(object): """ xml = None if len(element) or element.text: - el = copy(element) + el = copy.copy(element) if el.text and not keep_text: el.text = '...' for child in el.iterchildren(): @@ -145,8 +144,8 @@ class ErrorHandler(object): def __init__(self, errors=None): """ - :param config: An initial dict of errors to register - :type config: dict + :param errors: An initial dict of errors to register + :type errors: dict """ #: The number of errors passed to this error handler self.errors = 0 @@ -267,12 +266,10 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 """ Base class for bcfg2-lint plugins that check things that require the running Bcfg2 server. """ - def __init__(self, core, config, errorhandler=None, files=None): + def __init__(self, core, errorhandler=None, files=None): """ :param core: The Bcfg2 server core :type core: Bcfg2.Server.Core.BaseCore - :param config: A :mod:`Bcfg2.Options` setup dict - :type config: dict :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler` that will be used to handle lint errors. If one is not provided, a new one will be @@ -282,7 +279,7 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 the bcfg2-lint ``--stdin`` option.) :type files: list of strings """ - Plugin.__init__(self, config, errorhandler=errorhandler, files=files) + Plugin.__init__(self, errorhandler=errorhandler, files=files) #: The server core self.core = core @@ -290,3 +287,166 @@ class ServerPlugin(Plugin): # pylint: disable=W0223 #: The metadata plugin self.metadata = self.core.metadata + + +class LintPluginAction(Bcfg2.Options.ComponentAction): + """ We want to load all lint plugins that pertain to server + plugins. In order to do this, we hijack the __call__() method of + this action and add all of the server plugins on the fly """ + + bases = ['Bcfg2.Server.Lint'] + + def __call__(self, parser, namespace, values, option_string=None): + for plugin in getattr(Bcfg2.Options.setup, "plugins", []): + module = sys.modules[plugin.__module__] + if hasattr(module, "%sLint" % plugin.name): + print("Adding lint plugin %s" % plugin) + values.append(plugin) + Bcfg2.Options.ComponentAction.__call__(self, parser, namespace, values, + option_string) + + +class CLI(object): + """ The bcfg2-lint CLI """ + options = Bcfg2.Server.Core.Core.options + [ + Bcfg2.Options.PathOption( + '--lint-config', default='/etc/bcfg2-lint.conf', + action=Bcfg2.Options.ConfigFileAction, + help='Specify bcfg2-lint configuration file'), + Bcfg2.Options.Option( + "--lint-plugins", cf=('lint', 'plugins'), + type=Bcfg2.Options.Types.comma_list, action=LintPluginAction, + help='bcfg2-lint plugin list'), + Bcfg2.Options.BooleanOption( + '--list-errors', help='Show error handling'), + Bcfg2.Options.BooleanOption( + '--stdin', help='Operate on a list of files supplied on stdin'), + Bcfg2.Options.Option( + cf=("errors", '*'), dest="lint_errors", + help="How to handle bcfg2-lint errors")] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Manage a running Bcfg2 server", + components=[self]) + parser.parse() + + self.logger = logging.getLogger(parser.prog) + + # automatically add Lint plugins for loaded server plugins + for plugin in Bcfg2.Options.setup.plugins: + try: + Bcfg2.Options.setup.lint_plugins.append( + getattr( + __import__("Bcfg2.Server.Lint.%s" % plugin.__name__, + fromlist=[plugin.__name__]), + plugin.__name__)) + self.logger.debug("Automatically adding lint plugin %s" % + plugin.__name__) + except ImportError: + # no lint plugin for this server plugin + self.logger.debug("No lint plugin for %s" % plugin.__name__) + pass + except AttributeError: + self.logger.error("Failed to load plugin %s: %s" % + (plugin.__name__, sys.exc_info()[1])) + + self.logger.debug("Running lint with plugins: %s" % + [p.__name__ + for p in Bcfg2.Options.setup.lint_plugins]) + + if Bcfg2.Options.setup.stdin: + self.files = [s.strip() for s in sys.stdin.readlines()] + else: + self.files = None + self.errorhandler = self.get_errorhandler() + self.serverlessplugins = [] + self.serverplugins = [] + for plugin in Bcfg2.Options.setup.lint_plugins: + if issubclass(plugin, ServerPlugin): + self.serverplugins.append(plugin) + else: + self.serverlessplugins.append(plugin) + + def run(self): + if Bcfg2.Options.setup.list_errors: + for plugin in self.serverplugins + self.serverlessplugins: + self.errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) + + print("%-35s %-35s" % ("Error name", "Handler")) + for err, handler in self.errorhandler.errortypes.items(): + print("%-35s %-35s" % (err, handler.__name__)) + return 0 + + if not self.serverplugins and not self.serverlessplugins: + self.logger.error("No lint plugins loaded!") + return 1 + + self.run_serverless_plugins() + + if self.serverplugins: + if self.errorhandler.errors: + # it would be swell if we could try to start the server + # even if there were errors with the serverless plugins, + # but since XML parsing errors occur in the FAM thread + # (not in the core server thread), there's no way we can + # start the server and try to catch exceptions -- + # bcfg2-lint isn't in the same stack as the exceptions. + # so we're forced to assume that a serverless plugin error + # will prevent the server from starting + print("Serverless plugins encountered errors, skipping server " + "plugins") + else: + self.run_server_plugins() + + if (self.errorhandler.errors or + self.errorhandler.warnings or + Bcfg2.Options.setup.verbose): + print("%d errors" % self.errorhandler.errors) + print("%d warnings" % self.errorhandler.warnings) + + if self.errorhandler.errors: + return 2 + elif self.errorhandler.warnings: + return 3 + else: + return 0 + + def get_errorhandler(self): + """ get a Bcfg2.Server.Lint.ErrorHandler object """ + return Bcfg2.Server.Lint.ErrorHandler( + errors=Bcfg2.Options.setup.lint_errors) + + def run_serverless_plugins(self): + """ Run serverless plugins """ + self.logger.debug("Running serverless plugins: %s" % + [p.__name__ for p in self.serverlessplugins]) + for plugin in self.serverlessplugins: + self.logger.debug(" Running %s" % plugin.__name__) + plugin(files=self.files, errorhandler=self.errorhandler).Run() + + def run_server_plugins(self): + """ run plugins that require a running server to run """ + core = Bcfg2.Server.Core.Core() + core.load_plugins() + core.block_for_fam_events(handle_events=True) + try: + self.logger.debug("Running server plugins: %s" % + [p.__name__ for p in self.serverplugins]) + for plugin in self.serverplugins: + self.logger.debug(" Running %s" % plugin.__name__) + plugin(core, + files=self.files, errorhandler=self.errorhandler).Run() + finally: + core.shutdown() + + def _run_plugin(self, plugin, args=None): + if args is None: + args = [] + start = time.time() + # python 2.5 doesn't support mixing *magic and keyword arguments + kwargs = dict(files=self.files, errorhandler=self.errorhandler) + rv = plugin(*args, **kwargs).Run() + self.logger.debug(" Ran %s in %0.2f seconds" % (plugin.__name__, + time.time() - start)) + return rv diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index e79207291..678a1c95d 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -15,12 +15,13 @@ import time import threading import lxml.etree import multiprocessing +import Bcfg2.Options import Bcfg2.Server.Plugin from itertools import cycle from Bcfg2.Cache import Cache from Bcfg2.Compat import Queue, Empty, wraps -from Bcfg2.Server.Core import BaseCore, exposed -from Bcfg2.Server.BuiltinCore import Core as BuiltinCore +from Bcfg2.Server.Core import Core, exposed +from Bcfg2.Server.BuiltinCore import BuiltinCore from multiprocessing.connection import Listener, Client @@ -167,7 +168,7 @@ class DualEvent(object): return self._threading_event.wait(timeout=timeout) -class ChildCore(BaseCore): +class ChildCore(Core): """ A child process for :class:`Bcfg2.MultiprocessingCore.Core`. This core builds configurations from a given :class:`multiprocessing.Pipe`. Note that this is a full-fledged @@ -186,12 +187,10 @@ class ChildCore(BaseCore): #: every ``poll_wait`` seconds. poll_wait = 3.0 - def __init__(self, name, setup, rpc_q, terminate): + def __init__(self, name, rpc_q, terminate): """ :param name: The name of this child :type name: string - :param setup: A Bcfg2 options dict - :type setup: Bcfg2.Options.OptionParser :param read_q: The queue the child will read from for RPC communications from the parent process. :type read_q: multiprocessing.Queue @@ -202,7 +201,7 @@ class ChildCore(BaseCore): themselves down. :type terminate: multiprocessing.Event """ - BaseCore.__init__(self, setup) + Core.__init__(self) #: The name of this child self.name = name @@ -216,7 +215,7 @@ class ChildCore(BaseCore): # override this setting so that the child doesn't try to write # the pidfile - self.setup['daemon'] = False + Bcfg2.Options.setup.daemon = False # ensure that the child doesn't start a perflog thread self.perflog_thread = None @@ -283,7 +282,7 @@ class ChildCore(BaseCore): self.shutdown() def shutdown(self): - BaseCore.shutdown(self) + Core.shutdown(self) self.logger.info("%s: Closing RPC command queue" % self.name) self.rpc_q.close() @@ -328,7 +327,7 @@ class ChildCore(BaseCore): return lxml.etree.tostring(self.BuildConfiguration(client)) -class Core(BuiltinCore): +class MultiprocessingCore(BuiltinCore): """ A multiprocessing core that delegates building the actual client configurations to :class:`Bcfg2.Server.MultiprocessingCore.ChildCore` objects. The @@ -336,14 +335,34 @@ class Core(BuiltinCore): :func:`GetConfig` are delegated to children. All other calls are handled by the parent process. """ + options = BuiltinCore.options + [ + Bcfg2.Options.Option( + '--children', dest="core_children", + cf=('server', 'children'), type=int, + default=multiprocessing.cpu_count(), + help='Spawn this number of children for the multiprocessing core')] + #: How long to wait for a child process to shut down cleanly #: before it is terminated. shutdown_timeout = 10.0 - def __init__(self, setup): - BuiltinCore.__init__(self, setup) - if setup['children'] is None: - setup['children'] = multiprocessing.cpu_count() + def __init__(self): + BuiltinCore.__init__(self) + + #: A dict of child name -> one end of the + #: :class:`multiprocessing.Pipe` object used to communicate + #: with that child. (The child is given the other end of the + #: Pipe.) + self.pipes = dict() + + #: A queue that keeps track of which children are available to + #: render a configuration. A child is popped from the queue + #: when it starts to render a config, then it's pushed back on + #: when it's done. This lets us use a blocking call to + #: :func:`Queue.Queue.get` when waiting for an available + #: child. + self.available_children = \ + Queue(maxsize=Bcfg2.Options.setup.core_children) #: The flag that indicates when to stop child threads and #: processes @@ -363,12 +382,12 @@ class Core(BuiltinCore): self.children = None def _run(self): - for cnum in range(self.setup['children']): + for cnum in range(Bcfg2.Options.setup.core_children): name = "Child-%s" % cnum self.logger.debug("Starting child %s" % name) child_q = self.rpc_q.add_subscriber(name) - childcore = ChildCore(name, self.setup, child_q, self.terminate) + childcore = ChildCore(name, child_q, self.terminate) child = multiprocessing.Process(target=childcore.run, name=name) child.start() self.logger.debug("Child %s started with PID %s" % (name, diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py index ed1282ba0..a85867134 100644 --- a/src/lib/Bcfg2/Server/Plugin/__init__.py +++ b/src/lib/Bcfg2/Server/Plugin/__init__.py @@ -14,6 +14,7 @@ documentation it's not necessary to use the submodules. E.g., you can import os import sys +import Bcfg2.Options sys.path.append(os.path.dirname(__file__)) # pylint: disable=W0401 @@ -21,3 +22,31 @@ from Bcfg2.Server.Plugin.base import * from Bcfg2.Server.Plugin.interfaces import * from Bcfg2.Server.Plugin.helpers import * from Bcfg2.Server.Plugin.exceptions import * + + +class _OptionContainer(object): + options = [ + Bcfg2.Options.Common.default_paranoid, + Bcfg2.Options.Option( + cf=('mdata', 'owner'), dest="default_owner", default='root', + help='Default Path owner'), + Bcfg2.Options.Option( + cf=('mdata', 'group'), dest="default_group", default='root', + help='Default Path group'), + Bcfg2.Options.Option( + cf=('mdata', 'important'), dest="default_important", + default='false', choices=['true', 'false'], + help='Default Path priority (importance)'), + Bcfg2.Options.Option( + cf=('mdata', 'mode'), dest="default_mode", default='644', + help='Default mode for Path'), + Bcfg2.Options.Option( + cf=('mdata', 'secontext'), dest="default_secontext", + default='__default__', help='Default SELinux context'), + Bcfg2.Options.Option( + cf=('mdata', 'sensitive'), dest="default_sensitive", + default='false', + help='Default Path sensitivity setting')] + + +Bcfg2.Options.get_parser().add_component(_OptionContainer) diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 03feceb6f..b2d9fa7c8 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -1,66 +1,10 @@ """This module provides the base class for Bcfg2 server plugins.""" import os -import logging +from Bcfg2.Logger import Debuggable from Bcfg2.Utils import ClassName -class Debuggable(object): - """ Mixin to add a debugging interface to an object and expose it - via XML-RPC on :class:`Bcfg2.Server.Plugin.base.Plugin` objects """ - - #: List of names of methods to be exposed as XML-RPC functions - __rmi__ = ['toggle_debug', 'set_debug'] - - #: How exposed XML-RPC functions should be dispatched to child - #: processes. - __child_rmi__ = __rmi__[:] - - def __init__(self, name=None): - """ - :param name: The name of the logger object to get. If none is - supplied, the full name of the class (including - module) will be used. - :type name: string - - .. autoattribute:: __rmi__ - """ - if name is None: - name = "%s.%s" % (self.__class__.__module__, - self.__class__.__name__) - self.debug_flag = False - self.logger = logging.getLogger(name) - - def set_debug(self, debug): - """ Explicitly enable or disable debugging. This method is exposed - via XML-RPC. - - :returns: bool - The new value of the debug flag - """ - self.debug_flag = debug - return debug - - def toggle_debug(self): - """ Turn debugging output on or off. This method is exposed - via XML-RPC. - - :returns: bool - The new value of the debug flag - """ - return self.set_debug(not self.debug_flag) - - def debug_log(self, message, flag=None): - """ Log a message at the debug level. - - :param message: The message to log - :type message: string - :param flag: Override the current debug flag with this value - :type flag: bool - :returns: None - """ - if (flag is None and self.debug_flag) or flag: - self.logger.error(message) - - class Plugin(Debuggable): """ The base class for all Bcfg2 Server plugins. """ diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index a63e9c5f7..0266af909 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -13,8 +13,10 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Options import Bcfg2.Server.FileMonitor +from Bcfg2.Utils import ClassName +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import CmpMixin, wraps -from Bcfg2.Server.Plugin.base import Debuggable, Plugin +from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.interfaces import Generator from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ PluginExecutionError @@ -31,63 +33,8 @@ try: except ImportError: HAS_DJANGO = False -#: A dict containing default metadata for Path entries from bcfg2.conf -DEFAULT_FILE_METADATA = Bcfg2.Options.OptionParser( - dict(configfile=Bcfg2.Options.CFILE, - owner=Bcfg2.Options.MDATA_OWNER, - group=Bcfg2.Options.MDATA_GROUP, - mode=Bcfg2.Options.MDATA_MODE, - secontext=Bcfg2.Options.MDATA_SECONTEXT, - important=Bcfg2.Options.MDATA_IMPORTANT, - paranoid=Bcfg2.Options.MDATA_PARANOID, - sensitive=Bcfg2.Options.MDATA_SENSITIVE)) -DEFAULT_FILE_METADATA.parse([Bcfg2.Options.CFILE.cmd, Bcfg2.Options.CFILE]) -del DEFAULT_FILE_METADATA['args'] -del DEFAULT_FILE_METADATA['configfile'] - LOGGER = logging.getLogger(__name__) -#: a compiled regular expression for parsing info and :info files -INFO_REGEX = re.compile(r'owner:\s*(?P<owner>\S+)|' + - r'group:\s*(?P<group>\S+)|' + - r'mode:\s*(?P<mode>\w+)|' + - r'secontext:\s*(?P<secontext>\S+)|' + - r'paranoid:\s*(?P<paranoid>\S+)|' + - r'sensitive:\s*(?P<sensitive>\S+)|' + - r'encoding:\s*(?P<encoding>\S+)|' + - r'important:\s*(?P<important>\S+)|' + - r'mtime:\s*(?P<mtime>\w+)') - - -def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): - """ Bind the file metadata in the given - :class:`Bcfg2.Server.Plugin.helpers.InfoXML` object to the given - entry. - - :param entry: The abstract entry to bind the info to - :type entry: lxml.etree._Element - :param metadata: The client metadata to get info for - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param infoxml: The info.xml file to pull file metadata from - :type infoxml: Bcfg2.Server.Plugin.helpers.InfoXML - :param default: Default metadata to supply when the info.xml file - does not include a particular attribute - :type default: dict - :returns: None - :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` - """ - for attr, val in list(default.items()): - entry.set(attr, val) - if infoxml: - mdata = dict() - infoxml.pnode.Match(metadata, mdata, entry=entry) - if 'Info' not in mdata: - msg = "Failed to set metadata for file %s" % entry.get('name') - LOGGER.error(msg) - raise PluginExecutionError(msg) - for attr, val in list(mdata['Info'][None].items()): - entry.set(attr, val) - class track_statistics(object): # pylint: disable=C0103 """ Decorator that tracks execution time for the given @@ -139,53 +86,76 @@ def removecomment(stream): yield kind, data, pos +def bind_info(entry, metadata, infoxml=None, default=None): + """ Bind the file metadata in the given + :class:`Bcfg2.Server.Plugin.helpers.InfoXML` object to the given + entry. + + :param entry: The abstract entry to bind the info to + :type entry: lxml.etree._Element + :param metadata: The client metadata to get info for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :param infoxml: The info.xml file to pull file metadata from + :type infoxml: Bcfg2.Server.Plugin.helpers.InfoXML + :param default: Default metadata to supply when the info.xml file + does not include a particular attribute + :type default: dict + :returns: None + :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginExecutionError` + """ + if default is None: + default = default_path_metadata() + for attr, val in list(default.items()): + entry.set(attr, val) + if infoxml: + mdata = dict() + infoxml.pnode.Match(metadata, mdata, entry=entry) + if 'Info' not in mdata: + msg = "Failed to set metadata for file %s" % entry.get('name') + LOGGER.error(msg) + raise PluginExecutionError(msg) + for attr, val in list(mdata['Info'][None].items()): + entry.set(attr, val) + + def default_path_metadata(): """ Get the default Path entry metadata from the config. :returns: dict of metadata attributes and their default values """ - attrs = Bcfg2.Options.PATH_METADATA_OPTIONS.keys() - setup = Bcfg2.Options.get_option_parser() - if not set(attrs).issubset(setup.keys()): - setup.add_options(Bcfg2.Options.PATH_METADATA_OPTIONS) - setup.reparse(argv=[Bcfg2.Options.CFILE.cmd, Bcfg2.Options.CFILE]) - return dict([(k, setup[k]) for k in attrs]) + return dict([(k, getattr(Bcfg2.Options.setup, "default_%s" % k)) + for k in ['owner', 'group', 'mode', 'secontext', 'important', + 'paranoid', 'sensitive']]) class DatabaseBacked(Plugin): """ Provides capabilities for a plugin to read and write to a - database. + database. The plugin must add an option to flag database use with + something like: + + options = Bcfg2.Server.Plugin.Plugins.options + [ + Bcfg2.Options.BooleanOption( + cf=('metadata', 'use_database'), dest="metadata_db", + help="Use database capabilities of the Metadata plugin") + + This must be done manually due to various limitations in Python. .. private-include: _use_db .. private-include: _must_lock """ - #: The option to look up in :attr:`section` to determine whether or - #: not to use the database capabilities of this plugin. The option - #: is retrieved with - #: :py:func:`ConfigParser.SafeConfigParser.getboolean`, and so must - #: conform to the possible values that function can handle. - option = "use_database" - - def _section(self): - """ The section to look in for :attr:`DatabaseBacked.option` - """ - return self.name.lower() - section = property(_section) - @property def _use_db(self): """ Whether or not this plugin is configured to use the database. """ - use_db = self.core.setup.cfp.getboolean(self.section, - self.option, - default=False) + use_db = getattr(Bcfg2.Options.setup, "%s_db" % self.name.lower(), + False) if use_db and HAS_DJANGO and self.core.database_available: return True elif not use_db: return False else: - self.logger.error("%s is true but django not found" % self.option) + self.logger.error("use_database is true but django not found") return False @property @@ -193,11 +163,7 @@ class DatabaseBacked(Plugin): """ Whether or not the backend database must acquire a thread lock before writing, because it does not allow multiple threads to write.""" - engine = \ - self.core.setup.cfp.get(Bcfg2.Options.DB_ENGINE.cf[0], - Bcfg2.Options.DB_ENGINE.cf[1], - default=Bcfg2.Options.DB_ENGINE.default) - return engine == 'sqlite3' + return Bcfg2.Options.setup.db_engine == 'sqlite3' @staticmethod def get_db_lock(func): @@ -679,10 +645,9 @@ class StructFile(XMLFileBacked): dict(Group=lambda el, md, *args: el.get('name') in md.groups, Client=lambda el, md, *args: el.get('name') == md.hostname) - def __init__(self, filename, should_monitor=False): - XMLFileBacked.__init__(self, filename, should_monitor=should_monitor) - self.setup = Bcfg2.Options.get_option_parser() - self.encoding = self.setup['encoding'] + def __init__(self, filename, should_monitor=False, create=None): + XMLFileBacked.__init__(self, filename, should_monitor=should_monitor, + create=create) self.template = None def Index(self): @@ -692,9 +657,10 @@ class StructFile(XMLFileBacked): self.xdata.nsmap['py'] == 'http://genshi.edgewall.org/')): try: loader = genshi.template.TemplateLoader() - self.template = loader.load(self.name, - cls=genshi.template.MarkupTemplate, - encoding=self.encoding) + self.template = \ + loader.load(self.name, + cls=genshi.template.MarkupTemplate, + encoding=Bcfg2.Options.setup.encoding) except LookupError: err = sys.exc_info()[1] self.logger.error('Genshi lookup error in %s: %s' % (self.name, @@ -709,10 +675,9 @@ class StructFile(XMLFileBacked): err)) if HAS_CRYPTO: - strict = self.xdata.get( - "decrypt", - self.setup.cfp.get(Bcfg2.Server.Encryption.CFG_SECTION, - "decrypt", default="strict")) == "strict" + lax_decrypt = self.xdata.get( + "lax_decryption", + str(Bcfg2.Options.setup.lax_decryption)).lower() == "true" for el in self.xdata.xpath("//*[@encrypted]"): try: el.text = self._decrypt(el).encode('ascii', @@ -723,17 +688,17 @@ class StructFile(XMLFileBacked): except Bcfg2.Server.Encryption.EVPError: msg = "Failed to decrypt %s element in %s" % (el.tag, self.name) - if strict: - raise PluginExecutionError(msg) - else: + if lax_decrypt: self.logger.warning(msg) + else: + raise PluginExecutionError(msg) Index.__doc__ = XMLFileBacked.Index.__doc__ def _decrypt(self, element): """ Decrypt a single encrypted properties file element """ if not element.text or not element.text.strip(): return - passes = Bcfg2.Server.Encryption.get_passphrases() + passes = Bcfg2.Options.setup.passphrases try: passphrase = passes[element.get("encrypted")] try: @@ -780,7 +745,7 @@ class StructFile(XMLFileBacked): """ stream = self.template.generate( metadata=metadata, - repo=self.setup['repo']).filter(removecomment) + repo=Bcfg2.Options.setup.repository).filter(removecomment) return lxml.etree.XML(stream.render('xml', strip_whitespace=False), parser=Bcfg2.Server.XMLParser) @@ -897,7 +862,7 @@ class InfoXML(StructFile): metadata (permissions, owner, etc.) of files. """ encryption = False - _include_tests = StructFile._include_tests + _include_tests = copy.copy(StructFile._include_tests) _include_tests['Path'] = lambda el, md, entry, *args: \ entry.get("name") == el.get("name") @@ -1152,7 +1117,7 @@ class SpecificData(Debuggable): """ A file that is specific to certain clients, groups, or all clients. """ - def __init__(self, name, specific, encoding): # pylint: disable=W0613 + def __init__(self, name, specific): # pylint: disable=W0613 """ :param name: The full path to the file :type name: string @@ -1161,8 +1126,6 @@ class SpecificData(Debuggable): object describing what clients this file applies to. :type specific: Bcfg2.Server.Plugin.helpers.Specificity - :param encoding: The encoding to use for data in this file - :type encoding: string """ Debuggable.__init__(self) self.name = name @@ -1210,7 +1173,7 @@ class EntrySet(Debuggable): #: considered a plain string and filenames must match exactly. basename_is_regex = False - def __init__(self, basename, path, entry_type, encoding): + def __init__(self, basename, path, entry_type): """ :param basename: The filename or regular expression that files in this EntrySet must match. See @@ -1225,12 +1188,10 @@ class EntrySet(Debuggable): be an object factory or similar callable. See below for the expected signature. :type entry_type: callable - :param encoding: The encoding of all files in this entry set. - :type encoding: string The ``entry_type`` callable must have the following signature:: - entry_type(filepath, specificity, encoding) + entry_type(filepath, specificity) Where the parameters are: @@ -1241,8 +1202,6 @@ class EntrySet(Debuggable): object describing what clients this file applies to. :type specific: Bcfg2.Server.Plugin.helpers.Specificity - :param encoding: The encoding to use for data in this file - :type encoding: string Additionally, the object returned by ``entry_type`` must have a ``specific`` attribute that is sortable (e.g., a @@ -1257,7 +1216,6 @@ class EntrySet(Debuggable): self.entries = {} self.metadata = default_path_metadata() self.infoxml = None - self.encoding = encoding if self.basename_is_regex: base_pat = basename @@ -1274,6 +1232,12 @@ class EntrySet(Debuggable): #: be overridden on a per-entry basis in :func:`entry_init`. self.specific = re.compile(pattern) + def set_debug(self, debug): + rv = Debuggable.set_debug(self, debug) + for entry in self.entries.values(): + entry.set_debug(debug) + return rv + def get_matching(self, metadata): """ Get a list of all entries that apply to the given client. This gets all matching entries; for example, there could be an @@ -1391,8 +1355,7 @@ class EntrySet(Debuggable): self.logger.error("Could not process filename %s; ignoring" % fpath) return - self.entries[event.filename] = entry_type(fpath, spec, - self.encoding) + self.entries[event.filename] = entry_type(fpath, spec) self.entries[event.filename].handle_event(event) def specificity_from_filename(self, fname, specific=None): @@ -1539,7 +1502,6 @@ class GroupSpool(Plugin, Generator): self.entries = {} self.handles = {} self.AddDirectoryMonitor('') - self.encoding = core.setup['encoding'] __init__.__doc__ = Plugin.__init__.__doc__ def add_entry(self, event): @@ -1563,8 +1525,7 @@ class GroupSpool(Plugin, Generator): dirpath = self.data + ident self.entries[ident] = self.es_cls(self.filename_pattern, dirpath, - self.es_child_cls, - self.encoding) + self.es_child_cls) self.Entries[self.entry_type][ident] = \ self.entries[ident].bind_entry if not os.path.isdir(epath): diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index ed4afb9b2..30275f6ad 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -6,6 +6,7 @@ import copy import threading import lxml.etree import Bcfg2.Server +import Bcfg2.Options from Bcfg2.Compat import Queue, Empty, Full, cPickle from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.exceptions import PluginInitError, \ @@ -530,6 +531,11 @@ class Version(Plugin): create = False + options = Plugin.options + [ + Bcfg2.Options.PathOption(cf=('server', 'vcs_root'), + default='<repository>', + help='Server VCS repository root')] + #: The path to the VCS metadata file or directory, relative to the #: base of the Bcfg2 repository. E.g., for Subversion this would #: be ".svn" @@ -540,12 +546,8 @@ class Version(Plugin): def __init__(self, core, datastore): Plugin.__init__(self, core, datastore) - if core.setup['vcs_root']: - self.vcs_root = core.setup['vcs_root'] - else: - self.vcs_root = datastore if self.__vcs_metadata_path__: - self.vcs_path = os.path.join(self.vcs_root, + self.vcs_path = os.path.join(Bcfg2.Options.setup.vcs_root, self.__vcs_metadata_path__) if not os.path.exists(self.vcs_path): diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 2473a3ed2..f91bac634 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -6,7 +6,6 @@ import sys import copy import Bcfg2.Server import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint from genshi.template import TemplateError @@ -45,6 +44,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = BundleFile + patterns = re.compile(r'^.*\.(?:xml|genshi)$') def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -123,57 +123,3 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, return bundleset BuildStructures.__doc__ = \ Bcfg2.Server.Plugin.Structure.BuildStructures.__doc__ - - -class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various :ref:`Bundler - <server-plugins-structures-bundler-index>` checks. """ - - def Run(self): - self.missing_bundles() - for bundle in self.core.plugins['Bundler'].entries.values(): - if self.HandlesFile(bundle.name): - self.bundle_names(bundle) - - @classmethod - def Errors(cls): - return {"bundle-not-found": "error", - "unused-bundle": "warning", - "explicit-bundle-name": "error", - "genshi-extension-bundle": "error"} - - def missing_bundles(self): - """ Find bundles listed in Metadata but not implemented in - Bundler. """ - if self.files is None: - # when given a list of files on stdin, this check is - # useless, so skip it - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) - - allbundles = self.core.plugins['Bundler'].bundles.keys() - for bundle in ref_bundles: - if bundle not in allbundles: - self.LintError("bundle-not-found", - "Bundle %s referenced, but does not exist" % - bundle) - - for bundle in allbundles: - if bundle not in ref_bundles: - self.LintError("unused-bundle", - "Bundle %s defined, but is not referenced " - "in Metadata" % bundle) - - def bundle_names(self, bundle): - """ Verify that deprecated bundle .genshi bundles and explicit - bundle names aren't used """ - if bundle.xdata.get('name'): - self.LintError("explicit-bundle-name", - "Deprecated explicit bundle name in %s" % - bundle.name) - - if bundle.name.endswith(".genshi"): - self.LintError("genshi-extension-bundle", - "Bundle %s uses deprecated .genshi extension" % - bundle.name) diff --git a/src/lib/Bcfg2/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index e0cbdf72a..f91cc1943 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -14,13 +14,13 @@ class Bzr(Bcfg2.Server.Plugin.Version): def __init__(self, core, datastore): Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) self.logger.debug("Initialized Bazaar plugin with directory %s at " - "revision = %s" % (self.vcs_root, + "revision = %s" % (Bcfg2.Options.setup.vcs_root, self.get_revision())) def get_revision(self): """Read Bazaar revision information for the Bcfg2 repository.""" try: - working_tree = WorkingTree.open(self.vcs_root) + working_tree = WorkingTree.open(Bcfg2.Options.setup.vcs_root) revision = str(working_tree.branch.revno()) if (working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns()): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index a859da0ba..50de498e6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -3,6 +3,7 @@ based on an XML specification of which SSH keypairs should granted access. """ import lxml.etree +import Bcfg2.Options from Bcfg2.Server.Plugin import StructFile, PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator, CFG from Bcfg2.Server.Plugins.Metadata import ClientMetadata @@ -27,15 +28,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): self.core = CFG.core __init__.__doc__ = CfgGenerator.__init__.__doc__ - @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (self.setup.cfp.has_section("sshkeys") and - self.setup.cfp.has_option("sshkeys", "category")): - return self.setup.cfp.get("sshkeys", "category") - return None - def handle_event(self, event): CfgGenerator.handle_event(self, event) StructFile.HandleEvent(self, event) @@ -61,12 +53,13 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): key_md = ClientMetadata("dummy", group, [group], [], set(), set(), dict(), None, None, None, None) - elif (self.category and - not metadata.group_in_category(self.category)): + elif (Bcfg2.Options.setup.sshkeys_category and + not metadata.group_in_category( + Bcfg2.Options.setup.sshkeys_category)): self.logger.warning("Cfg: %s ignoring Allow from %s: " "No group in category %s" % (metadata.hostname, pubkey_name, - self.category)) + Bcfg2.Options.setup.sshkeys_category)) continue else: key_md = metadata diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 4c8adceec..476dc1fc6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,6 +2,7 @@ <http://www.cheetahtemplate.org/>`_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator @@ -27,19 +28,19 @@ class CfgCheetahGenerator(CfgGenerator): #: :class:`Cheetah.Template.Template` compiler settings settings = dict(useStackFrames=False) - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CHEETAH: raise PluginExecutionError("Cheetah is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): - template = Template(self.data.decode(self.encoding), + template = Template(self.data.decode(Bcfg2.Options.setup.encoding), compilerSettings=self.settings) template.metadata = metadata template.name = entry.get('realname', entry.get('name')) template.path = entry.get('realname', entry.get('name')) template.source_path = self.name - template.repo = self.setup['repo'] + template.repo = Bcfg2.Options.setup.repository return template.respond() get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 516eba2f6..e2a2f696a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -21,8 +21,8 @@ class CfgEncryptedGenerator(CfgGenerator): #: .genshi.crypt and .cheetah.crypt files __priority__ = 50 - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 0521485e8..f69ab8e5f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -37,7 +37,7 @@ class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): #: when it's read in __loader_cls__ = EncryptedTemplateLoader - def __init__(self, fname, spec, encoding): - CfgGenshiGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenshiGenerator.__init__(self, fname, spec) if not HAS_CRYPTO: raise PluginExecutionError("M2Crypto is not available") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index d06b864ac..953473a12 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -15,8 +15,8 @@ class CfgExternalCommandVerifier(CfgVerifier): #: Handle :file:`:test` files __basenames__ = [':test'] - def __init__(self, name, specific, encoding): - CfgVerifier.__init__(self, name, specific, encoding) + def __init__(self, name, specific): + CfgVerifier.__init__(self, name, specific) self.cmd = [] self.exc = Executor(timeout=30) __init__.__doc__ = CfgVerifier.__init__.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index e056c871a..7ba8c4491 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -5,9 +5,9 @@ import re import sys import traceback +import Bcfg2.Options from Bcfg2.Server.Plugin import PluginExecutionError, removecomment from Bcfg2.Server.Plugins.Cfg import CfgGenerator - from genshi.template import TemplateLoader, NewTextTemplate from genshi.template.eval import UndefinedError, Suite @@ -70,8 +70,8 @@ class CfgGenshiGenerator(CfgGenerator): #: occurred. pyerror_re = re.compile(r'<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') - def __init__(self, fname, spec, encoding): - CfgGenerator.__init__(self, fname, spec, encoding) + def __init__(self, fname, spec): + CfgGenerator.__init__(self, fname, spec) self.template = None self.loader = self.__loader_cls__(max_cache_size=0) __init__.__doc__ = CfgGenerator.__init__.__doc__ @@ -87,13 +87,15 @@ class CfgGenshiGenerator(CfgGenerator): metadata=metadata, path=self.name, source_path=self.name, - repo=self.setup['repo']).filter(removecomment) + repo=Bcfg2.Options.setup.repository).filter(removecomment) try: try: - return stream.render('text', encoding=self.encoding, + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding, strip_whitespace=False) except TypeError: - return stream.render('text', encoding=self.encoding) + return stream.render('text', + encoding=Bcfg2.Options.setup.encoding) except UndefinedError: # a failure in a genshi expression _other_ than %{ python ... %} err = sys.exc_info()[1] @@ -172,8 +174,9 @@ class CfgGenshiGenerator(CfgGenerator): def handle_event(self, event): CfgGenerator.handle_event(self, event) try: - self.template = self.loader.load(self.name, cls=NewTextTemplate, - encoding=self.encoding) + self.template = \ + self.loader.load(self.name, cls=NewTextTemplate, + encoding=Bcfg2.Options.setup.encoding) except: raise PluginExecutionError("Failed to load template: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 862726788..7bb5d3cf5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -3,8 +3,8 @@ import os import shutil import tempfile +import Bcfg2.Options from Bcfg2.Utils import Executor -from Bcfg2.Options import get_option_parser from Bcfg2.Server.Plugin import StructFile from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator @@ -25,6 +25,14 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): #: Handle XML specifications of private keys __basenames__ = ['privkey.xml'] + options = [ + Bcfg2.Options.Option( + cf=("sshkeys", "category"), dest="sshkeys_category", + help="Metadata category that generated SSH keys are specific to"), + Bcfg2.Options.Option( + cf=("sshkeys", "passphrase"), dest="sshkeys_passphrase", + help="Passphrase used to encrypt generated SSH private keys")] + def __init__(self, fname): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) @@ -32,27 +40,15 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): pubkey_path = os.path.dirname(self.name) + ".pub" pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) - self.setup = get_option_parser() self.cmd = Executor() __init__.__doc__ = CfgCreator.__init__.__doc__ @property - def category(self): - """ The name of the metadata category that generated keys are - specific to """ - if (self.setup.cfp.has_section("sshkeys") and - self.setup.cfp.has_option("sshkeys", "category")): - return self.setup.cfp.get("sshkeys", "category") - return None - - @property def passphrase(self): """ The passphrase used to encrypt private keys """ - if (HAS_CRYPTO and - self.setup.cfp.has_section("sshkeys") and - self.setup.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Server.Encryption.get_passphrases()[ - self.setup.cfp.get("sshkeys", "passphrase")] + if HAS_CRYPTO and Bcfg2.Options.setup.sshkeys_passphrase: + return Bcfg2.Options.setup.passphrases[ + Bcfg2.Options.setup.sshkeys_passphrase] return None def handle_event(self, event): @@ -141,7 +137,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): """ if spec is None: spec = self.XMLMatch(metadata) - category = spec.get("category", self.category) + category = spec.get("category", Bcfg2.Options.setup.sshkeys_category) if category is None: per_host_default = "true" else: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index fc3de3d68..a7fa92201 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -3,18 +3,14 @@ import re import os import sys -import stat import errno import operator import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -import Bcfg2.Server.Lint -from fnmatch import fnmatch from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ - any, oct_mode +from Bcfg2.Compat import u_str, unicode, b64encode, any, oct_mode # pylint: enable=W0622 #: CFG is a reference to the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` @@ -25,27 +21,8 @@ from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ #: facility for passing it otherwise. CFG = None -_HANDLERS = [] - -def handlers(): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if not _HANDLERS: - for submodule in walk_packages(path=__path__, prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if issubclass(hdlr, CfgBaseFileMatcher): - _HANDLERS.append(hdlr) - _HANDLERS.sort(key=operator.attrgetter("__priority__")) - return _HANDLERS - - -class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, - Bcfg2.Server.Plugin.Debuggable): +class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): """ .. currentmodule:: Bcfg2.Server.Plugins.Cfg CfgBaseFileMatcher is the parent class for all Cfg handler @@ -89,12 +66,8 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, #: Flag to indicate an experimental handler. experimental = False - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.encoding = encoding - self.setup = Bcfg2.Options.get_option_parser() + def __init__(self, name, specific): + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ """ .. ----- @@ -185,7 +158,7 @@ class CfgGenerator(CfgBaseFileMatcher): client. See :class:`Bcfg2.Server.Plugin.helpers.EntrySet` for more details on how the best handler is chosen.""" - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # we define an __init__ that just calls the parent __init__, # so that we can set the docstring on __init__ to something # different from the parent __init__ -- namely, the parent @@ -193,7 +166,7 @@ class CfgGenerator(CfgBaseFileMatcher): # which we use to delineate the actual docs from the # .. autoattribute hacks we have to do to get private # attributes included in sphinx 1.0 """ - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def get_data(self, entry, metadata): # pylint: disable=W0613 @@ -213,9 +186,9 @@ class CfgFilter(CfgBaseFileMatcher): """ CfgFilters modify the initial content of a file after it has been generated by a :class:`Bcfg2.Server.Plugins.Cfg.CfgGenerator`. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def modify_data(self, entry, metadata, data): @@ -253,7 +226,7 @@ class CfgInfo(CfgBaseFileMatcher): .. ----- .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def bind_info_to_entry(self, entry, metadata): """ Assign the appropriate attributes to the entry, modifying @@ -276,9 +249,9 @@ class CfgVerifier(CfgBaseFileMatcher): etc.), or both. """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific): # see comment on CfgGenerator.__init__ above - CfgBaseFileMatcher.__init__(self, name, specific, encoding) + CfgBaseFileMatcher.__init__(self, name, specific) __init__.__doc__ = CfgBaseFileMatcher.__init__.__doc__.split(".. -----")[0] def verify_entry(self, entry, metadata, data): @@ -317,7 +290,7 @@ class CfgCreator(CfgBaseFileMatcher): .. ----- .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgCreator.__specific__ """ - CfgBaseFileMatcher.__init__(self, fname, None, None) + CfgBaseFileMatcher.__init__(self, fname, None) def create_data(self, entry, metadata): """ Create new data for the given entry and write it to disk @@ -431,26 +404,30 @@ class CfgDefaultInfo(CfgInfo): bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, - Bcfg2.Server.Plugin.Debuggable): +class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ Handle a collection of host- and group-specific Cfg files with multiple different Cfg handlers in a single directory. """ - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - Bcfg2.Server.Plugin.Debuggable.__init__(self) + def __init__(self, basename, path, entry_type): + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type) self.specific = None self._handlers = None - self.setup = Bcfg2.Options.get_option_parser() __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): - rv = Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + rv = Bcfg2.Server.Plugin.EntrySet.set_debug(self, debug) for entry in self.entries.values(): entry.set_debug(debug) return rv + @property + def handlers(self): + """ A list of Cfg handler classes. """ + if self._handlers is None: + self._handlers = Bcfg2.Options.setup.cfg_handlers + self._handlers.sort(key=operator.attrgetter("__priority__")) + return self._handlers + def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -467,7 +444,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in handlers(): + for hdlr in self.handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -560,7 +537,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # most specific to least specific. data = fltr.modify_data(entry, metadata, data) - if self.setup['validate']: + if Bcfg2.Options.setup.cfg_validation: try: self._validate_data(entry, metadata, data) except CfgVerificationError: @@ -576,7 +553,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, if not isinstance(data, unicode): if not isinstance(data, str): data = data.decode('utf-8') - data = u_str(data, self.encoding) + data = u_str(data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) @@ -757,10 +734,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, self.logger.error(msg) raise PluginExecutionError(msg) try: - etext = new_entry['text'].encode(self.encoding) + etext = new_entry['text'].encode(Bcfg2.Options.setup.encoding) except: msg = "Cfg: Cannot encode content of %s as %s" % \ - (name, self.encoding) + (name, Bcfg2.Options.setup.encoding) self.logger.error(msg) raise PluginExecutionError(msg) open(name, 'w').write(etext) @@ -785,6 +762,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, flag=log) +class CfgHandlerAction(Bcfg2.Options.ComponentAction): + bases = ['Bcfg2.Server.Plugins.Cfg'] + + class Cfg(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.PullTarget): """ The Cfg plugin provides a repository to describe configuration @@ -796,17 +777,27 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData + options = Bcfg2.Server.Plugin.GroupSpool.options + [ + Bcfg2.Options.BooleanOption( + '--cfg-validation', cf=('cfg', 'validation'), default=True, + help='Run validation on Cfg files'), + Bcfg2.Options.Option( + cf=("cfg", "handlers"), dest="cfg_handlers", + help="Cfg handlers to load", + type=Bcfg2.Options.Types.comma_list, action=CfgHandlerAction, + default=['CfgAuthorizedKeysGenerator', 'CfgEncryptedGenerator', + 'CfgCheetahGenerator', 'CfgEncryptedCheetahGenerator', + 'CfgGenshiGenerator', 'CfgEncryptedGenshiGenerator', + 'CfgExternalCommandVerifier', 'CfgInfoXML', + 'CfgPlaintextGenerator', + 'CfgPrivateKeyCreator', 'CfgPublicKeyCreator'])] + def __init__(self, core, datastore): global CFG # pylint: disable=W0603 Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullTarget.__init__(self) CFG = self - - setup = Bcfg2.Options.get_option_parser() - if 'validate' not in setup: - setup.add_option('validate', Bcfg2.Options.CFG_VALIDATION) - setup.reparse() __init__.__doc__ = Bcfg2.Server.Plugin.GroupSpool.__init__.__doc__ def has_generator(self, entry, metadata): @@ -840,96 +831,3 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, log) AcceptPullData.__doc__ = \ Bcfg2.Server.Plugin.PullTarget.AcceptPullData.__doc__ - - -class CfgLint(Bcfg2.Server.Lint.ServerPlugin): - """ warn about usage of .cat and .diff files """ - - def Run(self): - for basename, entry in list(self.core.plugins['Cfg'].entries.items()): - self.check_pubkey(basename, entry) - self.check_missing_files() - - @classmethod - def Errors(cls): - return {"no-pubkey-xml": "warning", - "unknown-cfg-files": "error", - "extra-cfg-files": "error"} - - def check_pubkey(self, basename, entry): - """ check that privkey.xml files have corresponding pubkey.xml - files """ - if "privkey.xml" not in entry.entries: - return - privkey = entry.entries["privkey.xml"] - if not self.HandlesFile(privkey.name): - return - - pubkey = basename + ".pub" - if pubkey not in self.core.plugins['Cfg'].entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - else: - pubset = self.core.plugins['Cfg'].entries[pubkey] - if "pubkey.xml" not in pubset.entries: - self.LintError("no-pubkey-xml", - "%s has no corresponding pubkey.xml at %s" % - (basename, pubkey)) - - def _list_path_components(self, path): - """ Get a list of all components of a path. E.g., - ``self._list_path_components("/foo/bar/foobaz")`` would return - ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed - to be in order.""" - rv = [] - remaining, component = os.path.split(path) - while component != '': - rv.append(component) - remaining, component = os.path.split(remaining) - return rv - - def check_missing_files(self): - """ check that all files on the filesystem are known to Cfg """ - cfg = self.core.plugins['Cfg'] - - # first, collect ignore patterns from handlers - ignore = set() - for hdlr in handlers(): - ignore.update(hdlr.__ignore__) - - # next, get a list of all non-ignored files on the filesystem - all_files = set() - for root, _, files in os.walk(cfg.data): - for fname in files: - fpath = os.path.join(root, fname) - # check against the handler ignore patterns and the - # global FAM ignore list - if (not any(fname.endswith("." + i) for i in ignore) and - not any(fnmatch(fpath, p) - for p in self.config['ignore']) and - not any(fnmatch(c, p) - for p in self.config['ignore'] - for c in self._list_path_components(fpath))): - all_files.add(fpath) - - # next, get a list of all files known to Cfg - cfg_files = set() - for root, eset in cfg.entries.items(): - cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) - for fname in eset.entries.keys()) - - # finally, compare the two - unknown_files = all_files - cfg_files - extra_files = cfg_files - all_files - if unknown_files: - self.LintError( - "unknown-cfg-files", - "Files on the filesystem could not be understood by Cfg: %s" % - "; ".join(unknown_files)) - if extra_files: - self.LintError( - "extra-cfg-files", - "Cfg has entries for files that do not exist on the " - "filesystem: %s\nThis is probably a bug." % - "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index 0054a8a37..09fbfaea7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -20,7 +20,7 @@ class Cvs(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read cvs revision information for the Bcfg2 repository.""" result = self.cmd.run(["env LC_ALL=C", "cvs", "log"], - shell=True, cwd=self.vcs_root) + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: return result.stdout.splitlines()[0].strip() except (IndexError, AttributeError): diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index 2c6dde393..b48809cac 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -20,7 +20,7 @@ class Darcs(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read Darcs changeset information for the Bcfg2 repository.""" result = self.cmd.run(["env LC_ALL=C", "darcs", "changes"], - shell=True, cwd=self.vcs_root) + shell=True, cwd=Bcfg2.Options.setup.vcs_root) if result.success: return result.stdout.splitlines()[0].strip() else: diff --git a/src/lib/Bcfg2/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index 04c14aa96..79e2ca0e2 100644 --- a/src/lib/Bcfg2/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -9,6 +9,8 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, """Set default attributes on bound entries""" __author__ = 'bcfg-dev@mcs.anl.gov' + options = Bcfg2.Server.Plugin.PrioDir.options + # Rules is a Generator that happens to implement all of the # functionality we want, so we overload it, but Defaults should # _not_ handle any entries; it does its stuff in the structure diff --git a/src/lib/Bcfg2/Server/Plugins/Deps.py b/src/lib/Bcfg2/Server/Plugins/Deps.py index 312b03bae..fa821aad3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Deps.py +++ b/src/lib/Bcfg2/Server/Plugins/Deps.py @@ -48,7 +48,8 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, prereqs = self.calculate_prereqs(metadata, entries) self.cache[(entries, groups)] = prereqs - newstruct = lxml.etree.Element("Independent") + newstruct = lxml.etree.Element("Independent", + name=self.__class__.__name__) for tag, name in prereqs: lxml.etree.SubElement(newstruct, tag, name=name) structures.append(newstruct) diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index a3bba14f3..45511eb52 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -8,7 +8,6 @@ import os import sys import errno import lxml.etree -import Bcfg2.Options import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor @@ -220,12 +219,12 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, return self.logger.info("Writing %s for %s" % (infoxml, data.get("name"))) + default_mdata = Bcfg2.Server.Plugin.default_path_metadata() info = lxml.etree.Element( "Info", - owner=data.get("owner", Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", Bcfg2.Options.MDATA_GROUP.value), - mode=data.get("mode", Bcfg2.Options.MDATA_MODE.value), - encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) + owner=data.get("owner", default_mdata['owner']), + group=data.get("group", default_mdata['group']), + mode=data.get("mode", default_mdata['mode'])) root = lxml.etree.Element("FileInfo") root.append(info) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 05cf4e5d4..f6aa3221a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -20,7 +20,7 @@ class Fossil(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read fossil revision information for the Bcfg2 repository.""" result = self.cmd.run(["env LC_ALL=C", "fossil", "info"], - shell=True, cwd=self.vcs_root) + shell=True, cwd=Bcfg2.Options.setup.vcs_root) try: revision = None for line in result.stdout.splitlines(): diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 58a5c58f0..d0502ed6a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -23,7 +23,7 @@ class Git(Version): def __init__(self, core, datastore): Version.__init__(self, core, datastore) if HAS_GITPYTHON: - self.repo = git.Repo(self.vcs_root) + self.repo = git.Repo(Bcfg2.Options.setup.vcs_root) self.cmd = None else: self.logger.debug("Git: GitPython not found, using CLI interface " @@ -45,7 +45,8 @@ class Git(Version): return self.repo.head.commit.hexsha else: cmd = ["git", "--git-dir", self.vcs_path, - "--work-tree", self.vcs_root, "rev-parse", "HEAD"] + "--work-tree", Bcfg2.Options.setup.vcs_root, + "rev-parse", "HEAD"] self.debug_log("Git: Running %s" % cmd) result = self.cmd.run(cmd) if not result.success: @@ -53,7 +54,7 @@ class Git(Version): return result.stdout except: raise PluginExecutionError("Git: Error getting revision from %s: " - "%s" % (self.vcs_root, + "%s" % (Bcfg2.Options.setup.vcs_root, sys.exc_info()[1])) def Update(self, ref=None): @@ -62,14 +63,15 @@ class Git(Version): """ self.logger.info("Git: Git.Update(ref='%s')" % ref) self.debug_log("Git: Performing garbage collection on repo at %s" % - self.vcs_root) + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.gc('--auto')) except git.GitCommandError: self.logger.warning("Git: Failed to perform garbage collection: %s" % sys.exc_info()[1]) - self.debug_log("Git: Fetching all refs for repo at %s" % self.vcs_root) + self.debug_log("Git: Fetching all refs for repo at %s" % + Bcfg2.Options.setup.vcs_root) try: self._log_git_cmd(self.repo.git.fetch('--all')) except git.GitCommandError: @@ -102,5 +104,5 @@ class Git(Version): "upstream: %s" % sys.exc_info()[1]) self.logger.info("Git: Repo at %s updated to %s" % - (self.vcs_root, self.get_revision())) + (Bcfg2.Options.setup.vcs_root, self.get_revision())) return True diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 3e5508160..90cbd083d 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -122,40 +122,3 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, def get_additional_groups(self, metadata): return self.config.process_patterns(metadata.hostname) - - -class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns - <server-plugins-grouping-grouppatterns>` patterns for validity. - This is simply done by trying to create a - :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for - each pattern, and catching exceptions and presenting them as - ``bcfg2-lint`` errors.""" - - def Run(self): - cfg = self.core.plugins['GroupPatterns'].config - for entry in cfg.xdata.xpath('//GroupPattern'): - groups = [g.text for g in entry.findall('Group')] - self.check(entry, groups, ptype='NamePattern') - self.check(entry, groups, ptype='NameRange') - - @classmethod - def Errors(cls): - return {"pattern-fails-to-initialize": "error"} - - def check(self, entry, groups, ptype="NamePattern"): - """ Check a single pattern for validity """ - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - - for el in entry.findall(ptype): - pat = el.text - try: - pmap(pat) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("pattern-fails-to-initialize", - "Failed to initialize %s %s for %s: %s" % - (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 3fd3918bd..f9a9f858c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -20,7 +20,7 @@ class Hg(Bcfg2.Server.Plugin.Version): def get_revision(self): """Read hg revision information for the Bcfg2 repository.""" try: - repo_path = self.vcs_root + "/" + repo_path = Bcfg2.Options.setup.vcs_root + "/" repo = hg.repository(ui.ui(), repo_path) tip = repo.changelog.tip() return repo.changelog.rev(tip) diff --git a/src/lib/Bcfg2/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index 8e8b078d9..6fc89b4f3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -3,7 +3,6 @@ import logging import sys import time import traceback -import Bcfg2.Options import Bcfg2.Server.Plugin logger = logging.getLogger('Bcfg2.Plugins.Ldap') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index f355fd7de..24adee4f4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -12,23 +12,28 @@ import socket import logging import lxml.etree import Bcfg2.Server -import Bcfg2.Server.Lint +import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo -try: - from django.db import models - HAS_DJANGO = True -except ImportError: - HAS_DJANGO = False -LOGGER = logging.getLogger(__name__) +MetadataClientModel = None +HAS_DJANGO = False -if HAS_DJANGO: +def load_django_models(): + global MetadataClientModel, ClientVersions, HAS_DJANGO + + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return + class MetadataClientModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): """ django model for storing clients in the database """ @@ -39,12 +44,12 @@ if HAS_DJANGO: Bcfg2.Server.Plugin.DatabaseBacked): """ dict-like object to make it easier to access client bcfg2 versions from the database """ - create = False def __getitem__(self, key): try: - return MetadataClientModel.objects.get(hostname=key).version + return MetadataClientModel.objects.get( + hostname=key).version except MetadataClientModel.DoesNotExist: raise KeyError(key) @@ -350,6 +355,8 @@ class MetadataQuery(object): def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): + self.logger = logging.getLogger(self.__class__.__name__) + #: Get :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` #: object for the given hostname. #: @@ -402,8 +409,9 @@ class MetadataQuery(object): @wraps(func) def inner(arg): if isinstance(arg, str): - LOGGER.warning("%s: %s takes a list as argument, not a string" - % (self.__class__.__name__, func.__name__)) + self.logger.warning("%s: %s takes a list as argument, not a " + "string" % (self.__class__.__name__, + func.__name__)) return func(arg) # pylint: enable=C0111 @@ -493,6 +501,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, __author__ = 'bcfg-dev@mcs.anl.gov' sort_order = 500 + options = Bcfg2.Server.Plugin.DatabaseBacked.options + [ + Bcfg2.Options.Common.password, + Bcfg2.Options.BooleanOption( + cf=('metadata', 'use_database'), dest="metadata_db", + help="Use database capabilities of the Metadata plugin"), + Bcfg2.Options.Option( + cf=('communication', 'authentication'), default='cert+password', + choices=['cert', 'bootstrap', 'cert+password'], + help='Default client authentication method')] + def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) Bcfg2.Server.Plugin.Caching.__init__(self) @@ -541,7 +559,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.session_cache = {} self.default = None self.pdirty = False - self.password = core.setup['password'] + self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, lambda: list(self.clients), self.get_client_names_by_groups, @@ -1296,6 +1314,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return False resolved = self.resolve_client(addresspair) if resolved.lower() == client.lower(): + self.logger.debug("Client %s address validates" % client) return True else: self.logger.error("Got request for %s from incorrect address %s" % @@ -1315,7 +1334,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) auth_type = self.auth.get(client, - self.core.setup['authentication']) + Bcfg2.Options.setup.authentication) elif user == 'root': id_method = 'address' try: @@ -1346,6 +1365,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # remember the cert-derived client name for this connection if client in self.floating: self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s certificate validates" % client) # we are done if cert+password not required return True @@ -1372,13 +1392,14 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # populate the session cache if user != 'root': self.session_cache[address] = (time.time(), client) + self.logger.debug("Client %s authenticated successfully" % client) return True # pylint: enable=R0911,R0912 def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, - self.core.setup['authentication']) == 'bootstrap': + Bcfg2.Options.setup.authentication) == 'bootstrap': self.update_client(metadata.hostname, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): @@ -1494,149 +1515,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, (group.get('name'), parent.get('name'))) return rv - -class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin for :ref:`Metadata - <server-plugins-grouping-metadata>`. This checks for several things: - - * ``<Client>`` tags nested inside other ``<Client>`` tags; - * Deprecated options (like ``location="floating"``); - * Profiles that don't exist, or that aren't profile groups; - * Groups or clients that are defined multiple times; - * Multiple default groups or a default group that isn't a profile - group. - """ - - def Run(self): - self.nested_clients() - self.deprecated_options() - self.bogus_profiles() - self.duplicate_groups() - self.duplicate_default_groups() - self.duplicate_clients() - self.default_is_profile() - - @classmethod - def Errors(cls): - return {"nested-client-tags": "warning", - "deprecated-clients-options": "warning", - "nonexistent-profile-group": "error", - "non-profile-set-as-profile": "error", - "duplicate-group": "error", - "duplicate-client": "error", - "multiple-default-groups": "error", - "default-is-not-profile": "error"} - - def deprecated_options(self): - """ Check for the ``location='floating'`` option, which has - been deprecated in favor of ``floating='true'``. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - clientdata = self.metadata.clients_xml.xdata - for el in clientdata.xpath("//Client"): - loc = el.get("location") - if loc: - if loc == "floating": - floating = True - else: - floating = False - self.LintError("deprecated-clients-options", - "The location='%s' option is deprecated. " - "Please use floating='%s' instead:\n%s" % - (loc, floating, self.RenderXML(el))) - - def nested_clients(self): - """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag, - which is either redundant or will never match. """ - groupdata = self.metadata.groups_xml.xdata - for el in groupdata.xpath("//Client//Client"): - self.LintError("nested-client-tags", - "Client %s nested within Client tag: %s" % - (el.get("name"), self.RenderXML(el))) - - def bogus_profiles(self): - """ Check for clients that have profiles that are either not - flagged as profile groups in ``groups.xml``, or don't exist. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - for client in self.metadata.clients_xml.xdata.findall('.//Client'): - profile = client.get("profile") - if profile not in self.metadata.groups: - self.LintError("nonexistent-profile-group", - "%s has nonexistent profile group %s:\n%s" % - (client.get("name"), profile, - self.RenderXML(client))) - elif not self.metadata.groups[profile].is_profile: - self.LintError("non-profile-set-as-profile", - "%s is set as profile for %s, but %s is not a " - "profile group:\n%s" % - (profile, client.get("name"), profile, - self.RenderXML(client))) - - def duplicate_default_groups(self): - """ Check for multiple default groups. """ - defaults = [] - for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): - if grp.get("default", "false").lower() == "true": - defaults.append(self.RenderXML(grp)) - if len(defaults) > 1: - self.LintError("multiple-default-groups", - "Multiple default groups defined:\n%s" % - "\n".join(defaults)) - - def duplicate_clients(self): - """ Check for clients that are defined more than once. """ - if not hasattr(self.metadata, "clients_xml"): - # using metadata database - return - self.duplicate_entries( - self.metadata.clients_xml.xdata.xpath("//Client"), - "client") - - def duplicate_groups(self): - """ Check for groups that are defined more than once. We - count a group tag as a definition if it a) has profile or - public set; or b) has any children.""" - allgroups = [ - g - for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") - if g.get("profile") or g.get("public") or g.getchildren()] - self.duplicate_entries(allgroups, "group") - - def duplicate_entries(self, allentries, etype): - """ Generic duplicate entry finder. - - :param allentries: A list of all entries to check for - duplicates. - :type allentries: list of lxml.etree._Element - :param etype: The entry type. This will be used to determine - the error name (``duplicate-<etype>``) and for - display to the end user. - :type etype: string - """ - entries = dict() - for el in allentries: - if el.get("name") in entries: - entries[el.get("name")].append(self.RenderXML(el)) - else: - entries[el.get("name")] = [self.RenderXML(el)] - for ename, els in entries.items(): - if len(els) > 1: - self.LintError("duplicate-%s" % etype, - "%s %s is defined multiple times:\n%s" % - (etype.title(), ename, "\n".join(els))) - - def default_is_profile(self): - """ Ensure that the default group is a profile group. """ - if (self.metadata.default and - not self.metadata.groups[self.metadata.default].is_profile): - xdata = \ - self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % - self.metadata.default)[0] - self.LintError("default-is-not-profile", - "Default group is not a profile group:\n%s" % - self.RenderXML(xdata)) + @staticmethod + def options_parsed_hook(): + load_django_models() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 4d05f9d97..0df8624f6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -73,20 +73,17 @@ The Collection Module --------------------- """ -import sys import copy -import logging import lxml.etree +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Server.FileMonitor import get_fam -from Bcfg2.Options import get_option_parser +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Server.FileMonitor import get_fam from Bcfg2.Server.Statistics import track_statistics -LOGGER = logging.getLogger(__name__) - -class Collection(list, Bcfg2.Server.Plugin.Debuggable): +class Collection(list, Debuggable): """ ``Collection`` objects represent the set of :class:`Bcfg2.Server.Plugins.Packages.Source` objects that apply to a given client, and can be used to query all software @@ -119,15 +116,14 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): .. ----- .. autoattribute:: __package_groups__ """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) list.__init__(self, sources) - self.debug_flag = debug + self.debug_flag = self.debug_flag or debug self.metadata = metadata self.basepath = basepath self.cachepath = cachepath self.virt_pkgs = dict() self.fam = get_fam() - self.setup = get_option_parser() try: self.ptype = sources[0].ptype @@ -447,9 +443,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ for pkg in pkglist: lxml.etree.SubElement(entry, 'BoundPackage', name=pkg, - version=self.setup.cfp.get("packages", - "version", - default="auto"), + version=Bcfg2.Options.setup.packages_version, type=self.ptype, origin='Packages') def get_new_packages(self, initial, complete): @@ -601,22 +595,9 @@ def get_collection_class(source_type): :type source_type: string :returns: type - the Collection subclass that should be used to instantiate an object to contain sources of the given type. """ - modname = "Bcfg2.Server.Plugins.Packages.%s" % source_type.title() - try: - module = sys.modules[modname] - except KeyError: - try: - module = __import__(modname).Server.Plugins.Packages - except ImportError: - msg = "Packages: Unknown source type %s" % source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - - try: - cclass = getattr(module, source_type.title() + "Collection") - except AttributeError: - msg = "Packages: No collection class found for %s sources" % \ - source_type - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - return cclass + cls = None + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % source_type.title()): + return getattr(mod, "%sCollection" % source_type.title()) + raise Bcfg2.Server.Plugin.PluginExecutionError( + "Packages: No collection class found for %s sources" % source_type) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 9ff2d53a0..1af046ec0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -4,14 +4,12 @@ import os import sys import Bcfg2.Server.Plugin -from Bcfg2.Options import get_option_parser from Bcfg2.Server.Statistics import track_statistics from Bcfg2.Server.Plugins.Packages.Source import SourceInitError # pylint: disable=E0012,R0924 -class PackagesSources(Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): +class PackagesSources(Bcfg2.Server.Plugin.StructFile): """ PackagesSources handles parsing of the :mod:`Bcfg2.Server.Plugins.Packages` ``sources.xml`` file, and the creation of the appropriate @@ -37,7 +35,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` - If ``sources.xml`` cannot be read """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) Bcfg2.Server.Plugin.StructFile.__init__(self, filename, should_monitor=True) @@ -54,8 +51,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, err = sys.exc_info()[1] self.logger.error("Could not create Packages cache at %s: %s" % (self.cachepath, err)) - #: The Bcfg2 options dict - self.setup = get_option_parser() #: The :class:`Bcfg2.Server.Plugins.Packages.Packages` that #: instantiated this ``PackagesSources`` object @@ -69,10 +64,9 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, self.parsed = set() def set_debug(self, debug): - Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) + Bcfg2.Server.Plugin.StructFile.set_debug(self, debug) for source in self.entries: source.set_debug(debug) - set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -138,15 +132,13 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, xsource.get("url")))) return None - try: - module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cls = getattr(module, "%sSource" % stype.title()) - except (ImportError, AttributeError): - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown source type %s (%s)" % (stype, - err)) + cls = None + for mod in Bcfg2.Options.setup.packages_backends: + if mod.__name__.endswith(".%s" % stype.title()): + cls = getattr(mod, "%sSource" % stype.title()) + break + else: + self.logger.error("Packages: Unknown source type %s" % stype) return None try: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 767ac13ac..e1659dbb3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -50,7 +50,7 @@ import os import re import sys import Bcfg2.Server.Plugin -from Bcfg2.Options import get_option_parser +from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ cPickle, md5 @@ -93,7 +93,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 +class Source(Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single <Source> tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -121,7 +121,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :type source: lxml.etree._Element :raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError` """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) + Debuggable.__init__(self) #: The base filesystem path under which cache data for this #: source should be stored @@ -130,9 +130,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: The XML tag that describes this source self.xsource = xsource - #: A Bcfg2 options dict - self.setup = get_option_parser() - #: A set of package names that are deemed "essential" by this #: source self.essentialpkgs = set() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 75dab3f76..0d49473c6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -59,12 +59,10 @@ import errno import socket import logging import lxml.etree -from lockfile import FileLock - -import Bcfg2.Server.FileMonitor import Bcfg2.Server.Plugin +import Bcfg2.Server.FileMonitor +from lockfile import FileLock from Bcfg2.Utils import Executor -from Bcfg2.Options import get_option_parser # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -109,6 +107,30 @@ PULPSERVER = None PULPCONFIG = None +options = [ + Bcfg2.Options.PathOption( + cf=("packages:yum", "helper"), dest="yum_helper", + help="Path to the bcfg2-yum-helper executable"), + Bcfg2.Options.BooleanOption( + cf=("packages:yum", "use_yum_libraries"), + help="Use Python yum libraries"), + Bcfg2.Options.PathOption( + cf=("packages:yum", "gpg_keypath"), default="/etc/pki/rpm-gpg", + help="GPG key path on the client"), + Bcfg2.Options.Option( + cf=("packages:yum", "*"), dest="yum_options", + help="Other yum options to include in generated yum configs")] +if HAS_PULP: + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "username"), dest="pulp_username", + help="Username for Pulp authentication")) + options.append( + Bcfg2.Options.Option( + cf=("packages:pulp", "password"), dest="pulp_password", + help="Password for Pulp authentication")) + + def _setup_pulp(): """ Connect to a Pulp server and pass authentication credentials. This only needs to be called once, but multiple calls won't hurt @@ -124,20 +146,6 @@ def _setup_pulp(): raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: - setup = get_option_parser() - try: - username = setup.cfp.get("packages:pulp", "username") - password = setup.cfp.get("packages:pulp", "password") - except ConfigParser.NoSectionError: - msg = "Packages: No [pulp] section found in bcfg2.conf" - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - except ConfigParser.NoOptionError: - msg = "Packages: Required option not found in bcfg2.conf: %s" % \ - sys.exc_info()[1] - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server @@ -145,7 +153,9 @@ def _setup_pulp(): int(serveropts['port']), serveropts['scheme'], serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) + PULPSERVER.set_basic_auth_credentials( + Bcfg2.Options.setup.pulp_username, + Bcfg2.Options.setup.pulp_password) server.set_active_server(PULPSERVER) return PULPSERVER @@ -357,10 +367,8 @@ class YumCollection(Collection): the default location. """ if not self._helper: # pylint: disable=W0212 - try: - self.__class__._helper = self.setup.cfp.get("packages:yum", - "helper") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.__class__._helper = Bcfg2.Options.setup.yum_helper + if not self.__class__._helper: # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") @@ -375,9 +383,7 @@ class YumCollection(Collection): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries @property def has_pulp_sources(self): @@ -413,15 +419,15 @@ class YumCollection(Collection): debuglevel="0", sslverify="0", reposdir="/dev/null") - if self.setup['debug']: + if Bcfg2.Options.setup.debug: mainopts['debuglevel'] = "5" - elif self.setup['verbose']: + elif Bcfg2.Options.setup.verbose: mainopts['debuglevel'] = "2" try: - for opt in self.setup.cfp.options("packages:yum"): + for opt, val in Bcfg2.Options.setup.yum_options.items(): if opt not in self.option_blacklist: - mainopts[opt] = self.setup.cfp.get("packages:yum", opt) + mainopts[opt] = val except ConfigParser.NoSectionError: pass @@ -543,8 +549,7 @@ class YumCollection(Collection): for key in needkeys: # figure out the path of the key on the client - keydir = self.setup.cfp.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") + keydir = Bcfg2.Options.setup.gpg_keypath remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() @@ -763,8 +768,7 @@ class YumCollection(Collection): """ Given a package tuple, return a dict of attributes suitable for applying to either a Package or an Instance tag """ - attrs = dict(version=self.setup.cfp.get("packages", "version", - default="auto")) + attrs = dict(version=Bcfg2.Options.setup.packages_version) if attrs['version'] == 'any' or not isinstance(pkgtup, tuple): return attrs @@ -923,14 +927,10 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - if self.setup['verbose']: + if Bcfg2.Options.setup.verbose: cmd.append("-v") if self.debug_flag: - if not self.setup['verbose']: - # ensure that running in debug gets -vv, even if - # verbose is not enabled - cmd.append("-v") - cmd.append("-v") + cmd.append("-d") cmd.append(command) self.debug_log("Packages: running %s" % " ".join(cmd)) @@ -1053,9 +1053,7 @@ class YumSource(Source): def use_yum(self): """ True if we should use the yum Python libraries, False otherwise """ - return HAS_YUM and self.setup.cfp.getboolean("packages:yum", - "use_yum_libraries", - default=False) + return HAS_YUM and Bcfg2.Options.setup.use_yum_libraries def save_state(self): """ If using the builtin yum parser, save state to diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py new file mode 100644 index 000000000..32db0b32d --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -0,0 +1,384 @@ +""" Libraries for bcfg2-yum-helper plugin, used if yum library support +is enabled. The yum libs have horrific memory leaks, so apparently +the right way to get around that in long-running processes it to have +a short-lived helper. No, seriously -- check out the yum-updatesd +code. It's pure madness. """ + +import os +import sys +import yum +import logging +import Bcfg2.Options +import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout +try: + import json +except ImportError: + import simplejson as json + + +def pkg_to_tuple(package): + """ json doesn't distinguish between tuples and lists, but yum + does, so we convert a package in list format to one in tuple + format """ + if isinstance(package, list): + return tuple(package) + else: + return package + + +def pkgtup_to_string(package): + """ given a package tuple, return a human-readable string + describing the package """ + if package[3] in ['auto', 'any']: + return package[0] + + rv = [package[0], "-"] + if package[2]: + rv.extend([package[2], ':']) + rv.extend([package[3], '-', package[4]]) + if package[1]: + rv.extend(['.', package[1]]) + return ''.join(str(e) for e in rv) + + +class YumHelper(object): + """ Yum helper base object """ + + def __init__(self, cfgfile, verbose=1): + self.cfgfile = cfgfile + self.yumbase = yum.YumBase() + # pylint: disable=E1121,W0212 + try: + self.yumbase.preconf.debuglevel = verbose + self.yumbase.preconf.fn = cfgfile + self.yumbase._getConfig() + except AttributeError: + self.yumbase._getConfig(cfgfile, debuglevel=verbose) + # pylint: enable=E1121,W0212 + self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 + self._groups = None + + def get_groups(self): + """ getter for the groups property """ + if self._groups is not None: + return self._groups + else: + return ["noarch"] + + def set_groups(self, groups): + """ setter for the groups property """ + self._groups = set(groups).union(["noarch"]) + + groups = property(get_groups, set_groups) + + def get_package_object(self, pkgtup, silent=False): + """ given a package tuple, get a yum package object """ + try: + matches = yum.packageSack.packagesNewestByName( + self.yumbase.pkgSack.searchPkgTuple(pkgtup)) + except yum.Errors.PackageSackError: + if not silent: + self.logger.warning("Package '%s' not found" % + self.get_package_name(pkgtup)) + matches = [] + except yum.Errors.RepoError: + err = sys.exc_info()[1] + self.logger.error("Temporary failure loading metadata for %s: %s" % + (self.get_package_name(pkgtup), err)) + matches = [] + + pkgs = self._filter_arch(matches) + if pkgs: + return pkgs[0] + else: + return None + + def get_group(self, group, ptype="default"): + """ Resolve a package group name into a list of packages """ + if group.startswith("@"): + group = group[1:] + + try: + if self.yumbase.comps.has_group(group): + group = self.yumbase.comps.return_group(group) + else: + self.logger.error("%s is not a valid group" % group) + return [] + except yum.Errors.GroupsError: + err = sys.exc_info()[1] + self.logger.warning(err) + return [] + + if ptype == "default": + return [p + for p, d in list(group.default_packages.items()) + if d] + elif ptype == "mandatory": + return [p + for p, m in list(group.mandatory_packages.items()) + if m] + elif ptype == "optional" or ptype == "all": + return group.packages + else: + self.logger.warning("Unknown group package type '%s'" % ptype) + return [] + + def _filter_arch(self, packages): + """ filter packages in the given list that do not have an + architecture in the list of groups for this client """ + matching = [] + for pkg in packages: + if pkg.arch in self.groups: + matching.append(pkg) + else: + self.logger.debug("%s has non-matching architecture (%s)" % + (pkg, pkg.arch)) + if matching: + return matching + else: + # no packages match architecture; we'll assume that the + # user knows what s/he is doing and this is a multiarch + # box. + return packages + + def get_package_name(self, package): + """ get the name of a package or virtual package from the + internal representation used by this Collection class """ + if isinstance(package, tuple): + if len(package) == 3: + return yum.misc.prco_tuple_to_string(package) + else: + return pkgtup_to_string(package) + else: + return str(package) + + def complete(self, packagelist): + """ resolve dependencies and generate a complete package list + from the given list of initial packages """ + packages = set() + unknown = set() + for pkg in packagelist: + if isinstance(pkg, tuple): + pkgtup = pkg + else: + pkgtup = (pkg, None, None, None, None) + pkgobj = self.get_package_object(pkgtup) + if not pkgobj: + self.logger.debug("Unknown package %s" % + self.get_package_name(pkg)) + unknown.add(pkg) + else: + if self.yumbase.tsInfo.exists(pkgtup=pkgobj.pkgtup): + self.logger.debug("%s added to transaction multiple times" + % pkgobj) + else: + self.logger.debug("Adding %s to transaction" % pkgobj) + self.yumbase.tsInfo.addInstall(pkgobj) + self.yumbase.resolveDeps() + + for txmbr in self.yumbase.tsInfo: + packages.add(txmbr.pkgtup) + return list(packages), list(unknown) + + +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock + def clean_cache(self): + """ clean the yum cache """ + for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", + "ExpireCache"]: + # for reasons that are entirely obvious, all of the yum + # API clean* methods return a tuple of 0 (zero, always + # zero) and a list containing a single message about how + # many files were deleted. so useful. thanks, yum. + msg = getattr(self.yumbase, "clean%s" % mdtype)()[1][0] + if not msg.startswith("0 "): + self.logger.info(msg) + + @acquire_lock + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + try: + repo.getGroups() + except yum.Errors.RepoMDError: + pass # this repo has no groups + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + # this does something with the groups cache as a side effect + self.yumbase.comps # pylint: disable=W0104 + + +class HelperSubcommand(Bcfg2.Options.Subcommand): + # the value to JSON encode and print out if the command fails + fallback = None + + # whether or not this command accepts input on stdin + accept_input = True + + def __init__(self): + Bcfg2.Options.Subcommand.__init__(self) + self.verbosity = 0 + if Bcfg2.Options.setup.debug: + self.verbosity = 5 + elif Bcfg2.Options.setup.verbose: + self.verbosity = 1 + + def run(self, setup): + try: + data = json.loads(sys.stdin.read()) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error decoding JSON input: %s" % + sys.exc_info()[1]) + print(json.dumps(self.fallback)) + return 2 + + try: + print(json.dumps(self._run(setup, data))) + except: # pylint: disable=W0702 + self.logger.error("Unexpected error running %s: %s" % + self.__class__.__name__.lower(), + sys.exc_info()[1], exc_info=1) + print(json.dumps(self.fallback)) + return 2 + return 0 + + def _run(self, setup, data): + raise NotImplementedError + + +class DepSolverSubcommand(HelperSubcommand): + def __init__(self): + HelperSubcommand.__init__(self) + self.depsolver = DepSolver(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class CacheManagerSubcommand(HelperSubcommand): + fallback = False + accept_input = False + + def __init__(self): + HelperSubcommand.__init__(self) + self.cachemgr = CacheManager(Bcfg2.Options.setup.yum_config, + self.verbosity) + + +class Clean(CacheManagerSubcommand): + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.clean_cache() + return True + + +class MakeCache(CacheManagerSubcommand): + def _run(self, setup, data): # pylint: disable=W0613 + self.cachemgr.populate_cache() + return True + + +class Complete(DepSolverSubcommand): + fallback = dict(packages=[], unknown=[]) + + def _run(self, _, data): + self.depsolver.groups = data['groups'] + self.fallback['unknown'] = data['packages'] + (packages, unknown) = self.depsolver.complete( + [pkg_to_tuple(p) for p in data['packages']]) + return dict(packages=list(packages), unknown=list(unknown)) + + +class GetGroups(DepSolverSubcommand): + def _run(self, _, data): + rv = dict() + for gdata in data: + if "type" in gdata: + packages = self.depsolver.get_group(gdata['group'], + ptype=gdata['type']) + else: + packages = self.depsolver.get_group(gdata['group']) + rv[gdata['group']] = list(packages) + return rv + + +Get_Groups = GetGroups + + +class CLI(Bcfg2.Options.CommandRegistry): + options = [ + Bcfg2.Options.PathOption( + "-c", "--yum-config", help="Yum config file"), + Bcfg2.Options.PositionalArgument( + "command", help="Yum helper command", + choices=['clean', 'complete', 'get_groups'])] + + def __init__(self): + Bcfg2.Options.CommandRegistry.__init__(self) + Bcfg2.Options.register_commands(self.__class__, globals().values(), + parent=HelperSubcommand) + parser = Bcfg2.Options.get_parser("Bcfg2 yum helper", + components=[self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + def run(self): + if not os.path.exists(Bcfg2.Options.setup.yum_config): + self.logger.error("Config file %s not found" % + Bcfg2.Options.setup.yum_config) + return 1 + return self.runcommand() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 20a75c678..e6240f39a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -7,21 +7,30 @@ import sys import glob import shutil import lxml.etree -import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ - MutableMapping +from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources from Bcfg2.Server.Statistics import track_statistics -#: The default path for generated yum configs -YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" -#: The default path for generated apt configs -APT_CONFIG_DEFAULT = \ - "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +def packages_boolean(value): + """ For historical reasons, the Packages booleans 'resolver' and + 'metadata' both accept "enabled" in addition to the normal boolean + values. """ + if value == 'disabled': + return False + elif value == 'enabled': + return True + else: + return value + + +class PackagesBackendAction(Bcfg2.Options.ComponentAction): + bases = ['Bcfg2.Server.Plugins.Packages'] + module = True class OnDemandDict(MutableMapping): @@ -86,6 +95,37 @@ class Packages(Bcfg2.Server.Plugin.Plugin, .. private-include: _build_packages""" + options = [ + Bcfg2.Options.Option( + cf=("packages", "backends"), dest="packages_backends", + help="Packages backends to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesBackendAction, default=['Yum', 'Apt', 'Pac']), + Bcfg2.Options.PathOption( + cf=("packages", "cache"), dest="packages_cache", + help="Path to the Packages cache", + default='<repository>/Packages/cache'), + Bcfg2.Options.Option( + cf=("packages", "resolver"), dest="packages_resolver", + help="Disable the Packages resolver", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "metadata"), dest="packages_metadata", + help="Disable all Packages metadata processing", + type=packages_boolean, default=True), + Bcfg2.Options.Option( + cf=("packages", "version"), dest="packages_version", + help="Set default Package entry version", default="auto", + choices=["auto", "any"]), + Bcfg2.Options.PathOption( + cf=("packages", "yum_config"), + help="The default path for generated yum configs", + default="/etc/yum.repos.d/bcfg2.repo"), + Bcfg2.Options.PathOption( + cf=("packages", "apt_config"), + help="The default path for generated apt configs", + default="/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list")] + #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. conflicts = ['Pkgmgr'] @@ -108,9 +148,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: Packages does a potentially tremendous amount of on-disk #: caching. ``cachepath`` holds the base directory to where #: data should be cached. - self.cachepath = \ - self.core.setup.cfp.get("packages", "cache", - default=os.path.join(self.data, 'cache')) + self.cachepath = Bcfg2.Options.setup.packages_cache #: Where Packages should store downloaded GPG key files self.keypath = os.path.join(self.cachepath, 'keys') @@ -186,40 +224,17 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :attr:`disableMetaData`) implies disabling the resolver. This property cannot be set. """ - if self.disableMetaData: - # disabling metadata without disabling the resolver Breaks - # Things - return True - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled", which are not handled according to the - # Python docs but appear to be handled properly by - # ConfigParser in at least some versions - return self.core.setup.cfp.get( - "packages", - "resolver", - default="enabled").lower() == "disabled" + # disabling metadata without disabling the resolver Breaks + # Things + return not Bcfg2.Options.setup.packages_metadata or \ + not Bcfg2.Options.setup.packages_resolver @property def disableMetaData(self): """ Report whether or not metadata processing is enabled. This property cannot be set. """ - try: - return not self.core.setup.cfp.getboolean("packages", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.core.setup.cfp.get( - "packages", - "metadata", - default="enabled").lower() == "disabled" + return not Bcfg2.Options.setup.packages_metadata def create_config(self, entry, metadata): """ Create yum/apt config for the specified client. @@ -268,9 +283,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ if entry.tag == 'Package': collection = self.get_collection(metadata) - entry.set('version', self.core.setup.cfp.get("packages", - "version", - default="auto")) + entry.set('version', Bcfg2.Options.setup.packages_version) entry.set('type', collection.ptype) elif entry.tag == 'Path': self.create_config(entry, metadata) @@ -299,14 +312,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): + if entry.get("name") in [Bcfg2.Options.setup.apt_config, + Bcfg2.Options.setup.yum_config]: return True return False @@ -339,7 +346,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :returns: None """ collection = self.get_collection(metadata) - indep = lxml.etree.Element('Independent') + indep = lxml.etree.Element('Independent', name=self.__class__.__name__) self._build_packages(metadata, indep, structures, collection=collection) collection.build_extra_structures(indep) diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 293ec8e1a..c85bc7d41 100644 --- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -1,12 +1,9 @@ '''This module implements a package management scheme for all images''' -import os import re import sys -import glob import logging import lxml.etree -import Bcfg2.Server.Lint import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError @@ -295,44 +292,3 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): def HandleEntry(self, entry, metadata): self.BindEntry(entry, metadata) - - -class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): - """ Find duplicate :ref:`Pkgmgr - <server-plugins-generators-pkgmgr>` entries with the same - priority. """ - - def Run(self): - pset = set() - for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', - '*.xml')): - if self.HandlesFile(pfile): - xdata = lxml.etree.parse(pfile).getroot() - # get priority, type, group - priority = xdata.get('priority') - ptype = xdata.get('type') - for pkg in xdata.xpath("//Package"): - if pkg.getparent().tag == 'Group': - grp = pkg.getparent().get('name') - if (type(grp) is not str and - grp.getparent().tag == 'Group'): - pgrp = grp.getparent().get('name') - else: - pgrp = 'none' - else: - grp = 'none' - pgrp = 'none' - ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) - # check if package is already listed with same - # priority, type, grp - if ptuple in pset: - self.LintError( - "duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) - else: - pset.add(ptuple) - - @classmethod - def Errors(cls): - return {"duplicate-packages": "error"} diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7b85f180d..0d264a5a6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -12,10 +12,19 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Server.Statistics import track_statistics -try: - from django.db import models - from django.core.exceptions import MultipleObjectsReturned - HAS_DJANGO = True +HAS_DJANGO = False +ProbesDataModel = None +ProbesGroupModel = None + + +def load_django_models(): + global ProbesDataModel, ProbesGroupModel, HAS_DJANGO + try: + from django.db import models + HAS_DJANGO = True + except ImportError: + HAS_DJANGO = False + return class ProbesDataModel(models.Model, Bcfg2.Server.Plugin.PluginDatabaseModel): @@ -30,8 +39,7 @@ try: """ The database model for storing probe groups """ hostname = models.CharField(max_length=255) group = models.CharField(max_length=255) -except ImportError: - HAS_DJANGO = False + try: import json @@ -120,11 +128,15 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): bangline = re.compile(r'^#!\s*(?P<interpreter>.*)$') basename_is_regex = True - def __init__(self, path, encoding, plugin_name): + options = [ + Bcfg2.Options.BooleanOption( + cf=('probes', 'use_database'), dest="probes_db", + help="Use database capabilities of the Probes plugin")] + + def __init__(self, path, plugin_name): self.plugin_name = plugin_name Bcfg2.Server.Plugin.EntrySet.__init__(self, r'[0-9A-Za-z_\-]+', path, - Bcfg2.Server.Plugin.SpecificData, - encoding) + Bcfg2.Server.Plugin.SpecificData) Bcfg2.Server.FileMonitor.get_fam().AddMonitor(path, self) def HandleEvent(self, event): @@ -196,8 +208,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: - self.probes = ProbeSet(self.data, core.setup['encoding'], - self.name) + self.probes = ProbeSet(self.data, self.name) except: err = sys.exc_info()[1] raise Bcfg2.Server.Plugin.PluginInitError(err) @@ -260,7 +271,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, ProbesGroupsModel.objects.get_or_create( hostname=client.hostname, group=group) - except MultipleObjectsReturned: + except ProbesGroupsModel.MultipleObjectsReturned: ProbesGroupsModel.objects.filter(hostname=client.hostname, group=group).delete() ProbesGroupsModel.objects.get_or_create( diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 8e54da19b..bbc00556b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -7,7 +7,7 @@ import sys import copy import logging import lxml.etree -from Bcfg2.Options import get_option_parser +import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError @@ -40,18 +40,14 @@ class PropertyFile(object): .. automethod:: _write """ self.name = name - self.setup = get_option_parser() def write(self): """ Write the data in this data structure back to the property file. This public method performs checking to ensure that writing is possible and then calls :func:`_write`. """ - if not self.setup.cfp.getboolean("properties", "writes_enabled", - default=True): - msg = "Properties files write-back is disabled in the " + \ - "configuration" - LOGGER.error(msg) - raise PluginExecutionError(msg) + if not Bcfg2.Options.setup.writes_enabled: + raise PluginExecutionError("Properties files write-back is " + "disabled in the configuration") try: self.validate_data() except PluginExecutionError: @@ -199,7 +195,7 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): validate_data.__doc__ = PropertyFile.validate_data.__doc__ def get_additional_data(self, metadata): - if self.setup.cfp.getboolean("properties", "automatch", default=False): + if Bcfg2.Options.setup.automatch: default_automatch = "true" else: default_automatch = "false" @@ -221,6 +217,13 @@ class Properties(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ The properties plugin maps property files into client metadata instances. """ + options = [ + Bcfg2.Options.BooleanOption( + cf=("properties", "writes_enabled"), default=True, + help="Enable or disable Properties write-back"), + Bcfg2.Options.BooleanOption( + cf=("properties", "automatch"), + help="Enable Properties automatch")] #: Extensions that are understood by Properties. extensions = ["xml"] diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 3354763d4..9d1019441 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -5,11 +5,10 @@ import time import platform import traceback import lxml.etree -from Bcfg2.Reporting.Transport import load_transport_from_config, \ - TransportError -from Bcfg2.Options import REPORTING_COMMON_OPTIONS +import Bcfg2.Options +from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ - Debuggable, PluginInitError, PluginExecutionError + PluginInitError, PluginExecutionError # required for reporting try: @@ -33,9 +32,11 @@ def _rpc_call(method): # pylint: disable=W0223 -class Reporting(Statistics, Threaded, PullSource, Debuggable): +class Reporting(Statistics, Threaded, PullSource): """ Unified statistics and reporting plugin """ - __rmi__ = Debuggable.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + __rmi__ = Statistics.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry'] + + options = [Bcfg2.Options.Common.reporting_transport] CLIENT_METADATA_FIELDS = ('profile', 'bundles', 'aliases', 'addresses', 'groups', 'categories', 'uuid', 'version') @@ -44,14 +45,10 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): Statistics.__init__(self, core, datastore) PullSource.__init__(self) Threaded.__init__(self) - Debuggable.__init__(self) self.whoami = platform.node() self.transport = None - core.setup.update(REPORTING_COMMON_OPTIONS) - core.setup.reparse() - if not HAS_SOUTH: msg = "Django south is required for Reporting" self.logger.error(msg) @@ -59,17 +56,15 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): def start_threads(self): try: - self.transport = load_transport_from_config(self.core.setup) + self.transport = Bcfg2.Options.setup.reporting_transport() except TransportError: - msg = "%s: Failed to load transport: %s" % \ - (self.name, traceback.format_exc().splitlines()[-1]) - self.logger.error(msg) - raise PluginInitError(msg) + raise PluginInitError("%s: Failed to instantiate transport: %s" % + (self.name, sys.exc_info()[1])) if self.debug_flag: self.transport.set_debug(self.debug_flag) def set_debug(self, debug): - rv = Debuggable.set_debug(self, debug) + rv = Statistics.set_debug(self, debug) if self.transport is not None: self.transport.set_debug(debug) return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 3d4e8671d..541116db3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -1,6 +1,7 @@ """This generator provides rule-based entry mappings.""" import re +import Bcfg2.Options import Bcfg2.Server.Plugin @@ -8,6 +9,11 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" __author__ = 'bcfg-dev@mcs.anl.gov' + options = Bcfg2.Server.Plugin.PrioDir.options + [ + Bcfg2.Options.BooleanOption( + cf=("rules", "regex"), dest="rules_regex", + help="Allow regular expressions in Rules")] + def __init__(self, core, datastore): Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) self._regex_cache = dict() @@ -42,4 +48,4 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): @property def _regex_enabled(self): """ Return True if rules regexes are enabled, False otherwise """ - return self.core.setup.cfp.getboolean("rules", "regex", default=False) + return Bcfg2.Options.setup.rules_regex diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index f350a7761..c858b881b 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -7,8 +7,9 @@ import socket import shutil import logging import tempfile -from itertools import chain +import Bcfg2.Options import Bcfg2.Server.Plugin +from itertools import chain from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Compat import any, u_str, b64encode # pylint: disable=W0622 @@ -20,8 +21,7 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): """ class to handle key data for HostKeyEntrySet """ def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) self.encoding = encoding def __lt__(self, other): @@ -62,27 +62,30 @@ class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle all kinds of host keys """ def __init__(self, basename, path): if basename.startswith("ssh_host_key"): - encoding = "base64" + self.encoding = "base64" else: - encoding = None - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData, - encoding) + self.encoding = None + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file'} - if encoding is not None: - self.metadata['encoding'] = encoding + if self.encoding is not None: + self.metadata['encoding'] = self.encoding if basename.endswith('.pub'): self.metadata['mode'] = '0644' else: self.metadata['mode'] = '0600' + def get_keydata_object(self, filepath, specificity): + return KeyData(filepath, specificity, + self.encoding or Bcfg2.Options.setup.encoding) + class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle the ssh_known_hosts file """ def __init__(self, path): Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path, - KeyData, None) + KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file', diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index b21732666..74d8833f4 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -3,17 +3,13 @@ certificates and their keys. """ import os import sys -import logging import tempfile import lxml.etree -import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Utils import Executor from Bcfg2.Compat import ConfigParser from Bcfg2.Server.Plugin import PluginExecutionError -LOGGER = logging.getLogger(__name__) - class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): """ Base class to handle key.xml and cert.xml """ @@ -31,10 +27,9 @@ class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): metadata.hostname, self.name)) elif len(entries) > 1: - LOGGER.warning("More than one matching %s entry found for %s in " - "%s; using first match" % (self.tag, - metadata.hostname, - self.name)) + self.logger.warning( + "More than one matching %s entry found for %s in %s; " + "using first match" % (self.tag, metadata.hostname, self.name)) rv = dict() for attr, default in self.attrs.items(): val = entries[0].get(attr.lower(), default) @@ -84,9 +79,9 @@ class SSLCADataFile(Bcfg2.Server.Plugin.SpecificData): class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): """ Entry set to handle SSLCA entries and XML files """ - def __init__(self, _, path, entry_type, encoding, parent=None): + def __init__(self, _, path, entry_type, parent=None): Bcfg2.Server.Plugin.EntrySet.__init__(self, os.path.basename(path), - path, entry_type, encoding) + path, entry_type) self.parent = parent self.key = None self.cert = None @@ -361,10 +356,32 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ __author__ = 'g.hagger@gmail.com' + + options = Bcfg2.Server.Plugin.GroupSpool.options + [ + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.PathOption( + cf=("sslca_*", "config"), + help="Path to the openssl config for the CA"), + Bcfg2.Options.Option( + cf=("sslca_*", "passphrase"), + help="Passphrase for the CA private key"), + Bcfg2.Options.PathOption( + cf=("sslca_*", "chaincert"), + help="Path to the SSL chaining certificate for verification"), + Bcfg2.Options.BooleanOption( + cf=("sslca_*", "root_ca"), + help="Whether or not <chaincert> is a root CA (as opposed to " + "an intermediate cert"))] + # python 2.5 doesn't support mixing *magic and keyword arguments es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) es_child_cls = SSLCADataFile def get_ca(self, name): """ get a dict describing a CA from the config file """ - return dict(self.core.setup.cfp.items("sslca_%s" % name)) + rv = dict() + prefix = "sslca_%s_" % name + for attr in dir(Bcfg2.Options.setup): + if attr.startswith(prefix): + rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index dfe864d48..679e38ff9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -4,8 +4,8 @@ additional XML-RPC methods for committing data to the repository and updating the repository. """ import sys +import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser try: import pysvn HAS_SVN = True @@ -16,6 +16,21 @@ except ImportError: class Svn(Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" + options = Bcfg2.Server.Plugin.Version.options + [ + Bcfg2.Options.Option( + cf=("svn", "conflict_resolution"), dest="svn_conflict_resolution", + type=lambda v: v.replace("-", "_"), + choices=dir(pysvn.wc_conflict_choice), + default=pysvn.wc_conflict_choice.postpone, + help="SVN conflict resolution method"), + Bcfg2.Options.Option( + cf=("svn", "user"), dest="svn_user", help="SVN username"), + Bcfg2.Options.Option( + cf=("svn", "password"), dest="svn_password", help="SVN password"), + Bcfg2.Options.BooleanOption( + cf=("svn", "always_trust"), dest="svn_trust_ssl", + help="Always trust SSL certs from SVN server")] + __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".svn" if HAS_SVN: @@ -36,62 +51,29 @@ class Svn(Bcfg2.Server.Plugin.Version): self.cmd = Executor() else: self.client = pysvn.Client() - # pylint: disable=E1101 - choice = pysvn.wc_conflict_choice.postpone - try: - resolution = self.core.setup.cfp.get( - "svn", - "conflict_resolution").replace('-', '_') - if resolution in ["edit", "launch", "working"]: - self.logger.warning("Svn: Conflict resolver %s requires " - "manual intervention, using %s" % - choice) - else: - choice = getattr(pysvn.wc_conflict_choice, resolution) - except AttributeError: - self.logger.warning("Svn: Conflict resolver %s does not " - "exist, using %s" % choice) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: No conflict resolution method " - "selected, using %s" % choice) - # pylint: enable=E1101 self.debug_log("Svn: Conflicts will be resolved with %s" % - choice) - self.client.callback_conflict_resolver = \ - self.get_conflict_resolver(choice) + Bcfg2.Options.setup.svn_conflict_resolution) + self.client.callback_conflict_resolver = self.conflict_resolver - try: - if self.core.setup.cfp.get( - "svn", - "always_trust").lower() == "true": - self.client.callback_ssl_server_trust_prompt = \ - self.ssl_server_trust_prompt - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.debug("Svn: Using subversion cache for SSL " - "certificate trust") + if Bcfg2.Options.setup.svn_trust_ssl: + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt - try: - if (self.core.setup.cfp.get("svn", "user") and - self.core.setup.cfp.get("svn", "password")): - self.client.callback_get_login = \ - self.get_login - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.logger.info("Svn: Using subversion cache for " - "password-based authetication") + if (Bcfg2.Options.setup.svn_user and + Bcfg2.Options.setup.svn_password): + self.client.callback_get_login = self.get_login self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) - # pylint: disable=W0613 - def get_login(self, realm, username, may_save): + def get_login(self, realm, username, may_save): # pylint: disable=W0613 """ PySvn callback to get credentials for HTTP basic authentication """ self.logger.debug("Svn: Logging in with username: %s" % - self.core.setup.cfp.get("svn", "user")) - return True, \ - self.core.setup.cfp.get("svn", "user"), \ - self.core.setup.cfp.get("svn", "password"), \ - False - # pylint: enable=W0613 + Bcfg2.Options.setup.svn_user) + return (True, + Bcfg2.Options.setup.svn_user, + Bcfg2.Options.setup.svn_password, + False) def ssl_server_trust_prompt(self, trust_dict): """ PySvn callback to always trust SSL certificates from SVN server """ @@ -102,22 +84,19 @@ class Svn(Bcfg2.Server.Plugin.Version): trust_dict['realm'])) return True, trust_dict['failures'], False - def get_conflict_resolver(self, choice): - """ Get a PySvn conflict resolution callback """ - def callback(conflict_description): - """ PySvn callback function to resolve conflicts """ - self.logger.info("Svn: Resolving conflict for %s with %s" % - (conflict_description['path'], choice)) - return choice, None, False - - return callback + def conflict_resolver(self, conflict_description): + """ PySvn callback function to resolve conflicts """ + self.logger.info("Svn: Resolving conflict for %s with %s" % + (conflict_description['path'], + Bcfg2.Options.setup.svn_conflict_resolution)) + return Bcfg2.Options.setup.svn_conflict_resolution, None, False def get_revision(self): """Read svn revision information for the Bcfg2 repository.""" msg = None if HAS_SVN: try: - info = self.client.info(self.vcs_root) + info = self.client.info(Bcfg2.Options.setup.vcs_root) self.revision = info.revision self.svn_root = info.url return str(self.revision.number) @@ -125,7 +104,7 @@ class Svn(Bcfg2.Server.Plugin.Version): msg = "Svn: Failed to get revision: %s" % sys.exc_info()[1] else: result = self.cmd.run(["env LC_ALL=C", "svn", "info", - self.vcs_root], + Bcfg2.Options.setup.vcs_root], shell=True) if result.success: self.revision = [line.split(': ')[1] @@ -141,7 +120,8 @@ class Svn(Bcfg2.Server.Plugin.Version): '''Svn.Update() => True|False\nUpdate svn working copy\n''' try: old_revision = self.revision.number - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] except pysvn.ClientError: # pylint: disable=E1101 err = sys.exc_info()[1] # try to be smart about the error we got back @@ -163,7 +143,7 @@ class Svn(Bcfg2.Server.Plugin.Version): self.logger.debug("repository is current") else: self.logger.info("Updated %s from revision %s to %s" % - (self.vcs_root, old_revision, + (Bcfg2.Options.setup.vcs_root, old_revision, self.revision.number)) return True @@ -176,10 +156,11 @@ class Svn(Bcfg2.Server.Plugin.Version): return False try: - self.revision = self.client.checkin([self.vcs_root], + self.revision = self.client.checkin([Bcfg2.Options.setup.vcs_root], 'Svn: autocommit', recurse=True) - self.revision = self.client.update(self.vcs_root, recurse=True)[0] + self.revision = self.client.update(Bcfg2.Options.setup.vcs_root, + recurse=True)[0] self.logger.info("Svn: Commited changes. At %s" % self.revision.number) return True diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 77bdd6576..a32b7dea2 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -4,7 +4,6 @@ import re import imp import sys import logging -import Bcfg2.Server.Lint import Bcfg2.Server.Plugin LOGGER = logging.getLogger(__name__) @@ -93,75 +92,3 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.entries.values()]) - - -class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): - """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper - <server-plugins-connectors-templatehelper>` modules are valid. - This can check for: - - * A TemplateHelper module that cannot be imported due to syntax or - other compile-time errors; - * A TemplateHelper module that does not have an ``__export__`` - attribute, or whose ``__export__`` is not a list; - * Bogus symbols listed in ``__export__``, including symbols that - don't exist, that are reserved, or that start with underscores. - """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.reserved_keywords = dir(HelperModule("foo.py")) - - def Run(self): - for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper.name): - self.check_helper(helper.name) - - def check_helper(self, helper): - """ Check a single helper module. - - :param helper: The filename of the helper module - :type helper: string - """ - module_name = MODULE_RE.search(helper).group(1) - - try: - module = imp.load_source(safe_module_name(module_name), helper) - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - self.LintError("templatehelper-import-error", - "Failed to import %s: %s" % - (helper, err)) - return - - if not hasattr(module, "__export__"): - self.LintError("templatehelper-no-export", - "%s has no __export__ list" % helper) - return - elif not isinstance(module.__export__, list): - self.LintError("templatehelper-nonlist-export", - "__export__ is not a list in %s" % helper) - return - - for sym in module.__export__: - if not hasattr(module, sym): - self.LintError("templatehelper-nonexistent-export", - "%s: exported symbol %s does not exist" % - (helper, sym)) - elif sym in self.reserved_keywords: - self.LintError("templatehelper-reserved-export", - "%s: exported symbol %s is reserved" % - (helper, sym)) - elif sym.startswith("_"): - self.LintError("templatehelper-underscore-export", - "%s: exported symbol %s starts with underscore" - % (helper, sym)) - - @classmethod - def Errors(cls): - return {"templatehelper-import-error": "error", - "templatehelper-no-export": "error", - "templatehelper-nonlist-export": "error", - "templatehelper-nonexistent-export": "error", - "templatehelper-reserved-export": "error", - "templatehelper-underscore-export": "warning"} diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 862fd98c1..646124fcc 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -15,6 +15,10 @@ from Bcfg2.Compat import xmlrpclib, SimpleXMLRPCServer, SocketServer, \ b64decode +class XMLRPCACLCheckException(Exception): + """ Raised when ACL checks fail on an RPC request """ + + class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): """ An XML-RPC dispatcher. """ @@ -33,6 +37,8 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): def _marshaled_dispatch(self, address, data): params, method = xmlrpclib.loads(data) + if not self.instance.check_acls(address, method): + raise XMLRPCACLCheckException try: if '.' not in method: params = (address, ) + params @@ -42,12 +48,12 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): response = (response.decode('utf-8'), ) else: response = (response, ) - raw_response = xmlrpclib.dumps(response, methodresponse=1, + raw_response = xmlrpclib.dumps(response, methodresponse=True, allow_none=self.allow_none, encoding=self.encoding) except xmlrpclib.Fault: fault = sys.exc_info()[1] - raw_response = xmlrpclib.dumps(fault, + raw_response = xmlrpclib.dumps(fault, methodresponse=True, allow_none=self.allow_none, encoding=self.encoding) except: @@ -56,7 +62,8 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): # report exception back to server raw_response = xmlrpclib.dumps( xmlrpclib.Fault(1, "%s:%s" % (err[0].__name__, err[1])), - allow_none=self.allow_none, encoding=self.encoding) + methodresponse=True, allow_none=self.allow_none, + encoding=self.encoding) return raw_response @@ -209,9 +216,8 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): password = "" cert = self.request.getpeercert() client_address = self.request.getpeername() - return (self.server.instance.authenticate(cert, username, - password, client_address) and - self.server.instance.check_acls(client_address[0], None)) + return self.server.instance.authenticate(cert, username, + password, client_address) def parse_request(self): """Extends parse_request. @@ -241,7 +247,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): try: select.select([self.rfile.fileno()], [], [], 3) except select.error: - print("got select timeout") + self.logger.error("Got select timeout") raise chunk_size = min(size_remaining, max_chunk_size) L.append(self.rfile.read(chunk_size).decode('utf-8')) @@ -251,7 +257,12 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): data) if sys.hexversion >= 0x03000000: response = response.encode('utf-8') + except XMLRPCACLCheckException: + self.send_error(401, self.responses[401][0]) + self.end_headers() except: # pylint: disable=W0702 + self.logger.error("Unexpected dispatch error for %s: %s" % + (self.client_address, sys.exc_info()[1])) try: self.send_response(500) self.end_headers() @@ -262,12 +273,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): raise else: # got a valid XML RPC response - # first, check ACLs client_address = self.request.getpeername() - method = xmlrpclib.loads(data)[1] - if not self.server.instance.check_acls(client_address, method): - self.send_error(401, self.responses[401][0]) - self.end_headers() try: self.send_response(200) self.send_header("Content-type", "text/xml") diff --git a/src/lib/Bcfg2/Server/Test.py b/src/lib/Bcfg2/Server/Test.py new file mode 100644 index 000000000..912a8f19c --- /dev/null +++ b/src/lib/Bcfg2/Server/Test.py @@ -0,0 +1,281 @@ +""" bcfg2-test libraries and CLI """ + +import os +import sys +import shlex +import signal +import fnmatch +import logging +import Bcfg2.Logger +import Bcfg2.Server.Core +from math import ceil +from nose.core import TestProgram +from nose.suite import LazySuite +from unittest import TestCase + +try: + from multiprocessing import Process, Queue, active_children + HAS_MULTIPROC = True +except ImportError: + HAS_MULTIPROC = False + active_children = lambda: [] # pylint: disable=C0103 + + +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr + + +class CapturingLogger(object): + """ Fake logger that captures logging output so that errors are + only displayed for clients that fail tests """ + def __init__(self, *args, **kwargs): # pylint: disable=W0613 + self.output = [] + + def error(self, msg): + """ discard error messages """ + self.output.append(msg) + + def warning(self, msg): + """ discard error messages """ + self.output.append(msg) + + def info(self, msg): + """ discard error messages """ + self.output.append(msg) + + def debug(self, msg): + """ discard error messages """ + self.output.append(msg) + + def reset_output(self): + """ Reset the captured output """ + self.output = [] + + +class ClientTestFromQueue(TestCase): + """ A test case that tests a value that has been enqueued by a + child test process. ``client`` is the name of the client that has + been tested; ``result`` is the result from the :class:`ClientTest` + test. ``None`` indicates a successful test; a string value + indicates a failed test; and an exception indicates an error while + running the test. """ + __test__ = False # Do not collect + + def __init__(self, client, result): + TestCase.__init__(self) + self.client = client + self.result = result + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ parse the result from this test """ + if isinstance(self.result, Exception): + raise self.result + assert self.result is None, self.result + + +class ClientTest(TestCase): + """ A test case representing the build of all of the configuration for + a single host. Checks that none of the build config entities has + had a failure when it is building. Optionally ignores some config + files that we know will cause errors (because they are private + files we don't have access to, for instance) """ + __test__ = False # Do not collect + divider = "-" * 70 + + def __init__(self, core, client, ignore=None): + TestCase.__init__(self) + self.core = core + self.core.logger = CapturingLogger() + self.client = client + if ignore is None: + self.ignore = dict() + else: + self.ignore = ignore + + def ignore_entry(self, tag, name): + """ return True if an error on a given entry should be ignored + """ + if tag in self.ignore: + if name in self.ignore[tag]: + return True + else: + # try wildcard matching + for pattern in self.ignore[tag]: + if fnmatch.fnmatch(name, pattern): + return True + return False + + def shortDescription(self): + return "Building configuration for %s" % self.client + + def runTest(self): + """ run this individual test """ + config = self.core.BuildConfiguration(self.client) + output = self.core.logger.output[:] + if output: + output.append(self.divider) + self.core.logger.reset_output() + + # check for empty client configuration + assert len(config.findall("Bundle")) > 0, \ + "\n".join(output + ["%s has no content" % self.client]) + + # check for missing bundles + metadata = self.core.build_metadata(self.client) + sbundles = [el.get('name') for el in config.findall("Bundle")] + missing = [b for b in metadata.bundles if b not in sbundles] + assert len(missing) == 0, \ + "\n".join(output + ["Configuration is missing bundle(s): %s" % + ':'.join(missing)]) + + # check for unknown packages + unknown_pkgs = [el.get("name") + for el in config.xpath('//Package[@type="unknown"]') + if not self.ignore_entry(el.tag, el.get("name"))] + assert len(unknown_pkgs) == 0, \ + "Configuration contains unknown packages: %s" % \ + ", ".join(unknown_pkgs) + + failures = [] + msg = output + ["Failures:"] + for failure in config.xpath('//*[@failure]'): + if not self.ignore_entry(failure.tag, failure.get('name')): + failures.append(failure) + msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), + failure.get("failure"))) + + assert len(failures) == 0, "\n".join(msg) + + def __str__(self): + return "ClientTest(%s)" % self.client + + id = __str__ + + +class CLI(object): + options = [ + Bcfg2.Options.PositionalArgument( + "clients", help="Specific clients to build", nargs="*"), + Bcfg2.Options.Option( + "--nose-options", cf=("bcfg2_test", "nose_options"), + type=shlex.split, default=[], + help='Options to pass to nosetests. Only honored with ' + '--children 0'), + Bcfg2.Options.Option( + "--ignore", cf=('bcfg2_test', 'ignore_entries'), default=[], + dest="test_ignore", type=Bcfg2.Options.Types.comma_list, + help='Ignore these entries if they fail to build'), + Bcfg2.Options.Option( + "--children", cf=('bcfg2_test', 'children'), default=0, type=int, + help='Spawn this number of children for bcfg2-test (python 2.6+)')] + + def __init__(self): + parser = Bcfg2.Options.get_parser( + description="Verify that all clients build without failures", + components=[Bcfg2.Server.Core.Core, self]) + parser.parse() + self.logger = logging.getLogger(parser.prog) + + if Bcfg2.Options.setup.children and not HAS_MULTIPROC: + self.logger.warning("Python multiprocessing library not found, " + "running with no children") + Bcfg2.Options.setup.children = 0 + + def get_core(self): + """ Get a server core, with events handled """ + core = Bcfg2.Server.Core.Core() + core.load_plugins() + core.block_for_fam_events(handle_events=True) + signal.signal(signal.SIGINT, get_sigint_handler(core)) + return core + + def get_ignore(self): + """ Get a dict of entry tags and names to + ignore errors from """ + ignore = dict() + for entry in Bcfg2.Options.setup.test_ignore: + tag, name = entry.split(":") + try: + ignore[tag].append(name) + except KeyError: + ignore[tag] = [name] + return ignore + + def run_child(self, clients, queue): + """ Run tests for the given clients in a child process, returning + results via the given Queue """ + core = self.get_core() + ignore = self.get_ignore() + for client in clients: + try: + ClientTest(core, client, ignore).runTest() + queue.put((client, None)) + except AssertionError: + queue.put((client, str(sys.exc_info()[1]))) + except: + queue.put((client, sys.exc_info()[1])) + + core.shutdown() + + def run(self): + core = self.get_core() + clients = Bcfg2.Options.setup.clients or core.metadata.clients + ignore = self.get_ignore() + + if Bcfg2.Options.setup.children: + if Bcfg2.Options.setup.children > len(clients): + self.logger.info("Refusing to spawn more children than " + "clients to test, setting children=%s" % + len(clients)) + Bcfg2.Options.setup.children = len(clients) + perchild = int(ceil(len(clients) / + float(Bcfg2.Options.setup.children + 1))) + queue = Queue() + for child in range(Bcfg2.Options.setup.children): + start = child * perchild + end = (child + 1) * perchild + child = Process(target=self.run_child, + args=(clients[start:end], queue)) + child.start() + + def generate_tests(): + """ Read test results for the clients """ + start = Bcfg2.Options.setup.children * perchild + for client in clients[start:]: + yield ClientTest(core, client, ignore) + + for i in range(start): # pylint: disable=W0612 + yield ClientTestFromQueue(*queue.get()) + else: + def generate_tests(): + """ Run tests for the clients """ + for client in clients: + yield ClientTest(core, client, ignore) + + result = TestProgram( + argv=sys.argv[:1] + Bcfg2.Options.setup.nose_options, + suite=LazySuite(generate_tests), exit=False) + + # block until all children have completed -- should be + # immediate since we've already gotten all the results we + # expect + for child in active_children(): + child.join() + + core.shutdown() + if result.success: + os._exit(0) # pylint: disable=W0212 + else: + os._exit(1) # pylint: disable=W0212 diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 370854881..51cc835dc 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -1,44 +1,64 @@ """ Django database models for all plugins """ import sys -import copy import logging import Bcfg2.Options import Bcfg2.Server.Plugins from Bcfg2.Compat import walk_packages -from django.db import models LOGGER = logging.getLogger('Bcfg2.Server.models') MODELS = [] -def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): +def _get_all_plugins(): + rv = [] + for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, + prefix="Bcfg2.Server.Plugins."): + module = submodule[1].rsplit('.', 1)[-1] + if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: + # we only include direct children of + # Bcfg2.Server.Plugins -- e.g., all_plugins should + # include Bcfg2.Server.Plugins.Cfg, but not + # Bcfg2.Server.Plugins.Cfg.CfgInfoXML + rv.append(module) + return rv + + +_ALL_PLUGINS = _get_all_plugins() + + +class _OptionContainer(object): + # we want to provide a different default plugin list -- + # namely, _all_ plugins, so that the database is guaranteed to + # work, even if /etc/bcfg2.conf isn't set up properly + options = [ + Bcfg2.Options.Option( + cf=('server', 'plugins'), type=Bcfg2.Options.Types.comma_list, + default=_ALL_PLUGINS, dest="models_plugins", + action=Bcfg2.Options.PluginsAction)] + + @staticmethod + def options_parsed_hook(): + # basic invocation to ensure that a default set of models is + # loaded, and thus that this module will always work. + load_models() + +Bcfg2.Options.get_parser().add_component(_OptionContainer) + + +def load_models(plugins=None): """ load models from plugins specified in the config """ + # this has to be imported after options are parsed, because Django + # finalizes its settings as soon as it's loaded, which means that + # if we import this before Bcfg2.settings has been populated, + # Django gets a null configuration, and subsequent updates to + # Bcfg2.settings won't help. + from django.db import models global MODELS - if plugins is None: - # we want to provide a different default plugin list -- - # namely, _all_ plugins, so that the database is guaranteed to - # work, even if /etc/bcfg2.conf isn't set up properly - plugin_opt = copy.deepcopy(Bcfg2.Options.SERVER_PLUGINS) - all_plugins = [] - for submodule in walk_packages(path=Bcfg2.Server.Plugins.__path__, - prefix="Bcfg2.Server.Plugins."): - module = submodule[1].rsplit('.', 1)[-1] - if submodule[1] == "Bcfg2.Server.Plugins.%s" % module: - # we only include direct children of - # Bcfg2.Server.Plugins -- e.g., all_plugins should - # include Bcfg2.Server.Plugins.Cfg, but not - # Bcfg2.Server.Plugins.Cfg.CfgInfoXML - all_plugins.append(module) - plugin_opt.default = all_plugins - - setup = Bcfg2.Options.get_option_parser() - setup.add_option("plugins", plugin_opt) - setup.add_option("configfile", Bcfg2.Options.CFILE) - setup.reparse(argv=[Bcfg2.Options.CFILE.cmd, cfile]) - plugins = setup['plugins'] + if not plugins: + plugins = Bcfg2.Options.setup.models_plugins if MODELS: # load_models() has been called once, so first unload all of @@ -49,45 +69,22 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): delattr(sys.modules[__name__], model) MODELS = [] - for plugin in plugins: - try: - mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - plugin).Server.Plugins, plugin) - except ImportError: - try: - err = sys.exc_info()[1] - mod = __import__(plugin) - except: # pylint: disable=W0702 - if plugins != plugin_opt.default: - # only produce errors if the default plugin list - # was not used -- i.e., if the config file was set - # up. don't produce errors when trying to load - # all plugins, IOW. the error from the first - # attempt to import is probably more accurate than - # the second attempt. - LOGGER.error("Failed to load plugin %s: %s" % (plugin, - err)) - continue + for mod in plugins: for sym in dir(mod): obj = getattr(mod, sym) - if hasattr(obj, "__bases__") and models.Model in obj.__bases__: + if isinstance(obj, type) and issubclass(obj, models.Model): setattr(sys.modules[__name__], sym, obj) MODELS.append(sym) -# basic invocation to ensure that a default set of models is loaded, -# and thus that this module will always work. -load_models(quiet=True) - - -class InternalDatabaseVersion(models.Model): - """ Object that tell us to which version the database is """ - version = models.IntegerField() - updated = models.DateTimeField(auto_now_add=True) + class InternalDatabaseVersion(models.Model): + """ Object that tell us to which version the database is """ + version = models.IntegerField() + updated = models.DateTimeField(auto_now_add=True) - def __str__(self): - return "version %d updated the %s" % (self.version, + def __str__(self): + return "version %d updated %s" % (self.version, self.updated.isoformat()) - class Meta: # pylint: disable=C0111,W0232 - app_label = "reports" - get_latest_by = "version" + class Meta: # pylint: disable=C0111,W0232 + app_label = "reports" + get_latest_by = "version" |