From 7c31e9544d325bfc869cba1d15cbc57f1d6a9aff Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 19 Jul 2012 17:18:57 -0400 Subject: added CherryPy-based server core --- src/lib/Bcfg2/Component.py | 38 +---- src/lib/Bcfg2/Options.py | 33 ++-- src/lib/Bcfg2/Proxy.py | 14 +- src/lib/Bcfg2/Server/Admin/__init__.py | 9 +- src/lib/Bcfg2/Server/BuiltinCore.py | 102 +++++++++++ src/lib/Bcfg2/Server/CherryPyCore.py | 131 ++++++++++++++ src/lib/Bcfg2/Server/Core.py | 285 +++++++++++++++++++------------ src/lib/Bcfg2/Server/Plugins/Metadata.py | 81 +++++---- src/sbin/bcfg2-info | 8 +- src/sbin/bcfg2-lint | 5 +- src/sbin/bcfg2-server | 52 +++--- src/sbin/bcfg2-test | 9 +- 12 files changed, 515 insertions(+), 252 deletions(-) create mode 100644 src/lib/Bcfg2/Server/BuiltinCore.py create mode 100644 src/lib/Bcfg2/Server/CherryPyCore.py (limited to 'src') diff --git a/src/lib/Bcfg2/Component.py b/src/lib/Bcfg2/Component.py index 3ee3a14c8..bb0e64102 100644 --- a/src/lib/Bcfg2/Component.py +++ b/src/lib/Bcfg2/Component.py @@ -85,23 +85,6 @@ def run_component(component_cls, listen_all, location, daemon, pidfile_name, server.server_close() component.shutdown() -def exposed(func): - """Mark a method to be exposed publically. - - Examples: - class MyComponent (Component): - @expose - def my_method (self, param1, param2): - do_stuff() - - class MyComponent (Component): - def my_method (self, param1, param2): - do_stuff() - my_method = expose(my_method) - - """ - func.exposed = True - return func def automatic(func, period=10): """Mark a method to be run periodically.""" @@ -153,6 +136,11 @@ class Component (object): self.lock = threading.Lock() self.instance_statistics = Statistics() + def critical_error(self, operation): + """Log and err, traceback and return an xmlrpc fault to client.""" + logger.error(operation, exc_info=1) + raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR, "Critical unexpected failure: %s" % (operation)) + def do_tasks(self): """Perform automatic tasks for the component. @@ -250,14 +238,7 @@ class Component (object): raise xmlrpclib.Fault(getattr(e, "fault_code", 1), str(e)) return result - def listMethods(self): - """Custom XML-RPC introspective method list.""" - return [ - name for name, func in inspect.getmembers(self, callable) - if getattr(func, "exposed", False) - ] - listMethods = exposed(listMethods) - + @exposed def methodHelp(self, method_name): """Custom XML-RPC introspective method help. @@ -270,19 +251,18 @@ class Component (object): except NoExposedMethod: return "" return pydoc.getdoc(func) - methodHelp = exposed(methodHelp) + @exposed def get_name(self): """The name of the component.""" return self.name - get_name = exposed(get_name) + @exposed def get_implementation(self): """The implementation of the component.""" return self.implementation - get_implementation = exposed(get_implementation) + @exposed def get_statistics(self, _): """Get current statistics about component execution""" return self.instance_statistics.display() - get_statistics = exposed(get_statistics) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 1fdfa4274..fe1bad110 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -252,14 +252,6 @@ HELP = \ Option('Print this usage message', default=False, cmd='-h') -DEBUG = \ - Option("Enable debugging output", - default=False, - cmd='-d') -VERBOSE = \ - Option("Enable verbose output", - default=False, - cmd='-v') DAEMON = \ Option("Daemonize process, storing pid", default=None, @@ -427,6 +419,10 @@ SERVER_PROTOCOL = \ Option('Server Protocol', default='xmlrpc/ssl', cf=('communication', 'procotol')) +SERVER_BACKEND = \ + Option('Server Backend', + default='best', + cf=('server', 'backend')) # Client options CLIENT_KEY = \ @@ -741,6 +737,18 @@ LOGGING_FILE_PATH = \ cmd='-o', odesc='', cf=('logging', 'path')) +DEBUG = \ + Option("Enable debugging output", + default=False, + cmd='-d', + cook=get_bool, + cf=('logging', 'debug')) +VERBOSE = \ + Option("Enable verbose output", + default=False, + cmd='-v', + cook=get_bool, + cf=('logging', 'verbose')) # Plugin-specific options CFG_VALIDATION = \ @@ -810,7 +818,8 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, key=SERVER_KEY, cert=SERVER_CERT, ca=SERVER_CA, - protocol=SERVER_PROTOCOL) + protocol=SERVER_PROTOCOL, + backend=SERVER_BACKEND) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, @@ -889,9 +898,11 @@ class OptionParser(OptionSet): OptionParser bootstraps option parsing, getting the value of the config file """ - def __init__(self, args): + def __init__(self, args, argv=None): + if argv is None: + argv = sys.argv[1:] self.Bootstrap = OptionSet([('configfile', CFILE)], quiet=True) - self.Bootstrap.parse(sys.argv[1:], do_getopt=False) + self.Bootstrap.parse(argv, do_getopt=False) OptionSet.__init__(self, args, configfile=self.Bootstrap['configfile']) self.optinfo = copy.copy(args) diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index eff9544da..93899c82b 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -1,13 +1,3 @@ -"""RPC client access to cobalt components. - -Classes: -ComponentProxy -- an RPC client proxy to Cobalt components - -Functions: -load_config -- read configuration files - -""" - import logging import re import socket @@ -34,7 +24,6 @@ import time from Bcfg2.Bcfg2Py3k import httplib, xmlrpclib, urlparse version = sys.version_info[:2] -has_py23 = version >= (2, 3) has_py26 = version >= (2, 6) __all__ = ["ComponentProxy", @@ -220,8 +209,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") self.key = None - if has_py23: - rawsock.settimeout(self.timeout) + rawsock.settimeout(self.timeout) self.sock = ssl.SSLSocket(rawsock, cert_reqs=other_side_required, ca_certs=self.ca, suppress_ragged_eofs=True, keyfile=self.key, certfile=self.cert, diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index a7269a289..89a857430 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -117,14 +117,7 @@ class MetadataCore(Mode): if p not in self.__plugin_blacklist__] try: - self.bcore = \ - Bcfg2.Server.Core.Core(setup['repo'], - setup['plugins'], - setup['password'], - setup['encoding'], - filemonitor=setup['filemonitor'], - cfile=setup['configfile'], - setup=setup) + self.bcore = Bcfg2.Server.Core.Core(setup) except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed: %s" % msg) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py new file mode 100644 index 000000000..c844e06d3 --- /dev/null +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -0,0 +1,102 @@ +""" the core of the builtin bcfg2 server """ + +import os +import sys +import time +import socket +import logging +from Bcfg2.Server.Core import BaseCore +from Bcfg2.Bcfg2Py3k import xmlrpclib, urlparse + +logger = logging.getLogger() + +class NoExposedMethod (Exception): + """There is no method exposed with the given name.""" + + +class Core(BaseCore): + name = 'bcfg2-server' + + def _resolve_exposed_method(self, method_name): + """Resolve an exposed method. + + Arguments: + method_name -- name of the method to resolve + + """ + try: + func = getattr(self, method_name) + except AttributeError: + raise NoExposedMethod(method_name) + if not getattr(func, "exposed", False): + raise NoExposedMethod(method_name) + return func + + def _dispatch(self, method, args, dispatch_dict): + """Custom XML-RPC dispatcher for components. + + method -- XML-RPC method name + args -- tuple of paramaters to method + + """ + if method in dispatch_dict: + method_func = dispatch_dict[method] + else: + try: + method_func = self._resolve_exposed_method(method) + except NoExposedMethod: + self.logger.error("Unknown method %s" % (method)) + raise xmlrpclib.Fault(xmlrpclib.METHOD_NOT_FOUND, + "Unknown method %s" % method) + + try: + method_start = time.time() + try: + result = method_func(*args) + finally: + method_done = time.time() + except xmlrpclib.Fault: + raise + except Exception: + e = sys.exc_info()[1] + if getattr(e, "log", True): + self.logger.error(e, exc_info=True) + raise xmlrpclib.Fault(getattr(e, "fault_code", 1), str(e)) + return result + + def run(self): + if self.setup['daemon']: + self._daemonize() + + hostname, port = urlparse(self.setup['location'])[1].split(':') + server_address = socket.getaddrinfo(hostname, + port, + socket.AF_UNSPEC, + socket.SOCK_STREAM)[0][4] + try: + server = XMLRPCServer(self.setup['listen_all'], + server_address, + keyfile=self.setup['key'], + certfile=self.setup['cert'], + register=False, + timeout=1, + ca=self.setup['ca'], + protocol=self.setup['protocol']) + except: + err = sys.exc_info()[1] + self.logger.error("Server startup failed") + os._exit(1) + server.register_instance(self) + + try: + server.serve_forever() + finally: + server.server_close() + self.shutdown() + + def methodHelp(self, method_name): + try: + func = self._resolve_exposed_method(method_name) + except NoExposedMethod: + return "" + return func.__doc__ diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py new file mode 100644 index 000000000..91e7f89bd --- /dev/null +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -0,0 +1,131 @@ +""" the core of the CherryPy-powered server """ + +import sys +import base64 +import atexit +import cherrypy +import Bcfg2.Options +from Bcfg2.Bcfg2Py3k import urlparse, xmlrpclib +from Bcfg2.Server.Core import BaseCore +from cherrypy.lib import xmlrpcutil +from cherrypy._cptools import ErrorTool + +if cherrypy.engine.state == 0: + cherrypy.engine.start(blocking=False) + atexit.register(cherrypy.engine.stop) + +# define our own error handler that handles xmlrpclib.Fault objects +# and so allows for the possibility of returning proper error +# codes. this obviates the need to use the builtin CherryPy xmlrpc +# tool +def on_error(*args, **kwargs): + err = sys.exc_info()[1] + if not isinstance(err, xmlrpclib.Fault): + err = xmlrpclib.Fault(xmlrpclib.INTERNAL_ERROR, str(err)) + xmlrpcutil._set_response(xmlrpclib.dumps(err)) +cherrypy.tools.xmlrpc_error = ErrorTool(on_error) + + +class Core(BaseCore): + _cp_config = {'tools.xmlrpc_error.on': True, + 'tools.bcfg2_authn.on': True} + + def __init__(self, *args, **kwargs): + BaseCore.__init__(self, *args, **kwargs) + + cherrypy.tools.bcfg2_authn = cherrypy.Tool('on_start_resource', + self.do_authn) + + self.rmi = self._get_rmi() + + def do_authn(self): + try: + header = cherrypy.request.headers['Authorization'] + except KeyError: + self.critical_error("No authentication data presented") + auth_type, auth_content = header.split() + try: + # py3k compatibility + auth_content = base64.standard_b64decode(auth_content) + except TypeError: + auth_content = \ + base64.standard_b64decode(bytes(auth_content.encode('ascii'))) + try: + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(bytes(":", encoding='utf-8')) + password = pw.decode('utf-8') + except ValueError: + username = auth_content + password = "" + + # FIXME: Get client cert + cert = None + address = (cherrypy.request.remote.ip, cherrypy.request.remote.name) + return self.authenticate(cert, username, password, address) + + @cherrypy.expose + def default(self, *vpath, **params): + # needed to make enough changes to the stock XMLRPCController + # to support plugin.__rmi__ and prepending client address that + # we just rewrote. it clearly wasn't written with inheritance + # in mind :( + rpcparams, rpcmethod = xmlrpcutil.process_body() + if "." not in rpcmethod: + address = (cherrypy.request.remote.ip, cherrypy.request.remote.name) + rpcparams = (address, ) + rpcparams + + handler = getattr(self, rpcmethod) + if not handler or not getattr(handler, "exposed", False): + raise Exception('method "%s" is not supported' % attr) + else: + try: + handler = self.rmi[rpcmethod] + except: + raise Exception('method "%s" is not supported' % rpcmethod) + + body = handler(*rpcparams, **params) + + xmlrpcutil.respond(body, 'utf-8', True) + return cherrypy.serving.response.body + + def run(self): + hostname, port = urlparse(self.setup['location'])[1].split(':') + if self.setup['listen_all']: + hostname = '0.0.0.0' + + config = {'engine.autoreload.on': False, + 'server.socket_port': int(port)} + if self.setup['cert'] and self.setup['key']: + config.update({'server.ssl_module': 'pyopenssl', + 'server.ssl_certificate': self.setup['cert'], + 'server.ssl_private_key': self.setup['key']}) + if self.setup['debug']: + config['log.screen'] = True + cherrypy.config.update(config) + cherrypy.quickstart(self, config={'/': self.setup}) + + +def parse_opts(argv=None): + if argv is None: + argv = sys.argv[1:] + optinfo = dict() + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.DAEMON_COMMON_OPTIONS) + setup = Bcfg2.Options.OptionParser(optinfo, argv=argv) + setup.parse(argv) + return setup + +def application(environ, start_response): + """ running behind Apache as a WSGI app is not currently + supported, but I'm keeping this code here because I hope for it to + be supported some day. we'll need to set up an AMQP task queue + and related magic for that to happen, though. """ + cherrypy.config.update({'environment': 'embedded'}) + setup = parse_opts(argv=['-C', environ['config']]) + root = Core(setup, start_fam_thread=True) + cherrypy.tree.mount(root) + return cherrypy.tree(environ, start_response) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 6dbab64bd..1ee01585c 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -6,38 +6,28 @@ import select import sys import threading import time +import inspect +import lxml.etree from traceback import format_exc - -try: - import lxml.etree -except ImportError: - print("Failed to import lxml dependency. Shutting down server.") - raise SystemExit(1) - -from Bcfg2.Component import Component, exposed -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError import Bcfg2.Server +import Bcfg2.Logger import Bcfg2.Server.FileMonitor import Bcfg2.Server.Plugins.Metadata -# Compatibility imports from Bcfg2.Bcfg2Py3k import xmlrpclib +from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError + if sys.hexversion >= 0x03000000: from functools import reduce -logger = logging.getLogger('Bcfg2.Server.Core') - - -def critical_error(operation): - """Log and err, traceback and return an xmlrpc fault to client.""" - logger.error(operation, exc_info=1) - raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR, "Critical unexpected failure: %s" % (operation)) - try: import psyco psyco.full() except: pass +def exposed(func): + func.exposed = True + return func def sort_xml(node, key=None): for child in node: @@ -55,24 +45,31 @@ class CoreInitError(Exception): pass -class Core(Component): +class BaseCore(object): """The Core object is the container for all Bcfg2 Server logic and modules. """ - name = 'bcfg2-server' - implementation = 'bcfg2-server' - def __init__(self, repo, plugins, password, encoding, - cfile='/etc/bcfg2.conf', ca=None, setup=None, - filemonitor='default', start_fam_thread=False): - Component.__init__(self) - self.datastore = repo + def __init__(self, setup, start_fam_thread=False): + self.datastore = setup['repo'] + + self.logger = logging.getLogger('bcfg2-server') + if 'debug' in setup and setup['debug']: + level = logging.DEBUG + else: + level = logging.INFO + self.logger.setLevel(level) + Bcfg2.Logger.setup_logging('bcfg2-server', + to_console=True, + to_syslog=True, + to_file=setup['logging'], + level=level) try: - fm = Bcfg2.Server.FileMonitor.available[filemonitor] + fm = Bcfg2.Server.FileMonitor.available[setup['filemonitor']] except KeyError: - logger.error("File monitor driver %s not available; " - "forcing to default" % filemonitor) + self.logger.error("File monitor driver %s not available; " + "forcing to default" % filemonitor) fm = Bcfg2.Server.FileMonitor.available['default'] famargs = dict(ignore=[], debug=False) if 'ignore' in setup: @@ -82,68 +79,69 @@ class Core(Component): try: self.fam = fm(**famargs) except IOError: - msg = "Failed to instantiate fam driver %s" % filemonitor - logger.error(msg, exc_info=1) + msg = "Failed to instantiate fam driver %s" % setup['filemonitor'] + self.logger.error(msg, exc_info=1) raise CoreInitError(msg) self.pubspace = {} - self.cfile = cfile + self.cfile = setup['configfile'] self.cron = {} self.plugins = {} self.plugin_blacklist = {} self.revision = '-1' - self.password = password - self.encoding = encoding + self.password = setup['password'] + self.encoding = setup['encoding'] self.setup = setup atexit.register(self.shutdown) # Create an event to signal worker threads to shutdown self.terminate = threading.Event() - if '' in plugins: - plugins.remove('') + if '' in setup['plugins']: + setup['plugins'].remove('') - for plugin in plugins: + for plugin in setup['plugins']: if not plugin in self.plugins: self.init_plugins(plugin) # Remove blacklisted plugins for p, bl in list(self.plugin_blacklist.items()): if len(bl) > 0: - logger.error("The following plugins conflict with %s;" - "Unloading %s" % (p, bl)) + self.logger.error("The following plugins conflict with %s;" + "Unloading %s" % (p, bl)) for plug in bl: del self.plugins[plug] # This section logs the experimental plugins expl = [plug for (name, plug) in list(self.plugins.items()) if plug.experimental] if expl: - logger.info("Loading experimental plugin(s): %s" % \ - (" ".join([x.name for x in expl]))) - logger.info("NOTE: Interfaces subject to change") + self.logger.info("Loading experimental plugin(s): %s" % + (" ".join([x.name for x in expl]))) + self.logger.info("NOTE: Interfaces subject to change") # This section logs the deprecated plugins depr = [plug for (name, plug) in list(self.plugins.items()) if plug.deprecated] if depr: - logger.info("Loading deprecated plugin(s): %s" % \ - (" ".join([x.name for x in depr]))) + self.logger.info("Loading deprecated plugin(s): %s" % + (" ".join([x.name for x in depr]))) mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) if len(mlist) == 1: self.metadata = mlist[0] else: - logger.error("No Metadata Plugin loaded; failed to instantiate Core") + self.logger.error("No Metadata Plugin loaded; " + "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) self.pull_sources = self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) - self.ca = ca - self.fam_thread = threading.Thread(target=self._file_monitor_thread) + self.ca = setup['ca'] + self.fam_thread = \ + threading.Thread(name="%sFAMThread" % setup['filemonitor'], + target=self._file_monitor_thread) + self.lock = threading.Lock() + if start_fam_thread: self.fam_thread.start() - self.monitor_cfile() - - def monitor_cfile(self): - if self.setup: self.fam.AddMonitor(self.cfile, self.setup) def plugins_by_type(self, base_cls): @@ -179,6 +177,7 @@ class Core(Component): def init_plugins(self, plugin): """Handling for the plugins.""" + self.logger.debug("Loading plugin %s" % plugin) try: mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % (plugin)).Server.Plugins, plugin) @@ -186,7 +185,7 @@ class Core(Component): try: mod = __import__(plugin) except: - logger.error("Failed to load plugin %s" % (plugin)) + self.logger.error("Failed to load plugin %s" % plugin) return plug = getattr(mod, plugin) # Blacklist conflicting plugins @@ -196,10 +195,11 @@ class Core(Component): try: self.plugins[plugin] = plug(self, self.datastore) except PluginInitError: - logger.error("Failed to instantiate plugin %s" % (plugin)) + self.logger.error("Failed to instantiate plugin %s" % plugin, + exc_info=1) except: - logger.error("Unexpected instantiation failure for plugin %s" % - (plugin), exc_info=1) + self.logger.error("Unexpected instantiation failure for plugin %s" % + plugin, exc_info=1) def shutdown(self): """Shutting down the plugins.""" @@ -216,12 +216,13 @@ class Core(Component): getattr(plugin, hook)(metadata) except AttributeError: err = sys.exc_info()[1] - logger.error("Unknown attribute: %s" % err) + self.logger.error("Unknown attribute: %s" % err) raise except: err = sys.exc_info()[1] - logger.error("%s: Error invoking hook %s: %s" % (plugin, hook, - err)) + self.logger.error("%s: Error invoking hook %s: %s" % (plugin, + hook, + err)) def validate_structures(self, metadata, data): """Checks the data structure.""" @@ -230,12 +231,12 @@ class Core(Component): plugin.validate_structures(metadata, data) except Bcfg2.Server.Plugin.ValidationError: err = sys.exc_info()[1] - logger.error("Plugin %s structure validation failed: %s" \ - % (plugin.name, err.message)) + self.logger.error("Plugin %s structure validation failed: %s" % + (plugin.name, err)) raise except: - logger.error("Plugin %s: unexpected structure validation failure" \ - % (plugin.name), exc_info=1) + self.logger.error("Plugin %s: unexpected structure validation " + "failure" % plugin.name, exc_info=1) def validate_goals(self, metadata, data): """Checks that the config matches the goals enforced by the plugins.""" @@ -244,23 +245,23 @@ class Core(Component): plugin.validate_goals(metadata, data) except Bcfg2.Server.Plugin.ValidationError: err = sys.exc_info()[1] - logger.error("Plugin %s goal validation failed: %s" \ - % (plugin.name, err.message)) + self.logger.error("Plugin %s goal validation failed: %s" % + (plugin.name, err.message)) raise except: - logger.error("Plugin %s: unexpected goal validation failure" \ - % (plugin.name), exc_info=1) + self.logger.error("Plugin %s: unexpected goal validation " + "failure" % plugin.name, exc_info=1) def GetStructures(self, metadata): """Get all structures for client specified by metadata.""" structures = reduce(lambda x, y: x + y, - [struct.BuildStructures(metadata) for struct \ - in self.structures], []) + [struct.BuildStructures(metadata) + for struct in self.structures], []) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: - logger.error("Client %s configuration missing bundles: %s" \ - % (metadata.hostname, ':'.join(missing))) + self.logger.error("Client %s configuration missing bundles: %s" % + (metadata.hostname, ':'.join(missing))) return structures def BindStructure(self, structure, metadata): @@ -275,14 +276,14 @@ class Core(Component): exc = sys.exc_info()[1] if 'failure' not in entry.attrib: entry.set('failure', 'bind error: %s' % format_exc()) - logger.error("Failed to bind entry %s:%s: %s" % - (entry.tag, entry.get('name'), exc)) + self.logger.error("Failed to bind entry %s:%s: %s" % + (entry.tag, entry.get('name'), exc)) except Exception: exc = sys.exc_info()[1] if 'failure' not in entry.attrib: entry.set('failure', 'bind error: %s' % format_exc()) - logger.error("Unexpected failure in BindStructure: %s %s" \ - % (entry.tag, entry.get('name')), exc_info=1) + self.logger.error("Unexpected failure in BindStructure: %s %s" % + (entry.tag, entry.get('name')), exc_info=1) def Bind(self, entry, metadata): """Bind an entry using the appropriate generator.""" @@ -298,11 +299,11 @@ class Core(Component): return ret except: entry.set('name', oldname) - logger.error("Failed binding entry %s:%s with altsrc %s" \ - % (entry.tag, entry.get('name'), - entry.get('altsrc'))) - logger.error("Falling back to %s:%s" % (entry.tag, - entry.get('name'))) + self.logger.error("Failed binding entry %s:%s with altsrc %s" % + (entry.tag, entry.get('name'), + entry.get('altsrc'))) + self.logger.error("Falling back to %s:%s" % (entry.tag, + entry.get('name'))) glist = [gen for gen in self.generators if entry.get('name') in gen.Entries.get(entry.tag, {})] @@ -311,8 +312,8 @@ class Core(Component): metadata) elif len(glist) > 1: generators = ", ".join([gen.name for gen in glist]) - logger.error("%s %s served by multiple generators: %s" % \ - (entry.tag, entry.get('name'), generators)) + self.logger.error("%s %s served by multiple generators: %s" % + (entry.tag, entry.get('name'), generators)) g2list = [gen for gen in self.generators if gen.HandlesEntry(entry, metadata)] if len(g2list) == 1: @@ -324,12 +325,13 @@ class Core(Component): def BuildConfiguration(self, client): """Build configuration for clients.""" start = time.time() - config = lxml.etree.Element("Configuration", version='2.0', \ + config = lxml.etree.Element("Configuration", version='2.0', revision=self.revision) try: meta = self.build_metadata(client) except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - logger.error("Metadata consistency error for client %s" % client) + self.logger.error("Metadata consistency error for client %s" % + client) return lxml.etree.Element("error", type='metadata error') self.client_run_hook("start_client_run", meta) @@ -337,7 +339,7 @@ class Core(Component): try: structures = self.GetStructures(meta) except: - logger.error("error in GetStructures", exc_info=1) + self.logger.error("error in GetStructures", exc_info=1) return lxml.etree.Element("error", type='structure error') self.validate_structures(meta, structures) @@ -349,7 +351,8 @@ class Core(Component): key = (entry.tag, entry.get('name')) if key in esrcs: if esrcs[key] != entry.get('altsrc'): - logger.error("Found inconsistent altsrc mapping for entry %s:%s" % key) + self.logger.error("Found inconsistent altsrc mapping " + "for entry %s:%s" % key) else: esrcs[key] = entry.get('altsrc', None) del esrcs @@ -359,17 +362,49 @@ class Core(Component): self.BindStructure(astruct, meta) config.append(astruct) except: - logger.error("error in BindStructure", exc_info=1) + self.logger.error("error in BindStructure", exc_info=1) self.validate_goals(meta, config) self.client_run_hook("end_client_run", meta) sort_xml(config, key=lambda e: e.get('name')) - logger.info("Generated config for %s in %.03f seconds" % \ - (client, time.time() - start)) + self.logger.info("Generated config for %s in %.03f seconds" % + (client, time.time() - start)) return config + def run(self, **kwargs): + """ run the server core """ + raise NotImplementedError + + def _daemonize(self): + child_pid = os.fork() + if child_pid != 0: + return + + os.setsid() + + child_pid = os.fork() + if child_pid != 0: + os._exit(0) + + redirect_file = open("/dev/null", "w+") + os.dup2(redirect_file.fileno(), sys.__stdin__.fileno()) + os.dup2(redirect_file.fileno(), sys.__stdout__.fileno()) + os.dup2(redirect_file.fileno(), sys.__stderr__.fileno()) + + os.chdir(os.sep) + + pidfile = open(self.setup['daemon'] or "/dev/null", "w") + pidfile.write("%s\n" % os.getpid()) + pidfile.close() + + return os.getpid() + + def critical_error(self, operation): + """ this should be overridden by child classes """ + self.logger.fatal(operation, exc_info=1) + def GetDecisions(self, metadata, mode): """Get data for the decision list.""" result = [] @@ -377,8 +412,8 @@ class Core(Component): try: result += plugin.GetDecisions(metadata, mode) except: - logger.error("Plugin: %s failed to generate decision list" \ - % plugin.name, exc_info=1) + self.logger.error("Plugin: %s failed to generate decision list" + % plugin.name, exc_info=1) return result def build_metadata(self, client_name): @@ -405,12 +440,12 @@ class Core(Component): try: plugin.process_statistics(meta, statistics) except: - logger.error("Plugin %s failed to process stats from %s" \ - % (plugin.name, meta.hostname), - exc_info=1) + self.logger.error("Plugin %s failed to process stats from " + "%s" % (plugin.name, meta.hostname), + exc_info=1) - logger.info("Client %s reported state %s" % (client_name, - state.get('state'))) + self.logger.info("Client %s reported state %s" % (client_name, + state.get('state'))) self.client_run_hook("end_statistics", meta) def resolve_client(self, address, cleanup_cache=False, metadata=True): @@ -422,13 +457,41 @@ class Core(Component): else: meta = None except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - critical_error("Client metadata resolution error for %s; " - "check server log" % address[0]) + err = sys.exc_info()[1] + self.critical_error("Client metadata resolution error for %s: %s" % + (address[0], err)) except Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError: - critical_error('Metadata system runtime failure') + err = sys.exc_info()[1] + self.critical_error('Metadata system runtime failure for %s: %s' % + (address[0], err)) return (client, meta) + def critical_error(self, operation): + """Log and err, traceback and return an xmlrpc fault to client.""" + self.logger.error(operation, exc_info=1) + raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR, + "Critical failure: %s" % operation) + + def _get_rmi(self): + rmi = dict() + if self.plugins: + for pname, pinst in list(self.plugins.items()): + for mname in pinst.__rmi__: + rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + return rmi + # XMLRPC handlers start here + @exposed + def listMethods(self, address): + methods = [name + for name, func in inspect.getmembers(self, callable) + if getattr(func, "exposed", False)] + methods.extend(self._get_rmi().keys()) + return methods + + @exposed + def methodHelp(self, address, method_name): + raise NotImplementedError @exposed def DeclareVersion(self, address, version): @@ -439,8 +502,8 @@ class Core(Component): except (Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError, Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError): err = sys.exc_info()[1] - critical_error("Unable to set version for %s: %s" % - (client, err)) + self.critical_error("Unable to set version for %s: %s" % + (client, err)) return True @exposed @@ -455,7 +518,9 @@ class Core(Component): return lxml.etree.tostring(resp, encoding='UTF-8', xml_declaration=True) except: - critical_error("Error determining probes for %s" % client) + err = sys.exc_info()[1] + self.critical_error("Error determining probes for %s" % + (client, err)) @exposed def RecvProbeData(self, address, probedata): @@ -467,8 +532,9 @@ class Core(Component): xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) except: - critical_error("Failed to parse probe data from client %s" % - client) + err = sys.exc_info()[1] + self.critical_error("Failed to parse probe data from client %s: %s" + % (client, err)) sources = [] [sources.append(data.get('source')) for data in xpdata @@ -481,8 +547,10 @@ class Core(Component): try: self.plugins[source].ReceiveData(metadata, dl) except: - critical_error("Failed to process probe data from client %s" % - client) + err = sys.exc_info()[1] + self.critical_error("Failed to process probe data from client " + "%s: %s" % + (client, err)) return True @exposed @@ -494,7 +562,7 @@ class Core(Component): except (Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError, Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError): err = sys.exc_info()[1] - critical_error("Unable to assert profile for %s: %s" % + self.critical_error("Unable to assert profile for %s: %s" % (client, err)) return True @@ -507,7 +575,7 @@ class Core(Component): return lxml.etree.tostring(config, encoding='UTF-8', xml_declaration=True) except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - critical_error("Metadata consistency failure for %s" % client) + self.critical_error("Metadata consistency failure for %s" % client) @exposed def RecvStats(self, address, stats): @@ -524,7 +592,8 @@ class Core(Component): else: # No ca, so no cert validation can be done acert = None - return self.metadata.AuthenticateConnection(acert, user, password, address) + return self.metadata.AuthenticateConnection(acert, user, password, + address) @exposed def GetDecisionList(self, address, mode): diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 2abc96cc3..ceeeb074c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -59,13 +59,13 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): @property def xdata(self): if not self.data: - raise MetadataRuntimeError + raise MetadataRuntimeError("%s has no data" % self.basefile) return self.data @property def base_xdata(self): if not self.basedata: - raise MetadataRuntimeError + raise MetadataRuntimeError("%s has no data" % self.basefile) return self.basedata def load_xml(self): @@ -98,9 +98,9 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): try: datafile = open(tmpfile, 'w') except IOError: - e = sys.exc_info()[1] - self.logger.error("Failed to write %s: %s" % (tmpfile, e)) - raise MetadataRuntimeError + msg = "Failed to write %s: %s" % (tmpfile, sys.exc_info()[1]) + self.logger.error(msg) + raise MetadataRuntimeError(msg) # prep data dataroot = xmltree.getroot() newcontents = lxml.etree.tostring(dataroot, pretty_print=True) @@ -112,21 +112,24 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): datafile.write(newcontents) except: fcntl.lockf(fd, fcntl.LOCK_UN) - self.logger.error("Metadata: Failed to write new xml data to %s" % - tmpfile, exc_info=1) + msg = "Metadata: Failed to write new xml data to %s: %s" % \ + (tmpfile, sys.exc_info()[1]) + self.logger.error(msg, exc_info=1) os.unlink(tmpfile) - raise MetadataRuntimeError + raise MetadataRuntimeError(msg) datafile.close() - # check if clients.xml is a symlink if os.path.islink(fname): fname = os.readlink(fname) try: os.rename(tmpfile, fname) + except: - self.logger.error("Metadata: Failed to rename %s" % tmpfile) - raise MetadataRuntimeError + msg = "Metadata: Failed to rename %s: %s" % (tmpfile, + sys.exc_info()[1]) + self.logger.error(msg) + raise MetadataRuntimeError(msg) def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" @@ -317,8 +320,9 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def _add_xdata(self, config, tag, name, attribs=None, alias=False): node = self._search_xdata(tag, name, config.xdata, alias=alias) if node != None: - self.logger.error("%s \"%s\" already exists" % (tag, name)) - raise MetadataConsistencyError + msg = "%s \"%s\" already exists" % (tag, name) + self.logger.error(msg) + raise MetadataConsistencyError(msg) element = lxml.etree.SubElement(config.base_xdata.getroot(), tag, name=name) if attribs: @@ -343,14 +347,15 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def _update_xdata(self, config, tag, name, attribs, alias=False): node = self._search_xdata(tag, name, config.xdata, alias=alias) if node == None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise MetadataConsistencyError + msg = "Unexpected error finding %s \"%s\"" % (tag, name) + self.logger.error(msg) + raise MetadataConsistencyError(msg) for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) @@ -367,14 +372,15 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def _remove_xdata(self, config, tag, name, alias=False): node = self._search_xdata(tag, name, config.xdata) if node == None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise MetadataConsistencyError + msg = "Unexpected error finding %s \"%s\"" % (tag, name) + self.logger.error(msg) + raise MetadataConsistencyError(msg) xdict['xquery'][0].getparent().remove(xdict['xquery'][0]) self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) @@ -521,13 +527,15 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def set_profile(self, client, profile, addresspair): """Set group parameter for provided client.""" - self.logger.info("Asserting client %s profile to %s" % (client, profile)) + self.logger.info("Asserting client %s profile to %s" % (client, + profile)) if False in list(self.states.values()): - raise MetadataRuntimeError + raise MetadataRuntimeError("Metadata has not been read yet") if profile not in self.public: - self.logger.error("Failed to set client %s to private group %s" % - (client, profile)) - raise MetadataConsistencyError + msg = "Failed to set client %s to private group %s" % (client, + profile) + self.logger.error(msg) + raise MetadataConsistencyError(msg) if client in self.clients: self.logger.info("Changing %s group from %s to %s" % (client, self.clients[client], profile)) @@ -547,7 +555,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def set_version(self, client, version): """Set group parameter for provided client.""" - self.logger.info("Asserting client %s version to %s" % (client, version)) + self.logger.info("Setting client %s version to %s" % (client, version)) if client in self.clients: self.logger.info("Setting version on client %s to %s" % (client, version)) @@ -587,9 +595,9 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, address = addresspair[0] if address in self.addresses: if len(self.addresses[address]) != 1: - self.logger.error("Address %s has multiple reverse assignments; " - "a uuid must be used" % (address)) - raise MetadataConsistencyError + err = "Address %s has multiple reverse assignments; a uuid must be used" % address + self.logger.error(err) + raise MetadataConsistencyError(err) return self.addresses[address][0] try: cname = socket.gethostbyaddr(address)[0].lower() @@ -604,7 +612,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def get_initial_metadata(self, client): """Return the metadata for a given client.""" if False in list(self.states.values()): - raise MetadataRuntimeError + raise MetadataRuntimeError("Metadata has not been read yet") client = client.lower() if client in self.aliases: client = self.aliases[client] @@ -613,9 +621,10 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (bundles, groups, categories) = self.groups[profile] else: if self.default == None: - self.logger.error("Cannot set group for client %s; " - "no default group set" % client) - raise MetadataConsistencyError + msg = "Cannot set group for client %s; no default group set" % \ + client + self.logger.error(msg) + raise MetadataConsistencyError(msg) self.set_profile(client, self.default, (None, None)) profile = self.default [bundles, groups, categories] = self.groups[self.default] diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 297b2227d..55650f18b 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -169,20 +169,18 @@ def load_interpreters(): return interpreters -class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): +class infoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): """Main class for bcfg2-info.""" def __init__(self, repo, plgs, passwd, encoding, event_debug, filemonitor='default', setup=None): cmd.Cmd.__init__(self) try: - Bcfg2.Server.Core.Core.__init__(self, repo, plgs, passwd, - encoding, filemonitor=filemonitor, - setup=setup) + Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) if event_debug: self.fam.debug = True except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] - print("Core load failed because %s" % msg) + print("Core load failed: %s" % msg) raise SystemExit(1) self.prompt = '> ' self.cont = True diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index c664a67f2..09b2f715d 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -61,10 +61,7 @@ def get_errorhandler(config): def load_server(setup): """ load server """ - core = Bcfg2.Server.Core.Core(setup['repo'], setup['plugins'], - setup['password'], setup['encoding'], - filemonitor=setup['filemonitor'], - setup=setup) + core = Bcfg2.Server.Core.Core(setup) core.fam.handle_events_in_interval(4) return core diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index d03edc93e..2727dfe3a 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -2,13 +2,11 @@ """The XML-RPC Bcfg2 server.""" -import logging -import os.path +import os import sys - +import logging import Bcfg2.Logger import Bcfg2.Options -import Bcfg2.Component import Bcfg2.Server.Plugins.Metadata from Bcfg2.Server.Core import CoreInitError @@ -21,36 +19,30 @@ if __name__ == '__main__': optinfo.update(Bcfg2.Options.DAEMON_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) + # check whether the specified bcfg2.conf exists + if not os.path.exists(setup['configfile']): + print("Could not read %s" % setup['configfile']) + sys.exit(1) + + if setup['backend'] not in ['best', 'cherrypy', 'builtin']: + print("Unknown server backend %s, using 'best'" % setup['backend']) + setup['backend'] = 'best' + if setup['backend'] == 'cherrypy': + try: + from Bcfg2.Server.CherryPyCore import Core + except ImportError: + err = sys.exc_info()[1] + print("Unable to import CherryPy server core: %s" % err) + raise + elif setup['backend'] == 'builtin' or setup['backend'] == 'best': + from Bcfg2.Server.BuiltinCore import Core + try: - # check whether the specified bcfg2.conf exists - if not os.path.exists(setup['configfile']): - print("Could not read %s" % setup['configfile']) - sys.exit(1) - Bcfg2.Component.run_component(Bcfg2.Server.Core.Core, - listen_all=setup['listen_all'], - location=setup['location'], - daemon=setup['daemon'], - pidfile_name=setup['daemon'], - protocol=setup['protocol'], - to_file=setup['logging'], - cfile=setup['configfile'], - register=False, - cls_kwargs={'repo':setup['repo'], - 'plugins':setup['plugins'], - 'password':setup['password'], - 'encoding':setup['encoding'], - 'ca':setup['ca'], - 'filemonitor':setup['filemonitor'], - 'start_fam_thread':True, - 'setup':setup}, - keyfile=setup['key'], - certfile=setup['cert'], - ca=setup['ca'] - ) + core = Core(setup, start_fam_thread=True) + core.run() except CoreInitError: msg = sys.exc_info()[1] logger.error(msg) - logger.error("exiting") sys.exit(1) except KeyboardInterrupt: sys.exit(1) diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 653c24124..73800b5e3 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -75,14 +75,7 @@ def main(): if setup['verbose']: Bcfg2.Logger.setup_logging("bcfg2-test", to_syslog=False) - core = Bcfg2.Server.Core.Core( - setup['repo'], - setup['plugins'], - setup['password'], - setup['encoding'], - filemonitor=setup['filemonitor'], - setup=setup - ) + core = Bcfg2.Server.Core.Core(setup) ignore = dict() for entry in setup['test_ignore']: -- cgit v1.2.3-1-g7c22