diff options
Diffstat (limited to 'src/lib/Bcfg2/Server')
20 files changed, 869 insertions, 126 deletions
diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py index 7448855ce..a7e67c956 100644 --- a/src/lib/Bcfg2/Server/Admin/Perf.py +++ b/src/lib/Bcfg2/Server/Admin/Perf.py @@ -2,7 +2,7 @@ import sys import Bcfg2.Options -import Bcfg2.Proxy +import Bcfg2.Client.Proxy import Bcfg2.Server.Admin @@ -22,13 +22,13 @@ class Perf(Bcfg2.Server.Admin.Mode): opts = sys.argv[1:] opts.remove(self.__class__.__name__.lower()) setup.reparse(argv=opts) - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) + proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], + setup['user'], + setup['password'], + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca'], + timeout=setup['timeout']) data = proxy.get_statistics() for key in sorted(data.keys()): output.append((key, ) + diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index 7f9f32816..13077c7ad 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -2,7 +2,7 @@ import sys import Bcfg2.Options -import Bcfg2.Proxy +import Bcfg2.Client.Proxy import Bcfg2.Server.Admin from Bcfg2.Compat import xmlrpclib @@ -23,14 +23,14 @@ class Xcmd(Bcfg2.Server.Admin.Mode): opts = sys.argv[1:] opts.remove(self.__class__.__name__.lower()) setup.reparse(argv=opts) - Bcfg2.Proxy.RetryMethod.max_retries = 1 - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) + Bcfg2.Client.Proxy.RetryMethod.max_retries = 1 + proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], + setup['user'], + setup['password'], + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca'], + timeout=setup['timeout']) if len(setup['args']) == 0: print("Usage: xcmd <xmlrpc method> <optional arguments>") return @@ -46,7 +46,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): return else: raise - except Bcfg2.Proxy.ProxyError: + except Bcfg2.Client.Proxy.ProxyError: err = sys.exc_info()[1] print("Proxy Error: %s" % err) return diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 14b64ff40..663ee6f92 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -4,10 +4,10 @@ import sys import time import socket import daemon -import Bcfg2.Statistics +import Bcfg2.Server.Statistics from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse -from Bcfg2.SSLServer import XMLRPCServer +from Bcfg2.Server.SSLServer import XMLRPCServer from lockfile import LockFailed # pylint: disable=E0611 @@ -25,8 +25,8 @@ class Core(BaseCore): def __init__(self): BaseCore.__init__(self) - #: The :class:`Bcfg2.SSLServer.XMLRPCServer` instance powering - #: this server core + #: The :class:`Bcfg2.Server.SSLServer.XMLRPCServer` instance + #: powering this server core self.server = None daemon_args = dict(uid=self.setup['daemon_uid'], @@ -68,8 +68,9 @@ class Core(BaseCore): try: return method_func(*args) finally: - Bcfg2.Statistics.stats.add_value(method, - time.time() - method_start) + Bcfg2.Server.Statistics.stats.add_value( + method, + time.time() - method_start) except xmlrpclib.Fault: raise except Exception: diff --git a/src/lib/Bcfg2/Server/Cache.py b/src/lib/Bcfg2/Server/Cache.py new file mode 100644 index 000000000..842098eda --- /dev/null +++ b/src/lib/Bcfg2/Server/Cache.py @@ -0,0 +1,14 @@ +""" An implementation of a simple memory-backed cache. Right now this +doesn't provide many features, but more (time-based expiration, etc.) +can be added as necessary. """ + + +class Cache(dict): + """ an implementation of a simple memory-backed cache """ + + def expire(self, key=None): + """ expire all items, or a specific item, from the cache """ + if key is None: + self.clear() + elif key in self: + del self[key] diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py index 79768df20..936279508 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -3,7 +3,7 @@ server. """ import sys import time -import Bcfg2.Statistics +import Bcfg2.Server.Statistics from Bcfg2.Compat import urlparse, xmlrpclib, b64decode from Bcfg2.Server.Core import BaseCore import cherrypy @@ -96,8 +96,8 @@ class Core(BaseCore): try: body = handler(*rpcparams, **params) finally: - Bcfg2.Statistics.stats.add_value(rpcmethod, - time.time() - method_start) + Bcfg2.Server.Statistics.stats.add_value(rpcmethod, + time.time() - method_start) xmlrpcutil.respond(body, 'utf-8', True) return cherrypy.serving.response.body diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index c7797a711..a2c4ee1cb 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -13,14 +13,13 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Logger import Bcfg2.settings -import Bcfg2.Statistics +import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain -from Bcfg2.Cache import Cache +from Bcfg2.Server.Cache import Cache from Bcfg2.Options import get_option_parser, SERVER_FAM_IGNORE from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError, \ - track_statistics +from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError try: import psyco @@ -292,7 +291,7 @@ class BaseCore(object): #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() - #: A :class:`Bcfg2.Cache.Cache` object for caching client + #: A :class:`Bcfg2.Server.Cache.Cache` object for caching client #: metadata self.metadata_cache = Cache() @@ -429,11 +428,11 @@ class BaseCore(object): self.logger.error("%s: Error invoking hook %s: %s" % (plugin, hook, err)) finally: - Bcfg2.Statistics.stats.add_value("%s:client_run_hook:%s" % + Bcfg2.Server.Statistics.stats.add_value("%s:client_run_hook:%s" % (self.__class__.__name__, hook), time.time() - start) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def validate_structures(self, metadata, data): """ Checks the data structures by calling the :func:`Bcfg2.Server.Plugin.interfaces.StructureValidator.validate_structures` @@ -460,7 +459,7 @@ class BaseCore(object): self.logger.error("Plugin %s: unexpected structure validation " "failure" % plugin.name, exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def validate_goals(self, metadata, data): """ Checks that the config matches the goals enforced by :class:`Bcfg2.Server.Plugin.interfaces.GoalValidator` plugins @@ -485,7 +484,7 @@ class BaseCore(object): self.logger.error("Plugin %s: unexpected goal validation " "failure" % plugin.name, exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def GetStructures(self, metadata): """ Get all structures (i.e., bundles) for the given client @@ -502,7 +501,7 @@ class BaseCore(object): (metadata.hostname, ':'.join(missing))) return structures - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def BindStructures(self, structures, metadata, config): """ Given a list of structures (i.e. bundles), bind all the entries in them and add the structures to the config. @@ -522,7 +521,7 @@ class BaseCore(object): except: self.logger.error("error in BindStructure", exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def BindStructure(self, structure, metadata): """ Bind all elements in a single structure (i.e., bundle). @@ -595,7 +594,7 @@ class BaseCore(object): raise PluginExecutionError("No matching generator: %s:%s" % (entry.tag, entry.get('name'))) finally: - Bcfg2.Statistics.stats.add_value("%s:Bind:%s" % + Bcfg2.Server.Statistics.stats.add_value("%s:Bind:%s" % (self.__class__.__name__, entry.tag), time.time() - start) @@ -744,7 +743,7 @@ class BaseCore(object): % plugin.name, exc_info=1) return result - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def build_metadata(self, client_name): """ Build initial client metadata for a client @@ -1094,11 +1093,11 @@ class BaseCore(object): @exposed def get_statistics(self, _): """ Get current statistics about component execution from - :attr:`Bcfg2.Statistics.stats`. + :attr:`Bcfg2.Server.Statistics.stats`. :returns: dict - The statistics data as returned by - :func:`Bcfg2.Statistics.Statistics.display` """ - return Bcfg2.Statistics.stats.display() + :func:`Bcfg2.Server.Statistics.Statistics.display` """ + return Bcfg2.Server.Statistics.stats.display() @exposed def toggle_debug(self, address): diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py new file mode 100755 index 000000000..c931ed2a7 --- /dev/null +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -0,0 +1,195 @@ +""" Bcfg2.Server.Encryption provides a number of convenience methods +for handling encryption in Bcfg2. See :ref:`server-encryption` for +more details. """ + +import os +import Bcfg2.Options +from M2Crypto import Rand +from M2Crypto.EVP import Cipher, EVPError +from Bcfg2.Compat import StringIO, md5, b64encode, b64decode + +#: Constant representing the encryption operation for +#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This +#: makes our code more readable. +ENCRYPT = 1 + +#: Constant representing the decryption operation for +#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This +#: makes our code more readable. +DECRYPT = 0 + +#: Default initialization vector. For best security, you should use a +#: unique IV for each message. :func:`ssl_encrypt` does this in an +#: automated fashion. +IV = '\0' * 16 + +#: The config file section encryption options and passphrases are +#: stored in +CFG_SECTION = "encryption" + +#: The config option used to store the algorithm +CFG_ALGORITHM = "algorithm" + +#: Default cipher algorithm. To get a full list of valid algorithms, +#: you can run:: +#: +#: openssl list-cipher-algorithms | grep -v ' => ' | \ +#: tr 'A-Z-' 'a-z_' | sort -u +ALGORITHM = Bcfg2.Options.get_option_parser().cfp.get( # pylint: disable=E1103 + CFG_SECTION, + CFG_ALGORITHM, + default="aes_256_cbc").lower().replace("-", "_") + +Rand.rand_seed(os.urandom(1024)) + + +def _cipher_filter(cipher, instr): + """ M2Crypto reads and writes file-like objects, so this uses + StringIO to pass data through it """ + inbuf = StringIO(instr) + outbuf = StringIO() + while 1: + buf = inbuf.read() + if not buf: + break + outbuf.write(cipher.update(buf)) + outbuf.write(cipher.final()) + rv = outbuf.getvalue() + inbuf.close() + outbuf.close() + return rv + + +def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): + """ Encrypt a string with a key. For a higher-level encryption + interface, see :func:`ssl_encrypt`. + + :param plaintext: The plaintext data to encrypt + :type plaintext: string + :param key: The key to encrypt the data with + :type key: string + :param iv: The initialization vector + :type iv: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :param salt: The salt to use + :type salt: string + :returns: string - The decrypted data + """ + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) + return _cipher_filter(cipher, plaintext) + + +def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): + """ Decrypt a string with a key. For a higher-level decryption + interface, see :func:`ssl_decrypt`. + + :param crypted: The raw binary encrypted data + :type crypted: string + :param key: The encryption key to decrypt with + :type key: string + :param iv: The initialization vector + :type iv: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + """ + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) + return _cipher_filter(cipher, crypted) + + +def ssl_decrypt(data, passwd, algorithm=ALGORITHM): + """ Decrypt openssl-encrypted data. This can decrypt data + encrypted by :func:`ssl_encrypt`, or ``openssl enc``. It performs + a base64 decode first if the data is base64 encoded, and + automatically determines the salt and initialization vector (both + of which are embedded in the encrypted data). + + :param data: The encrypted data (either base64-encoded or raw + binary) to decrypt + :type data: string + :param passwd: The password to use to decrypt the data + :type passwd: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + """ + # base64-decode the data + data = b64decode(data) + salt = data[8:16] + hashes = [md5(passwd + salt).digest()] + for i in range(1, 3): + hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + key = hashes[0] + hashes[1] + iv = hashes[2] + + return str_decrypt(data[16:], key=key, iv=iv, algorithm=algorithm) + + +def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): + """ Encrypt data in a format that is openssl compatible. + + :param plaintext: The plaintext data to encrypt + :type plaintext: string + :param passwd: The password to use to encrypt the data + :type passwd: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :param salt: The salt to use. If none is provided, one will be + randomly generated. + :type salt: bytes + :returns: string - The base64-encoded, salted, encrypted string. + The string includes a trailing newline to make it fully + compatible with openssl command-line tools. + """ + if salt is None: + salt = Rand.rand_bytes(8) + + hashes = [md5(passwd + salt).digest()] + for i in range(1, 3): + hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + key = hashes[0] + hashes[1] + iv = hashes[2] + + crypted = str_encrypt(plaintext, key=key, salt=salt, iv=iv, + algorithm=algorithm) + return b64encode("Salted__" + salt + crypted) + "\n" + + +def get_passphrases(): + """ Get all candidate encryption passphrases from the config file. + + :returns: dict - a dict of ``<passphrase name>``: ``<passphrase>`` + """ + setup = Bcfg2.Options.get_option_parser() + if setup.cfp.has_section(CFG_SECTION): + return dict([(o, setup.cfp.get(CFG_SECTION, o)) + for o in setup.cfp.options(CFG_SECTION) + if o != CFG_ALGORITHM]) + else: + return dict() + + +def bruteforce_decrypt(crypted, passphrases=None, algorithm=ALGORITHM): + """ Convenience method to decrypt the given encrypted string by + trying the given passphrases or all passphrases (as returned by + :func:`get_passphrases`) sequentially until one is found that + works. + + :param crypted: The data to decrypt + :type crypted: string + :param passphrases: The passphrases to try. + :type passphrases: list + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + :raises: :class:`M2Crypto.EVP.EVPError`, if the data cannot be decrypted + """ + if passphrases is None: + passphrases = get_passphrases().values() + for passwd in passphrases: + try: + return ssl_decrypt(crypted, passwd, algorithm=algorithm) + except EVPError: + pass + raise EVPError("Failed to decrypt") diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 871b5eb4e..bc82ee6c2 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -4,14 +4,12 @@ import os import re import sys import copy -import time import genshi import logging import operator import lxml.etree import Bcfg2.Server import Bcfg2.Options -import Bcfg2.Statistics import Bcfg2.Server.FileMonitor from Bcfg2.Compat import CmpMixin, wraps from Bcfg2.Server.Plugin.base import Debuggable, Plugin @@ -20,7 +18,7 @@ from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ PluginExecutionError try: - import Bcfg2.Encryption + import Bcfg2.Server.Encryption HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -94,40 +92,6 @@ def bind_info(entry, metadata, infoxml=None, default=None): entry.set(attr, val) -class track_statistics(object): # pylint: disable=C0103 - """ Decorator that tracks execution time for the given - :class:`Plugin` method with :mod:`Bcfg2.Statistics` for reporting - via ``bcfg2-admin perf`` """ - - def __init__(self, name=None): - """ - :param name: The name under which statistics for this function - will be tracked. By default, the name will be - the name of the function concatenated with the - name of the class the function is a member of. - :type name: string - """ - # if this is None, it will be set later during __call_ - self.name = name - - def __call__(self, func): - if self.name is None: - self.name = func.__name__ - - @wraps(func) - def inner(obj, *args, **kwargs): - """ The decorated function """ - name = "%s:%s" % (obj.__class__.__name__, self.name) - - start = time.time() - try: - return func(obj, *args, **kwargs) - finally: - Bcfg2.Statistics.stats.add_value(name, time.time() - start) - - return inner - - class DatabaseBacked(Plugin): """ Provides capabilities for a plugin to read and write to a database. @@ -620,8 +584,8 @@ class StructFile(XMLFileBacked, Debuggable): if self.encryption and HAS_CRYPTO: strict = self.xdata.get( "decrypt", - self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, "decrypt", - default="strict")) == "strict" + self.setup.cfp.get(Bcfg2.Server.Encryption.CFG_SECTION, + "decrypt", default="strict")) == "strict" for el in self.xdata.xpath("//*[@encrypted]"): try: el.text = self._decrypt(el).encode('ascii', @@ -629,7 +593,7 @@ class StructFile(XMLFileBacked, Debuggable): except UnicodeDecodeError: self.logger.info("%s: Decrypted %s to gibberish, skipping" % (self.name, el.tag)) - except Bcfg2.Encryption.EVPError: + except Bcfg2.Server.Encryption.EVPError: msg = "Failed to decrypt %s element in %s" % (el.tag, self.name) if strict: @@ -642,19 +606,20 @@ class StructFile(XMLFileBacked, Debuggable): """ Decrypt a single encrypted properties file element """ if not element.text or not element.text.strip(): return - passes = Bcfg2.Encryption.get_passphrases() + passes = Bcfg2.Server.Encryption.get_passphrases() try: passphrase = passes[element.get("encrypted")] try: - return Bcfg2.Encryption.ssl_decrypt(element.text, passphrase) - except Bcfg2.Encryption.EVPError: + return Bcfg2.Server.Encryption.ssl_decrypt(element.text, + passphrase) + except Bcfg2.Server.Encryption.EVPError: # error is raised below pass except KeyError: # bruteforce_decrypt raises an EVPError with a sensible # error message, so we just let it propagate up the stack - return Bcfg2.Encryption.bruteforce_decrypt(element.text) - raise Bcfg2.Encryption.EVPError("Failed to decrypt") + return Bcfg2.Server.Encryption.bruteforce_decrypt(element.text) + raise Bcfg2.Server.Encryption.EVPError("Failed to decrypt") def _include_element(self, item, metadata): """ determine if an XML element matches the metadata """ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 3b3b95ff5..516eba2f6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -4,7 +4,7 @@ from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt, EVPError + from Bcfg2.Server.Encryption import bruteforce_decrypt, EVPError HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 215e4c1f1..a285eecd8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -6,7 +6,7 @@ from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt + from Bcfg2.Server.Encryption import bruteforce_decrypt HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -21,7 +21,7 @@ except ImportError: class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt the data on the fly as it's read in using - :func:`Bcfg2.Encryption.bruteforce_decrypt` """ + :func:`Bcfg2.Server.Encryption.bruteforce_decrypt` """ def _instantiate(self, cls, fileobj, filepath, filename, encoding=None): plaintext = StringIO(bruteforce_decrypt(fileobj.read())) return TemplateLoader._instantiate(self, cls, plaintext, filepath, diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 4d6639e4d..1711cc655 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -9,7 +9,7 @@ from Bcfg2.Server.Plugin import StructFile from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator try: - import Bcfg2.Encryption + from Bcfg2.Server.Encryption import get_passphrases, ssl_encrypt HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -50,9 +50,8 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if (HAS_CRYPTO and self.setup.cfp.has_section("sshkeys") and self.setup.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Encryption.get_passphrases()[self.setup.cfp.get( - "sshkeys", - "passphrase")] + return get_passphrases()[self.setup.cfp.get("sshkeys", + "passphrase")] return None def handle_event(self, event): @@ -198,8 +197,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): privkey = open(filename).read() if HAS_CRYPTO and self.passphrase: self.debug_log("Cfg: Encrypting key data at %s" % filename) - privkey = Bcfg2.Encryption.ssl_encrypt(privkey, - self.passphrase) + privkey = ssl_encrypt(privkey, self.passphrase) specificity['ext'] = '.crypt' self.write_data(privkey, **specificity) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index cf4234ed0..3ad64b242 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -81,6 +81,7 @@ import Bcfg2.Server.Plugin from Bcfg2.Server.FileMonitor import get_fam from Bcfg2.Options import get_option_parser from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Server.Statistics import track_statistics LOGGER = logging.getLogger(__name__) @@ -213,7 +214,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): cachefiles.add(source.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ Given a list of package group names, return a dict of ``<group name>: <list of packages>``. This method is provided @@ -234,7 +235,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): rv[group] = self.get_group(group, ptype) return rv - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_group(self, group, ptype=None): """ Get the list of packages of the given type in a package group. @@ -469,7 +470,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ return list(complete.difference(initial)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 876ee6090..782e077bb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -5,6 +5,7 @@ import os import sys import Bcfg2.Server.Plugin from Bcfg2.Options import get_option_parser +from Bcfg2.Server.Statistics import track_statistics from Bcfg2.Server.Plugins.Packages.Source import SourceInitError @@ -109,7 +110,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, load its data. """ return sorted(list(self.parsed)) == sorted(self.extras) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] @@ -122,7 +123,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, ``Index`` is responsible for calling :func:`source_from_xml` for each ``Source`` tag in each file. """ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def source_from_xml(self, xsource): """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 7a8b2827a..5cf90e188 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -54,6 +54,7 @@ from Bcfg2.Options import get_option_parser from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ urlopen, cPickle, md5 +from Bcfg2.Server.Statistics import track_statistics def fetch_url(url): @@ -320,7 +321,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.essentialpkgs), cache, 2) cache.close() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def setup_data(self, force_update=False): """ Perform all data fetching and setup tasks. For most backends, this involves downloading all metadata from the diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index c5d59eb24..4057ed230 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -69,6 +69,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url +from Bcfg2.Server.Statistics import track_statistics LOGGER = logging.getLogger(__name__) @@ -363,7 +364,7 @@ class YumCollection(Collection): cachefiles.add(self.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def write_config(self): """ Write the server-side config file to :attr:`cfgfile` based on the data from :func:`get_config`""" @@ -465,7 +466,7 @@ class YumCollection(Collection): return "# This config was generated automatically by the Bcfg2 " \ "Packages plugin\n\n" + buf.getvalue() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def build_extra_structures(self, independent): """ Add additional entries to the ``<Independent/>`` section of the final configuration. This adds several kinds of @@ -572,7 +573,7 @@ class YumCollection(Collection): name=self.pulp_cert_set.certpath) self.pulp_cert_set.bind_entry(crt, self.metadata) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _get_pulp_consumer(self, consumerapi=None): """ Get a Pulp consumer object for the client. @@ -601,7 +602,7 @@ class YumCollection(Collection): "%s" % err) return consumer - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _add_gpg_instances(self, keyentry, localkey, remotekey, keydata=None): """ Add GPG keys instances to a ``Package`` entry. This is called from :func:`build_extra_structures` to add GPG keys to @@ -644,7 +645,7 @@ class YumCollection(Collection): self.logger.error("Packages: Could not read GPG key %s: %s" % (localkey, err)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ If using the yum libraries, given a list of package group names, return a dict of ``<group name>: <list of packages>``. @@ -806,7 +807,7 @@ class YumCollection(Collection): new.append(pkg) return new - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): """ Build a complete list of all packages and their dependencies. @@ -846,7 +847,7 @@ class YumCollection(Collection): else: return set(), set() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def call_helper(self, command, inputdata=None): """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have horrific memory leaks, so apparently the right way to get @@ -1067,7 +1068,7 @@ class YumSource(Source): self.file_to_arch[self.escape_url(fullurl)] = arch return urls - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def read_files(self): """ When using the builtin yum parser, read and parse locally downloaded metadata files. This diverges from the stock @@ -1109,7 +1110,7 @@ class YumSource(Source): self.packages[key].difference(self.packages['global']) self.save_state() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_filelist(self, data, arch): """ parse filelists.xml.gz data """ if arch not in self.filemap: @@ -1123,7 +1124,7 @@ class YumSource(Source): self.filemap[arch][fentry.text] = \ set([pkg.get('name')]) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_primary(self, data, arch): """ parse primary.xml.gz data """ if arch not in self.packages: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 169dcd588..ca7b7c530 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -13,6 +13,7 @@ from Bcfg2.Compat import ConfigParser, urlopen, HTTPError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Statistics import track_statistics #: The default path for generated yum configs YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" @@ -235,7 +236,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True return False - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: @@ -270,7 +271,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection.build_extra_structures(indep) structures.append(indep) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _build_packages(self, metadata, independent, structures, collection=None): """ Perform dependency resolution and build the complete list @@ -341,7 +342,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, newpkgs.sort() collection.packages_to_entry(newpkgs, independent) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Refresh(self): """ Packages.Refresh() => True|False @@ -349,7 +350,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config(force_update=True) return True - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Reload(self): """ Packages.Refresh() => True|False @@ -445,7 +446,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if kfile not in keyfiles: os.unlink(kfile) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7b8ef2209..2ea5088de 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -10,6 +10,7 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor +from Bcfg2.Server.Statistics import track_statistics try: from django.db import models @@ -190,7 +191,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.load_data() __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def write_data(self, client): """ Write probe data out for use with bcfg2-info """ if self._use_db: @@ -293,12 +294,12 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def GetProbes(self, meta): return self.probes.get_probe_data(meta) GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def ReceiveData(self, client, datalist): if self.core.metadata_cache_mode in ['cautious', 'aggressive']: if client.hostname in self.cgroups: diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index b7ede594c..762f9f8f0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -10,11 +10,6 @@ import lxml.etree from Bcfg2.Options import get_option_parser import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError -try: - import Bcfg2.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False try: import json diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py new file mode 100644 index 000000000..eeaeb9516 --- /dev/null +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -0,0 +1,452 @@ +""" Bcfg2 SSL server used by the builtin server core +(:mod:`Bcfg2.Server.BuiltinCore`). This needs to be documented +better. """ + +import os +import sys +import socket +import select +import signal +import logging +import ssl +import threading +import time +from Bcfg2.Compat import xmlrpclib, SimpleXMLRPCServer, SocketServer, \ + b64decode + + +class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): + """ An XML-RPC dispatcher. """ + + def __init__(self, allow_none, encoding): + try: + SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, + allow_none, + encoding) + except: + # Python 2.4? + SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self) + + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) + self.allow_none = allow_none + self.encoding = encoding + + def _marshaled_dispatch(self, address, data): + params, method = xmlrpclib.loads(data) + try: + if '.' not in method: + params = (address, ) + params + response = self.instance._dispatch(method, params, self.funcs) + # py3k compatibility + if type(response) not in [bool, str, list, dict]: + response = (response.decode('utf-8'), ) + else: + response = (response, ) + raw_response = xmlrpclib.dumps(response, methodresponse=1, + allow_none=self.allow_none, + encoding=self.encoding) + except xmlrpclib.Fault: + fault = sys.exc_info()[1] + raw_response = xmlrpclib.dumps(fault, + allow_none=self.allow_none, + encoding=self.encoding) + except: + self.logger.error("Unexpected handler error", exc_info=1) + # report exception back to server + raw_response = xmlrpclib.dumps( + xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)), + allow_none=self.allow_none, encoding=self.encoding) + return raw_response + + +class SSLServer(SocketServer.TCPServer, object): + """ TCP server supporting SSL encryption. """ + allow_reuse_address = True + + def __init__(self, listen_all, server_address, RequestHandlerClass, + keyfile=None, certfile=None, reqCert=False, ca=None, + timeout=None, protocol='xmlrpc/ssl'): + """ + :param listen_all: Listen on all interfaces + :type listen_all: bool + :param server_address: Address to bind to the server + :param RequestHandlerClass: Request handler used by TCP server + :param keyfile: Full path to SSL encryption key file + :type keyfile: string + :param certfile: Full path to SSL certificate file + :type certfile: string + :param reqCert: Require client to present certificate + :type reqCert: bool + :param ca: Full path to SSL CA that signed the key and cert + :type ca: string + :param timeout: Timeout for non-blocking request handling + :param protocol: The protocol to serve. Supported values are + ``xmlrpc/ssl`` and ``xmlrpc/tlsv1``. + :type protocol: string + """ + # check whether or not we should listen on all interfaces + if listen_all: + listen_address = ('', server_address[1]) + else: + listen_address = (server_address[0], server_address[1]) + + # check for IPv6 address + if ':' in server_address[0]: + self.address_family = socket.AF_INET6 + + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) + + try: + SocketServer.TCPServer.__init__(self, listen_address, + RequestHandlerClass) + except socket.gaierror: + e = sys.exc_info()[1] + self.logger.error("Failed to bind to socket: %s" % e) + raise + except socket.error: + self.logger.error("Failed to bind to socket") + raise + + self.timeout = timeout + self.socket.settimeout(timeout) + self.keyfile = keyfile + if (keyfile is not None and + (keyfile == False or + not os.path.exists(keyfile) or + not os.access(keyfile, os.R_OK))): + msg = "Keyfile %s does not exist or is not readable" % keyfile + self.logger.error(msg) + raise Exception(msg) + self.certfile = certfile + if (certfile is not None and + (certfile == False or + not os.path.exists(certfile) or + not os.access(certfile, os.R_OK))): + msg = "Certfile %s does not exist or is not readable" % certfile + self.logger.error(msg) + raise Exception(msg) + self.ca = ca + if (ca is not None and + (ca == False or + not os.path.exists(ca) or + not os.access(ca, os.R_OK))): + msg = "CA %s does not exist or is not readable" % ca + self.logger.error(msg) + raise Exception(msg) + self.reqCert = reqCert + if ca and certfile: + self.mode = ssl.CERT_OPTIONAL + else: + self.mode = ssl.CERT_NONE + if protocol == 'xmlrpc/ssl': + self.ssl_protocol = ssl.PROTOCOL_SSLv23 + elif protocol == 'xmlrpc/tlsv1': + self.ssl_protocol = ssl.PROTOCOL_TLSv1 + else: + self.logger.error("Unknown protocol %s" % (protocol)) + raise Exception("unknown protocol %s" % protocol) + + def get_request(self): + (sock, sockinfo) = self.socket.accept() + sock.settimeout(self.timeout) # pylint: disable=E1101 + sslsock = ssl.wrap_socket(sock, + server_side=True, + certfile=self.certfile, + keyfile=self.keyfile, + cert_reqs=self.mode, + ca_certs=self.ca, + ssl_version=self.ssl_protocol) + return sslsock, sockinfo + + def close_request(self, request): + try: + request.unwrap() + except: + pass + try: + request.close() + except: + pass + + def _get_url(self): + port = self.socket.getsockname()[1] + hostname = socket.gethostname() + protocol = "https" + return "%s://%s:%i" % (protocol, hostname, port) + url = property(_get_url) + + +class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): + """ XML-RPC request handler. + + Adds support for HTTP authentication. + """ + + def __init__(self, *args, **kwargs): + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.__init__(self, *args, + **kwargs) + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) + + def authenticate(self): + try: + header = self.headers['Authorization'] + except KeyError: + self.logger.error("No authentication data presented") + return False + auth_content = b64decode(header.split()[1]) + try: + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(bytes(":", encoding='utf-8')) + password = pw.decode('utf-8') + except ValueError: + username = auth_content + password = "" + cert = self.request.getpeercert() + client_address = self.request.getpeername() + return self.server.instance.authenticate(cert, username, + password, client_address) + + def parse_request(self): + """Extends parse_request. + + Optionally check HTTP authentication when parsing. + """ + if not SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.parse_request(self): + return False + try: + if not self.authenticate(): + self.logger.error("Authentication Failure") + self.send_error(401, self.responses[401][0]) + return False + except: # pylint: disable=W0702 + self.logger.error("Unexpected Authentication Failure", exc_info=1) + self.send_error(401, self.responses[401][0]) + return False + return True + + ### need to override do_POST here + def do_POST(self): + try: + max_chunk_size = 10 * 1024 * 1024 + size_remaining = int(self.headers["content-length"]) + L = [] + while size_remaining: + try: + select.select([self.rfile.fileno()], [], [], 3) + except select.error: + print("got select timeout") + raise + chunk_size = min(size_remaining, max_chunk_size) + L.append(self.rfile.read(chunk_size).decode('utf-8')) + size_remaining -= len(L[-1]) + data = ''.join(L) + response = self.server._marshaled_dispatch(self.client_address, + data) + if sys.hexversion >= 0x03000000: + response = response.encode('utf-8') + except: # pylint: disable=W0702 + try: + self.send_response(500) + self.end_headers() + except: + (etype, msg) = sys.exc_info()[:2] + self.logger.error("Error sending 500 response (%s): %s" % + (etype.__name__, msg)) + raise + else: + # got a valid XML RPC response + try: + self.send_response(200) + self.send_header("Content-type", "text/xml") + self.send_header("Content-length", str(len(response))) + self.end_headers() + failcount = 0 + while True: + try: + # If we hit SSL3_WRITE_PENDING here try to resend. + self.wfile.write(response) + break + except ssl.SSLError: + e = sys.exc_info()[1] + if str(e).find("SSL3_WRITE_PENDING") < 0: + raise + self.logger.error("SSL3_WRITE_PENDING") + failcount += 1 + if failcount < 5: + continue + raise + except socket.error: + err = sys.exc_info()[1] + if err[0] == 32: + self.logger.warning("Connection dropped from %s" % + self.client_address[0]) + elif err[0] == 104: + self.logger.warning("Connection reset by peer: %s" % + self.client_address[0]) + else: + self.logger.warning("Socket error sending response to %s: " + "%s" % (self.client_address[0], err)) + except ssl.SSLError: + err = sys.exc_info()[1] + self.logger.warning("SSLError handling client %s: %s" % + (self.client_address[0], err)) + except: + etype, err = sys.exc_info()[:2] + self.logger.error("Unknown error sending response to %s: " + "%s (%s)" % + (self.client_address[0], err, + etype.__name__)) + + def finish(self): + # shut down the connection + if not self.wfile.closed: + try: + self.wfile.flush() + self.wfile.close() + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) + self.rfile.close() + + +class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, + XMLRPCDispatcher, object): + """ Component XMLRPCServer. """ + + def __init__(self, listen_all, server_address, RequestHandlerClass=None, + keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', + timeout=10, logRequests=False, + register=True, allow_none=True, encoding=None): + """ + :param listen_all: Listen on all interfaces + :type listen_all: bool + :param server_address: Address to bind to the server + :param RequestHandlerClass: request handler used by TCP server + :param keyfile: Full path to SSL encryption key file + :type keyfile: string + :param certfile: Full path to SSL certificate file + :type certfile: string + :param ca: Full path to SSL CA that signed the key and cert + :type ca: string + :param logRequests: Log all requests + :type logRequests: bool + :param register: Presence should be reported to service-location + :type register: bool + :param allow_none: Allow None values in XML-RPC + :type allow_non: bool + :param encoding: Encoding to use for XML-RPC + """ + + XMLRPCDispatcher.__init__(self, allow_none, encoding) + + if not RequestHandlerClass: + # pylint: disable=E0102 + class RequestHandlerClass(XMLRPCRequestHandler): + """A subclassed request handler to prevent + class-attribute conflicts.""" + # pylint: enable=E0102 + + SSLServer.__init__(self, + listen_all, + server_address, + RequestHandlerClass, + ca=ca, + timeout=timeout, + keyfile=keyfile, + certfile=certfile, + protocol=protocol) + self.logRequests = logRequests + self.serve = False + self.register = register + self.register_introspection_functions() + self.register_function(self.ping) + self.logger.info("service available at %s" % self.url) + self.timeout = timeout + + def _tasks_thread(self): + try: + while self.serve: + try: + if self.instance and hasattr(self.instance, 'do_tasks'): + self.instance.do_tasks() + except: + self.logger.error("Unexpected task failure", exc_info=1) + time.sleep(self.timeout) + except: + self.logger.error("tasks_thread failed", exc_info=1) + + def server_close(self): + SSLServer.server_close(self) + self.logger.info("server_close()") + + def _get_require_auth(self): + return getattr(self.RequestHandlerClass, "require_auth", False) + + def _set_require_auth(self, value): + self.RequestHandlerClass.require_auth = value + require_auth = property(_get_require_auth, _set_require_auth) + + def _get_credentials(self): + try: + return self.RequestHandlerClass.credentials + except AttributeError: + return dict() + + def _set_credentials(self, value): + self.RequestHandlerClass.credentials = value + credentials = property(_get_credentials, _set_credentials) + + def register_instance(self, instance, *args, **kwargs): + XMLRPCDispatcher.register_instance(self, instance, *args, **kwargs) + try: + name = instance.name + except AttributeError: + name = "unknown" + if hasattr(instance, 'plugins'): + for pname, pinst in list(instance.plugins.items()): + for mname in pinst.__rmi__: + xmname = "%s.%s" % (pname, mname) + fn = getattr(pinst, mname) + self.register_function(fn, name=xmname) + self.logger.info("serving %s at %s" % (name, self.url)) + + def serve_forever(self): + """Serve single requests until (self.serve == False).""" + self.serve = True + self.task_thread = threading.Thread(target=self._tasks_thread) + self.task_thread.start() + self.logger.info("serve_forever() [start]") + signal.signal(signal.SIGINT, self._handle_shutdown_signal) + signal.signal(signal.SIGTERM, self._handle_shutdown_signal) + + try: + while self.serve: + try: + self.handle_request() + except socket.timeout: + pass + except select.error: + pass + except: + self.logger.error("Got unexpected error in handle_request", + exc_info=1) + finally: + self.logger.info("serve_forever() [stop]") + + def shutdown(self): + """Signal that automatic service should stop.""" + self.serve = False + + def _handle_shutdown_signal(self, *_): + self.shutdown() + + def ping(self, *args): + """Echo response.""" + self.logger.info("ping(%s)" % (", ".join([repr(arg) for arg in args]))) + return args diff --git a/src/lib/Bcfg2/Server/Statistics.py b/src/lib/Bcfg2/Server/Statistics.py new file mode 100644 index 000000000..8a6ff54c4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Statistics.py @@ -0,0 +1,118 @@ +""" Module for tracking execution time statistics from the Bcfg2 +server core. This data is exposed by +:func:`Bcfg2.Server.Core.BaseCore.get_statistics`.""" + +import time +from Bcfg2.Compat import wraps + +class Statistic(object): + """ A single named statistic, tracking minimum, maximum, and + average execution time, and number of invocations. """ + + def __init__(self, name, initial_value): + """ + :param name: The name of this statistic + :type name: string + :param initial_value: The initial value to be added to this + statistic + :type initial_value: int or float + """ + self.name = name + self.min = float(initial_value) + self.max = float(initial_value) + self.ave = float(initial_value) + self.count = 1 + + def add_value(self, value): + """ Add a value to the statistic, recalculating the various + metrics. + + :param value: The value to add to this statistic + :type value: int or float + """ + self.min = min(self.min, value) + self.max = max(self.max, value) + self.ave = (((self.ave * (self.count - 1)) + value) / self.count) + self.count += 1 + + def get_value(self): + """ Get a tuple of all the stats tracked on this named item. + The tuple is in the format:: + + (<name>, (min, max, average, number of values)) + + This makes it very easy to cast to a dict in + :func:`Statistics.display`. + + :returns: tuple + """ + return (self.name, (self.min, self.max, self.ave, self.count)) + + +class Statistics(object): + """ A collection of named :class:`Statistic` objects. """ + + def __init__(self): + self.data = dict() + + def add_value(self, name, value): + """ Add a value to the named :class:`Statistic`. This just + proxies to :func:`Statistic.add_value` or the + :class:`Statistic` constructor as appropriate. + + :param name: The name of the :class:`Statistic` to add the + value to + :type name: string + :param value: The value to add to the Statistic + :type value: int or float + """ + if name not in self.data: + self.data[name] = Statistic(name, value) + else: + self.data[name].add_value(value) + + def display(self): + """ Return a dict of all :class:`Statistic` object values. + Keys are the statistic names, and values are tuples of the + statistic metrics as returned by + :func:`Statistic.get_value`. """ + return dict([value.get_value() for value in list(self.data.values())]) + + +#: A module-level :class:`Statistics` objects used to track all +#: execution time metrics for the server. +stats = Statistics() # pylint: disable=C0103 + + +class track_statistics(object): # pylint: disable=C0103 + """ Decorator that tracks execution time for the given method with + :mod:`Bcfg2.Server.Statistics` for reporting via ``bcfg2-admin + perf`` """ + + def __init__(self, name=None): + """ + :param name: The name under which statistics for this function + will be tracked. By default, the name will be + the name of the function concatenated with the + name of the class the function is a member of. + :type name: string + """ + # if this is None, it will be set later during __call_ + self.name = name + + def __call__(self, func): + if self.name is None: + self.name = func.__name__ + + @wraps(func) + def inner(obj, *args, **kwargs): + """ The decorated function """ + name = "%s:%s" % (obj.__class__.__name__, self.name) + + start = time.time() + try: + return func(obj, *args, **kwargs) + finally: + stats.add_value(name, time.time() - start) + + return inner |