From 25cb6db5ccb0c8e8302c220a90344a95baf3909b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 5 Feb 2013 14:04:09 -0500 Subject: moved some libraries in Bcfg2/ into more specific (Server/ or Client/) places --- src/lib/Bcfg2/Cache.py | 14 - src/lib/Bcfg2/Client/Client.py | 22 +- src/lib/Bcfg2/Client/Proxy.py | 358 ++++++++++++++++ src/lib/Bcfg2/Encryption.py | 195 --------- src/lib/Bcfg2/Proxy.py | 357 ---------------- src/lib/Bcfg2/SSLServer.py | 447 -------------------- src/lib/Bcfg2/Server/Admin/Perf.py | 16 +- src/lib/Bcfg2/Server/Admin/Xcmd.py | 20 +- src/lib/Bcfg2/Server/BuiltinCore.py | 13 +- src/lib/Bcfg2/Server/Cache.py | 14 + src/lib/Bcfg2/Server/CherryPyCore.py | 6 +- src/lib/Bcfg2/Server/Core.py | 31 +- src/lib/Bcfg2/Server/Encryption.py | 195 +++++++++ src/lib/Bcfg2/Server/Plugin/helpers.py | 55 +-- .../Server/Plugins/Cfg/CfgEncryptedGenerator.py | 2 +- .../Plugins/Cfg/CfgEncryptedGenshiGenerator.py | 4 +- .../Server/Plugins/Cfg/CfgPrivateKeyCreator.py | 10 +- .../Bcfg2/Server/Plugins/Packages/Collection.py | 7 +- .../Server/Plugins/Packages/PackagesSources.py | 5 +- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 3 +- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 21 +- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 11 +- src/lib/Bcfg2/Server/Plugins/Probes.py | 7 +- src/lib/Bcfg2/Server/Plugins/Properties.py | 5 - src/lib/Bcfg2/Server/SSLServer.py | 452 +++++++++++++++++++++ src/lib/Bcfg2/Server/Statistics.py | 118 ++++++ src/lib/Bcfg2/Statistics.py | 82 ---- src/lib/Bcfg2/__init__.py | 3 - 28 files changed, 1238 insertions(+), 1235 deletions(-) delete mode 100644 src/lib/Bcfg2/Cache.py create mode 100644 src/lib/Bcfg2/Client/Proxy.py delete mode 100755 src/lib/Bcfg2/Encryption.py delete mode 100644 src/lib/Bcfg2/Proxy.py delete mode 100644 src/lib/Bcfg2/SSLServer.py create mode 100644 src/lib/Bcfg2/Server/Cache.py create mode 100755 src/lib/Bcfg2/Server/Encryption.py create mode 100644 src/lib/Bcfg2/Server/SSLServer.py create mode 100644 src/lib/Bcfg2/Server/Statistics.py delete mode 100644 src/lib/Bcfg2/Statistics.py delete mode 100644 src/lib/Bcfg2/__init__.py (limited to 'src/lib/Bcfg2') diff --git a/src/lib/Bcfg2/Cache.py b/src/lib/Bcfg2/Cache.py deleted file mode 100644 index 842098eda..000000000 --- a/src/lib/Bcfg2/Cache.py +++ /dev/null @@ -1,14 +0,0 @@ -""" An implementation of a simple memory-backed cache. Right now this -doesn't provide many features, but more (time-based expiration, etc.) -can be added as necessary. """ - - -class Cache(dict): - """ an implementation of a simple memory-backed cache """ - - def expire(self, key=None): - """ expire all items, or a specific item, from the cache """ - if key is None: - self.clear() - elif key in self: - del self[key] diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index ea898c42b..e26a6d07a 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -8,10 +8,10 @@ import fcntl import socket import logging import tempfile -import Bcfg2.Proxy import Bcfg2.Logger import Bcfg2.Options import Bcfg2.Client.XML +import Bcfg2.Client.Proxy import Bcfg2.Client.Frame import Bcfg2.Client.Tools from Bcfg2.Compat import xmlrpclib @@ -121,7 +121,7 @@ class Client(object): def proxy(self): """ get an XML-RPC proxy to the server """ if self._proxy is None: - self._proxy = Bcfg2.Proxy.ComponentProxy( + self._proxy = Bcfg2.Client.Proxy.ComponentProxy( self.setup['server'], self.setup['user'], self.setup['password'], @@ -141,8 +141,8 @@ class Client(object): try: probes = Bcfg2.Client.XML.XML(str(self.proxy.GetProbes())) - except (Bcfg2.Proxy.ProxyError, - Bcfg2.Proxy.CertificateError, + except (Bcfg2.Client.Proxy.ProxyError, + Bcfg2.Client.Proxy.CertificateError, socket.gaierror, socket.error): err = sys.exc_info()[1] @@ -165,7 +165,7 @@ class Client(object): self.proxy.RecvProbeData(Bcfg2.Client.XML.tostring( probedata, xml_declaration=False).decode('UTF-8')) - except Bcfg2.Proxy.ProxyError: + except Bcfg2.Client.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to upload probe data: %s" % err) @@ -191,7 +191,7 @@ class Client(object): if self.setup['profile']: try: self.proxy.AssertProfile(self.setup['profile']) - except Bcfg2.Proxy.ProxyError: + except Bcfg2.Client.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to set client profile: %s" % err) @@ -206,8 +206,8 @@ class Client(object): "client version") else: self.logger.error("Failed to declare version: %s" % err) - except (Bcfg2.Proxy.ProxyError, - Bcfg2.Proxy.CertificateError, + except (Bcfg2.Client.Proxy.ProxyError, + Bcfg2.Client.Proxy.CertificateError, socket.gaierror, socket.error): err = sys.exc_info()[1] @@ -221,13 +221,13 @@ class Client(object): self.proxy.GetDecisionList(self.setup['decision']) self.logger.info("Got decision list from server:") self.logger.info(self.setup['decision_list']) - except Bcfg2.Proxy.ProxyError: + except Bcfg2.Client.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to get decision list: %s" % err) try: rawconfig = self.proxy.GetConfig().encode('UTF-8') - except Bcfg2.Proxy.ProxyError: + except Bcfg2.Client.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to download configuration from " "Bcfg2: %s" % err) @@ -324,7 +324,7 @@ class Client(object): self.proxy.RecvStats(Bcfg2.Client.XML.tostring( feedback, xml_declaration=False).decode('UTF-8')) - except Bcfg2.Proxy.ProxyError: + except Bcfg2.Client.Proxy.ProxyError: err = sys.exc_info()[1] self.logger.error("Failed to upload configuration statistics: " "%s" % err) diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py new file mode 100644 index 000000000..1276c3ce9 --- /dev/null +++ b/src/lib/Bcfg2/Client/Proxy.py @@ -0,0 +1,358 @@ +import logging +import re +import socket + +# The ssl module is provided by either Python 2.6 or a separate ssl +# package that works on older versions of Python (see +# http://pypi.python.org/pypi/ssl). If neither can be found, look for +# M2Crypto instead. +try: + import ssl + SSL_LIB = 'py26_ssl' + SSL_ERROR = ssl.SSLError +except ImportError: + from M2Crypto import SSL + import M2Crypto.SSL.Checker + SSL_LIB = 'm2crypto' + SSL_ERROR = SSL.SSLError + +import sys +import time + +# Compatibility imports +from Bcfg2.Compat import httplib, xmlrpclib, urlparse + +version = sys.version_info[:2] +has_py26 = version >= (2, 6) + +__all__ = ["ComponentProxy", + "RetryMethod", + "SSLHTTPConnection", + "XMLRPCTransport"] + + +class ProxyError(Exception): + """ ProxyError provides a consistent reporting interface to + the various xmlrpclib errors that might arise (mainly + ProtocolError and Fault) """ + def __init__(self, err): + msg = None + if isinstance(err, xmlrpclib.ProtocolError): + # cut out the password in the URL + url = re.sub(r'([^:]+):(.*?)@([^@]+:\d+/)', r'\1:******@\3', + err.url) + msg = "XML-RPC Protocol Error for %s: %s (%s)" % (url, + err.errmsg, + err.errcode) + elif isinstance(err, xmlrpclib.Fault): + msg = "XML-RPC Fault: %s (%s)" % (err.faultString, + err.faultCode) + else: + msg = str(err) + Exception.__init__(self, msg) + +class CertificateError(Exception): + def __init__(self, commonName): + self.commonName = commonName + def __str__(self): + return ("Got unallowed commonName %s from server" + % self.commonName) + +_orig_Method = xmlrpclib._Method + +class RetryMethod(xmlrpclib._Method): + """Method with error handling and retries built in.""" + log = logging.getLogger('xmlrpc') + max_retries = 3 + retry_delay = 1 + + def __call__(self, *args): + for retry in range(self.max_retries): + if retry >= self.max_retries - 1: + final = True + else: + final = False + msg = None + try: + return _orig_Method.__call__(self, *args) + except xmlrpclib.ProtocolError: + err = sys.exc_info()[1] + msg = "Server failure: Protocol Error: %s %s" % \ + (err.errcode, err.errmsg) + except xmlrpclib.Fault: + msg = sys.exc_info()[1] + except socket.error: + err = sys.exc_info()[1] + if hasattr(err, 'errno') and err.errno == 336265218: + msg = "SSL Key error: %s" % err + elif hasattr(err, 'errno') and err.errno == 185090050: + msg = "SSL CA error: %s" % err + elif final: + msg = "Server failure: %s" % err + except CertificateError: + err = sys.exc_info()[1] + msg = "Got unallowed commonName %s from server" % \ + err.commonName + except KeyError: + err = sys.exc_info()[1] + msg = "Server disallowed connection: %s" % err + except ProxyError: + err = sys.exc_info()[1] + msg = err + except: + raise + etype, err = sys.exc_info()[:2] + msg = "Unknown failure: %s (%s)" % (err, etype.__name__) + if msg: + if final: + self.log.error(msg) + raise ProxyError(msg) + else: + self.log.info(msg) + time.sleep(self.retry_delay) + +xmlrpclib._Method = RetryMethod + + +class SSLHTTPConnection(httplib.HTTPConnection): + """Extension of HTTPConnection that + implements SSL and related behaviors. + """ + + def __init__(self, host, port=None, strict=None, timeout=90, key=None, + cert=None, ca=None, scns=None, protocol='xmlrpc/ssl'): + """Initializes the `httplib.HTTPConnection` object and stores security + parameters + + Parameters + ---------- + host : string + Name of host to contact + port : int, optional + Port on which to contact the host. If none is specified, + the default port of 80 will be used unless the `host` + string has a port embedded in the form host:port. + strict : Boolean, optional + Passed to the `httplib.HTTPConnection` constructor and if + True, causes the `BadStatusLine` exception to be raised if + the status line cannot be parsed as a valid HTTP 1.0 or + 1.1 status. + timeout : int, optional + Causes blocking operations to timeout after `timeout` + seconds. + key : string, optional + The file system path to the local endpoint's SSL key. May + specify the same file as `cert` if using a file that + contains both. See + http://docs.python.org/library/ssl.html#ssl-certificates + for details. Required if using xmlrpc/ssl with client + certificate authentication. + cert : string, optional + The file system path to the local endpoint's SSL + certificate. May specify the same file as `cert` if using + a file that contains both. See + http://docs.python.org/library/ssl.html#ssl-certificates + for details. Required if using xmlrpc/ssl with client + certificate authentication. + ca : string, optional + The file system path to a set of concatenated certificate + authority certs, which are used to validate certificates + passed from the other end of the connection. + scns : array-like, optional + List of acceptable server commonNames. The peer cert's + common name must appear in this list, otherwise the + connect() call will throw a `CertificateError`. + protocol : {'xmlrpc/ssl', 'xmlrpc/tlsv1'}, optional + Communication protocol to use. + + """ + if not has_py26: + httplib.HTTPConnection.__init__(self, host, port, strict) + else: + httplib.HTTPConnection.__init__(self, host, port, strict, timeout) + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) + + self.key = key + self.cert = cert + self.ca = ca + self.scns = scns + self.protocol = protocol + self.timeout = timeout + + def connect(self): + """Initiates a connection using previously set attributes.""" + if SSL_LIB == 'py26_ssl': + self._connect_py26ssl() + elif SSL_LIB == 'm2crypto': + self._connect_m2crypto() + else: + raise Exception("No SSL module support") + + def _connect_py26ssl(self): + """Initiates a connection using the ssl module.""" + # check for IPv6 + hostip = socket.getaddrinfo(self.host, + self.port, + socket.AF_UNSPEC, + socket.SOCK_STREAM)[0][4][0] + if ':' in hostip: + rawsock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + else: + rawsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if self.protocol == 'xmlrpc/ssl': + ssl_protocol_ver = ssl.PROTOCOL_SSLv23 + elif self.protocol == 'xmlrpc/tlsv1': + ssl_protocol_ver = ssl.PROTOCOL_TLSv1 + else: + self.logger.error("Unknown protocol %s" % (self.protocol)) + raise Exception("unknown protocol %s" % self.protocol) + if self.ca: + other_side_required = ssl.CERT_REQUIRED + else: + other_side_required = ssl.CERT_NONE + self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") + if self.cert and not self.key: + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") + self.cert = None + if self.key and not self.cert: + self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") + self.key = None + + rawsock.settimeout(self.timeout) + self.sock = ssl.SSLSocket(rawsock, cert_reqs=other_side_required, + ca_certs=self.ca, suppress_ragged_eofs=True, + keyfile=self.key, certfile=self.cert, + ssl_version=ssl_protocol_ver) + self.sock.connect((self.host, self.port)) + peer_cert = self.sock.getpeercert() + if peer_cert and self.scns: + scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] + if scn not in self.scns: + raise CertificateError(scn) + self.sock.closeSocket = True + + def _connect_m2crypto(self): + """Initiates a connection using the M2Crypto module.""" + + if self.protocol == 'xmlrpc/ssl': + ctx = SSL.Context('sslv23') + elif self.protocol == 'xmlrpc/tlsv1': + ctx = SSL.Context('tlsv1') + else: + self.logger.error("Unknown protocol %s" % (self.protocol)) + raise Exception("unknown protocol %s" % self.protocol) + + if self.ca: + # Use the certificate authority to validate the cert + # presented by the server + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) + if ctx.load_verify_locations(self.ca) != 1: + raise Exception('No CA certs') + else: + self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") + + if self.cert and self.key: + # A cert/key is defined, use them to support client + # authentication to the server + ctx.load_cert(self.cert, self.key) + elif self.cert: + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") + elif self.key: + self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") + + self.sock = SSL.Connection(ctx) + if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', self.host): + # host is ip address + try: + hostname = socket.gethostbyaddr(self.host)[0] + except: + # fall back to ip address + hostname = self.host + else: + hostname = self.host + try: + self.sock.connect((hostname, self.port)) + # automatically checks cert matches host + except M2Crypto.SSL.Checker.WrongHost: + wr = sys.exc_info()[1] + raise CertificateError(wr) + + +class XMLRPCTransport(xmlrpclib.Transport): + def __init__(self, key=None, cert=None, ca=None, + scns=None, use_datetime=0, timeout=90): + if hasattr(xmlrpclib.Transport, '__init__'): + xmlrpclib.Transport.__init__(self, use_datetime) + self.key = key + self.cert = cert + self.ca = ca + self.scns = scns + self.timeout = timeout + + def make_connection(self, host): + host, self._extra_headers = self.get_host_info(host)[0:2] + return SSLHTTPConnection(host, + key=self.key, + cert=self.cert, + ca=self.ca, + scns=self.scns, + timeout=self.timeout) + + def request(self, host, handler, request_body, verbose=0): + """Send request to server and return response.""" + try: + conn = self.send_request(host, handler, request_body, False) + response = conn.getresponse() + errcode = response.status + errmsg = response.reason + headers = response.msg + except (socket.error, SSL_ERROR): + err = sys.exc_info()[1] + raise ProxyError(xmlrpclib.ProtocolError(host + handler, + 408, + str(err), + self._extra_headers)) + + if errcode != 200: + raise ProxyError(xmlrpclib.ProtocolError(host + handler, + errcode, + errmsg, + headers)) + + self.verbose = verbose + return self.parse_response(response) + + if sys.hexversion < 0x03000000: + def send_request(self, host, handler, request_body, debug): + """ send_request() changed significantly in py3k.""" + conn = self.make_connection(host) + xmlrpclib.Transport.send_request(self, conn, handler, request_body) + self.send_host(conn, host) + self.send_user_agent(conn) + self.send_content(conn, request_body) + return conn + + +def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, + allowedServerCNs=None, timeout=90, retries=3, delay=1): + + """Constructs proxies to components. + + Arguments: + component_name -- name of the component to connect to + + Additional arguments are passed to the ServerProxy constructor. + + """ + xmlrpclib._Method.max_retries = retries + xmlrpclib._Method.retry_delay = delay + + if user and password: + method, path = urlparse(url)[:2] + newurl = "%s://%s:%s@%s" % (method, user, password, path) + else: + newurl = url + ssl_trans = XMLRPCTransport(key, cert, ca, + allowedServerCNs, timeout=float(timeout)) + return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py deleted file mode 100755 index 341956e03..000000000 --- a/src/lib/Bcfg2/Encryption.py +++ /dev/null @@ -1,195 +0,0 @@ -""" Bcfg2.Encryption provides a number of convenience methods for -handling encryption in Bcfg2. See :ref:`server-encryption` for more -details. """ - -import os -import Bcfg2.Options -from M2Crypto import Rand -from M2Crypto.EVP import Cipher, EVPError -from Bcfg2.Compat import StringIO, md5, b64encode, b64decode - -#: Constant representing the encryption operation for -#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This -#: makes our code more readable. -ENCRYPT = 1 - -#: Constant representing the decryption operation for -#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This -#: makes our code more readable. -DECRYPT = 0 - -#: Default initialization vector. For best security, you should use a -#: unique IV for each message. :func:`ssl_encrypt` does this in an -#: automated fashion. -IV = '\0' * 16 - -#: The config file section encryption options and passphrases are -#: stored in -CFG_SECTION = "encryption" - -#: The config option used to store the algorithm -CFG_ALGORITHM = "algorithm" - -#: Default cipher algorithm. To get a full list of valid algorithms, -#: you can run:: -#: -#: openssl list-cipher-algorithms | grep -v ' => ' | \ -#: tr 'A-Z-' 'a-z_' | sort -u -ALGORITHM = Bcfg2.Options.get_option_parser().cfp.get( # pylint: disable=E1103 - CFG_SECTION, - CFG_ALGORITHM, - default="aes_256_cbc").lower().replace("-", "_") - -Rand.rand_seed(os.urandom(1024)) - - -def _cipher_filter(cipher, instr): - """ M2Crypto reads and writes file-like objects, so this uses - StringIO to pass data through it """ - inbuf = StringIO(instr) - outbuf = StringIO() - while 1: - buf = inbuf.read() - if not buf: - break - outbuf.write(cipher.update(buf)) - outbuf.write(cipher.final()) - rv = outbuf.getvalue() - inbuf.close() - outbuf.close() - return rv - - -def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): - """ Encrypt a string with a key. For a higher-level encryption - interface, see :func:`ssl_encrypt`. - - :param plaintext: The plaintext data to encrypt - :type plaintext: string - :param key: The key to encrypt the data with - :type key: string - :param iv: The initialization vector - :type iv: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :param salt: The salt to use - :type salt: string - :returns: string - The decrypted data - """ - cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) - return _cipher_filter(cipher, plaintext) - - -def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): - """ Decrypt a string with a key. For a higher-level decryption - interface, see :func:`ssl_decrypt`. - - :param crypted: The raw binary encrypted data - :type crypted: string - :param key: The encryption key to decrypt with - :type key: string - :param iv: The initialization vector - :type iv: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :returns: string - The decrypted data - """ - cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) - return _cipher_filter(cipher, crypted) - - -def ssl_decrypt(data, passwd, algorithm=ALGORITHM): - """ Decrypt openssl-encrypted data. This can decrypt data - encrypted by :func:`ssl_encrypt`, or ``openssl enc``. It performs - a base64 decode first if the data is base64 encoded, and - automatically determines the salt and initialization vector (both - of which are embedded in the encrypted data). - - :param data: The encrypted data (either base64-encoded or raw - binary) to decrypt - :type data: string - :param passwd: The password to use to decrypt the data - :type passwd: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :returns: string - The decrypted data - """ - # base64-decode the data - data = b64decode(data) - salt = data[8:16] - hashes = [md5(passwd + salt).digest()] - for i in range(1, 3): - hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) - key = hashes[0] + hashes[1] - iv = hashes[2] - - return str_decrypt(data[16:], key=key, iv=iv, algorithm=algorithm) - - -def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): - """ Encrypt data in a format that is openssl compatible. - - :param plaintext: The plaintext data to encrypt - :type plaintext: string - :param passwd: The password to use to encrypt the data - :type passwd: string - :param algorithm: The cipher algorithm to use - :type algorithm: string - :param salt: The salt to use. If none is provided, one will be - randomly generated. - :type salt: bytes - :returns: string - The base64-encoded, salted, encrypted string. - The string includes a trailing newline to make it fully - compatible with openssl command-line tools. - """ - if salt is None: - salt = Rand.rand_bytes(8) - - hashes = [md5(passwd + salt).digest()] - for i in range(1, 3): - hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) - key = hashes[0] + hashes[1] - iv = hashes[2] - - crypted = str_encrypt(plaintext, key=key, salt=salt, iv=iv, - algorithm=algorithm) - return b64encode("Salted__" + salt + crypted) + "\n" - - -def get_passphrases(): - """ Get all candidate encryption passphrases from the config file. - - :returns: dict - a dict of ````: ```` - """ - setup = Bcfg2.Options.get_option_parser() - if setup.cfp.has_section(CFG_SECTION): - return dict([(o, setup.cfp.get(CFG_SECTION, o)) - for o in setup.cfp.options(CFG_SECTION) - if o != CFG_ALGORITHM]) - else: - return dict() - - -def bruteforce_decrypt(crypted, passphrases=None, algorithm=ALGORITHM): - """ Convenience method to decrypt the given encrypted string by - trying the given passphrases or all passphrases (as returned by - :func:`get_passphrases`) sequentially until one is found that - works. - - :param crypted: The data to decrypt - :type crypted: string - :param passphrases: The passphrases to try. - :type passphrases: list - :param algorithm: The cipher algorithm to use - :type algorithm: string - :returns: string - The decrypted data - :raises: :class:`M2Crypto.EVP.EVPError`, if the data cannot be decrypted - """ - if passphrases is None: - passphrases = get_passphrases().values() - for passwd in passphrases: - try: - return ssl_decrypt(crypted, passwd, algorithm=algorithm) - except EVPError: - pass - raise EVPError("Failed to decrypt") diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py deleted file mode 100644 index 3b406c78e..000000000 --- a/src/lib/Bcfg2/Proxy.py +++ /dev/null @@ -1,357 +0,0 @@ -import logging -import re -import socket - -# The ssl module is provided by either Python 2.6 or a separate ssl -# package that works on older versions of Python (see -# http://pypi.python.org/pypi/ssl). If neither can be found, look for -# M2Crypto instead. -try: - import ssl - SSL_LIB = 'py26_ssl' - SSL_ERROR = ssl.SSLError -except ImportError: - from M2Crypto import SSL - import M2Crypto.SSL.Checker - SSL_LIB = 'm2crypto' - SSL_ERROR = SSL.SSLError - -import sys -import time - -# Compatibility imports -from Bcfg2.Compat import httplib, xmlrpclib, urlparse - -version = sys.version_info[:2] -has_py26 = version >= (2, 6) - -__all__ = ["ComponentProxy", - "RetryMethod", - "SSLHTTPConnection", - "XMLRPCTransport"] - - -class ProxyError(Exception): - """ ProxyError provides a consistent reporting interface to - the various xmlrpclib errors that might arise (mainly - ProtocolError and Fault) """ - def __init__(self, err): - msg = None - if isinstance(err, xmlrpclib.ProtocolError): - # cut out the password in the URL - url = re.sub(r'([^:]+):(.*?)@([^@]+:\d+/)', r'\1:******@\3', - err.url) - msg = "XML-RPC Protocol Error for %s: %s (%s)" % (url, - err.errmsg, - err.errcode) - elif isinstance(err, xmlrpclib.Fault): - msg = "XML-RPC Fault: %s (%s)" % (err.faultString, - err.faultCode) - else: - msg = str(err) - Exception.__init__(self, msg) - -class CertificateError(Exception): - def __init__(self, commonName): - self.commonName = commonName - def __str__(self): - return ("Got unallowed commonName %s from server" - % self.commonName) - -_orig_Method = xmlrpclib._Method - -class RetryMethod(xmlrpclib._Method): - """Method with error handling and retries built in.""" - log = logging.getLogger('xmlrpc') - max_retries = 3 - retry_delay = 1 - - def __call__(self, *args): - for retry in range(self.max_retries): - if retry >= self.max_retries - 1: - final = True - else: - final = False - msg = None - try: - return _orig_Method.__call__(self, *args) - except xmlrpclib.ProtocolError: - err = sys.exc_info()[1] - msg = "Server failure: Protocol Error: %s %s" % \ - (err.errcode, err.errmsg) - except xmlrpclib.Fault: - msg = sys.exc_info()[1] - except socket.error: - err = sys.exc_info()[1] - if hasattr(err, 'errno') and err.errno == 336265218: - msg = "SSL Key error: %s" % err - elif hasattr(err, 'errno') and err.errno == 185090050: - msg = "SSL CA error: %s" % err - elif final: - msg = "Server failure: %s" % err - except CertificateError: - err = sys.exc_info()[1] - msg = "Got unallowed commonName %s from server" % \ - err.commonName - except KeyError: - err = sys.exc_info()[1] - msg = "Server disallowed connection: %s" % err - except ProxyError: - err = sys.exc_info()[1] - msg = err - except: - raise - etype, err = sys.exc_info()[:2] - msg = "Unknown failure: %s (%s)" % (err, etype.__name__) - if msg: - if final: - self.log.error(msg) - raise ProxyError(msg) - else: - self.log.info(msg) - time.sleep(self.retry_delay) - -xmlrpclib._Method = RetryMethod - - -class SSLHTTPConnection(httplib.HTTPConnection): - """Extension of HTTPConnection that - implements SSL and related behaviors. - """ - - logger = logging.getLogger('Bcfg2.Proxy.SSLHTTPConnection') - - def __init__(self, host, port=None, strict=None, timeout=90, key=None, - cert=None, ca=None, scns=None, protocol='xmlrpc/ssl'): - """Initializes the `httplib.HTTPConnection` object and stores security - parameters - - Parameters - ---------- - host : string - Name of host to contact - port : int, optional - Port on which to contact the host. If none is specified, - the default port of 80 will be used unless the `host` - string has a port embedded in the form host:port. - strict : Boolean, optional - Passed to the `httplib.HTTPConnection` constructor and if - True, causes the `BadStatusLine` exception to be raised if - the status line cannot be parsed as a valid HTTP 1.0 or - 1.1 status. - timeout : int, optional - Causes blocking operations to timeout after `timeout` - seconds. - key : string, optional - The file system path to the local endpoint's SSL key. May - specify the same file as `cert` if using a file that - contains both. See - http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. - cert : string, optional - The file system path to the local endpoint's SSL - certificate. May specify the same file as `cert` if using - a file that contains both. See - http://docs.python.org/library/ssl.html#ssl-certificates - for details. Required if using xmlrpc/ssl with client - certificate authentication. - ca : string, optional - The file system path to a set of concatenated certificate - authority certs, which are used to validate certificates - passed from the other end of the connection. - scns : array-like, optional - List of acceptable server commonNames. The peer cert's - common name must appear in this list, otherwise the - connect() call will throw a `CertificateError`. - protocol : {'xmlrpc/ssl', 'xmlrpc/tlsv1'}, optional - Communication protocol to use. - - """ - if not has_py26: - httplib.HTTPConnection.__init__(self, host, port, strict) - else: - httplib.HTTPConnection.__init__(self, host, port, strict, timeout) - self.key = key - self.cert = cert - self.ca = ca - self.scns = scns - self.protocol = protocol - self.timeout = timeout - - def connect(self): - """Initiates a connection using previously set attributes.""" - if SSL_LIB == 'py26_ssl': - self._connect_py26ssl() - elif SSL_LIB == 'm2crypto': - self._connect_m2crypto() - else: - raise Exception("No SSL module support") - - def _connect_py26ssl(self): - """Initiates a connection using the ssl module.""" - # check for IPv6 - hostip = socket.getaddrinfo(self.host, - self.port, - socket.AF_UNSPEC, - socket.SOCK_STREAM)[0][4][0] - if ':' in hostip: - rawsock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - else: - rawsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if self.protocol == 'xmlrpc/ssl': - ssl_protocol_ver = ssl.PROTOCOL_SSLv23 - elif self.protocol == 'xmlrpc/tlsv1': - ssl_protocol_ver = ssl.PROTOCOL_TLSv1 - else: - self.logger.error("Unknown protocol %s" % (self.protocol)) - raise Exception("unknown protocol %s" % self.protocol) - if self.ca: - other_side_required = ssl.CERT_REQUIRED - else: - other_side_required = ssl.CERT_NONE - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") - if self.cert and not self.key: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") - self.cert = None - if self.key and not self.cert: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") - self.key = None - - rawsock.settimeout(self.timeout) - self.sock = ssl.SSLSocket(rawsock, cert_reqs=other_side_required, - ca_certs=self.ca, suppress_ragged_eofs=True, - keyfile=self.key, certfile=self.cert, - ssl_version=ssl_protocol_ver) - self.sock.connect((self.host, self.port)) - peer_cert = self.sock.getpeercert() - if peer_cert and self.scns: - scn = [x[0][1] for x in peer_cert['subject'] if x[0][0] == 'commonName'][0] - if scn not in self.scns: - raise CertificateError(scn) - self.sock.closeSocket = True - - def _connect_m2crypto(self): - """Initiates a connection using the M2Crypto module.""" - - if self.protocol == 'xmlrpc/ssl': - ctx = SSL.Context('sslv23') - elif self.protocol == 'xmlrpc/tlsv1': - ctx = SSL.Context('tlsv1') - else: - self.logger.error("Unknown protocol %s" % (self.protocol)) - raise Exception("unknown protocol %s" % self.protocol) - - if self.ca: - # Use the certificate authority to validate the cert - # presented by the server - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) - if ctx.load_verify_locations(self.ca) != 1: - raise Exception('No CA certs') - else: - self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") - - if self.cert and self.key: - # A cert/key is defined, use them to support client - # authentication to the server - ctx.load_cert(self.cert, self.key) - elif self.cert: - self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") - elif self.key: - self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") - - self.sock = SSL.Connection(ctx) - if re.match('\\d+\\.\\d+\\.\\d+\\.\\d+', self.host): - # host is ip address - try: - hostname = socket.gethostbyaddr(self.host)[0] - except: - # fall back to ip address - hostname = self.host - else: - hostname = self.host - try: - self.sock.connect((hostname, self.port)) - # automatically checks cert matches host - except M2Crypto.SSL.Checker.WrongHost: - wr = sys.exc_info()[1] - raise CertificateError(wr) - - -class XMLRPCTransport(xmlrpclib.Transport): - def __init__(self, key=None, cert=None, ca=None, - scns=None, use_datetime=0, timeout=90): - if hasattr(xmlrpclib.Transport, '__init__'): - xmlrpclib.Transport.__init__(self, use_datetime) - self.key = key - self.cert = cert - self.ca = ca - self.scns = scns - self.timeout = timeout - - def make_connection(self, host): - host, self._extra_headers = self.get_host_info(host)[0:2] - return SSLHTTPConnection(host, - key=self.key, - cert=self.cert, - ca=self.ca, - scns=self.scns, - timeout=self.timeout) - - def request(self, host, handler, request_body, verbose=0): - """Send request to server and return response.""" - try: - conn = self.send_request(host, handler, request_body, False) - response = conn.getresponse() - errcode = response.status - errmsg = response.reason - headers = response.msg - except (socket.error, SSL_ERROR): - err = sys.exc_info()[1] - raise ProxyError(xmlrpclib.ProtocolError(host + handler, - 408, - str(err), - self._extra_headers)) - - if errcode != 200: - raise ProxyError(xmlrpclib.ProtocolError(host + handler, - errcode, - errmsg, - headers)) - - self.verbose = verbose - return self.parse_response(response) - - if sys.hexversion < 0x03000000: - def send_request(self, host, handler, request_body, debug): - """ send_request() changed significantly in py3k.""" - conn = self.make_connection(host) - xmlrpclib.Transport.send_request(self, conn, handler, request_body) - self.send_host(conn, host) - self.send_user_agent(conn) - self.send_content(conn, request_body) - return conn - - -def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, - allowedServerCNs=None, timeout=90, retries=3, delay=1): - - """Constructs proxies to components. - - Arguments: - component_name -- name of the component to connect to - - Additional arguments are passed to the ServerProxy constructor. - - """ - xmlrpclib._Method.max_retries = retries - xmlrpclib._Method.retry_delay = delay - - if user and password: - method, path = urlparse(url)[:2] - newurl = "%s://%s:%s@%s" % (method, user, password, path) - else: - newurl = url - ssl_trans = XMLRPCTransport(key, cert, ca, - allowedServerCNs, timeout=float(timeout)) - return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py deleted file mode 100644 index 5e3c6232a..000000000 --- a/src/lib/Bcfg2/SSLServer.py +++ /dev/null @@ -1,447 +0,0 @@ -""" Bcfg2 SSL server used by the builtin server core -(:mod:`Bcfg2.Server.BuiltinCore`). This needs to be documented -better. """ - -import os -import sys -import socket -import select -import signal -import logging -import ssl -import threading -import time -from Bcfg2.Compat import xmlrpclib, SimpleXMLRPCServer, SocketServer, \ - b64decode - - -class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): - """ An XML-RPC dispatcher. """ - - logger = logging.getLogger("Bcfg2.SSLServer.XMLRPCDispatcher") - - def __init__(self, allow_none, encoding): - try: - SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, - allow_none, - encoding) - except: - # Python 2.4? - SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self) - - self.allow_none = allow_none - self.encoding = encoding - - def _marshaled_dispatch(self, address, data): - params, method = xmlrpclib.loads(data) - try: - if '.' not in method: - params = (address, ) + params - response = self.instance._dispatch(method, params, self.funcs) - # py3k compatibility - if type(response) not in [bool, str, list, dict]: - response = (response.decode('utf-8'), ) - else: - response = (response, ) - raw_response = xmlrpclib.dumps(response, methodresponse=1, - allow_none=self.allow_none, - encoding=self.encoding) - except xmlrpclib.Fault: - fault = sys.exc_info()[1] - raw_response = xmlrpclib.dumps(fault, - allow_none=self.allow_none, - encoding=self.encoding) - except: - self.logger.error("Unexpected handler error", exc_info=1) - # report exception back to server - raw_response = xmlrpclib.dumps( - xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)), - allow_none=self.allow_none, encoding=self.encoding) - return raw_response - - -class SSLServer(SocketServer.TCPServer, object): - """ TCP server supporting SSL encryption. """ - - allow_reuse_address = True - logger = logging.getLogger("Bcfg2.SSLServer.SSLServer") - - def __init__(self, listen_all, server_address, RequestHandlerClass, - keyfile=None, certfile=None, reqCert=False, ca=None, - timeout=None, protocol='xmlrpc/ssl'): - """ - :param listen_all: Listen on all interfaces - :type listen_all: bool - :param server_address: Address to bind to the server - :param RequestHandlerClass: Request handler used by TCP server - :param keyfile: Full path to SSL encryption key file - :type keyfile: string - :param certfile: Full path to SSL certificate file - :type certfile: string - :param reqCert: Require client to present certificate - :type reqCert: bool - :param ca: Full path to SSL CA that signed the key and cert - :type ca: string - :param timeout: Timeout for non-blocking request handling - :param protocol: The protocol to serve. Supported values are - ``xmlrpc/ssl`` and ``xmlrpc/tlsv1``. - :type protocol: string - """ - # check whether or not we should listen on all interfaces - if listen_all: - listen_address = ('', server_address[1]) - else: - listen_address = (server_address[0], server_address[1]) - - # check for IPv6 address - if ':' in server_address[0]: - self.address_family = socket.AF_INET6 - - try: - SocketServer.TCPServer.__init__(self, listen_address, - RequestHandlerClass) - except socket.gaierror: - e = sys.exc_info()[1] - self.logger.error("Failed to bind to socket: %s" % e) - raise - except socket.error: - self.logger.error("Failed to bind to socket") - raise - - self.timeout = timeout - self.socket.settimeout(timeout) - self.keyfile = keyfile - if (keyfile is not None and - (keyfile == False or - not os.path.exists(keyfile) or - not os.access(keyfile, os.R_OK))): - msg = "Keyfile %s does not exist or is not readable" % keyfile - self.logger.error(msg) - raise Exception(msg) - self.certfile = certfile - if (certfile is not None and - (certfile == False or - not os.path.exists(certfile) or - not os.access(certfile, os.R_OK))): - msg = "Certfile %s does not exist or is not readable" % certfile - self.logger.error(msg) - raise Exception(msg) - self.ca = ca - if (ca is not None and - (ca == False or - not os.path.exists(ca) or - not os.access(ca, os.R_OK))): - msg = "CA %s does not exist or is not readable" % ca - self.logger.error(msg) - raise Exception(msg) - self.reqCert = reqCert - if ca and certfile: - self.mode = ssl.CERT_OPTIONAL - else: - self.mode = ssl.CERT_NONE - if protocol == 'xmlrpc/ssl': - self.ssl_protocol = ssl.PROTOCOL_SSLv23 - elif protocol == 'xmlrpc/tlsv1': - self.ssl_protocol = ssl.PROTOCOL_TLSv1 - else: - self.logger.error("Unknown protocol %s" % (protocol)) - raise Exception("unknown protocol %s" % protocol) - - def get_request(self): - (sock, sockinfo) = self.socket.accept() - sock.settimeout(self.timeout) # pylint: disable=E1101 - sslsock = ssl.wrap_socket(sock, - server_side=True, - certfile=self.certfile, - keyfile=self.keyfile, - cert_reqs=self.mode, - ca_certs=self.ca, - ssl_version=self.ssl_protocol) - return sslsock, sockinfo - - def close_request(self, request): - try: - request.unwrap() - except: - pass - try: - request.close() - except: - pass - - def _get_url(self): - port = self.socket.getsockname()[1] - hostname = socket.gethostname() - protocol = "https" - return "%s://%s:%i" % (protocol, hostname, port) - url = property(_get_url) - - -class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): - """ XML-RPC request handler. - - Adds support for HTTP authentication. - """ - - logger = logging.getLogger("Bcfg2.SSLServer.XMLRPCRequestHandler") - - def authenticate(self): - try: - header = self.headers['Authorization'] - except KeyError: - self.logger.error("No authentication data presented") - return False - auth_content = b64decode(header.split()[1]) - try: - # py3k compatibility - try: - username, password = auth_content.split(":") - except TypeError: - username, pw = auth_content.split(bytes(":", encoding='utf-8')) - password = pw.decode('utf-8') - except ValueError: - username = auth_content - password = "" - cert = self.request.getpeercert() - client_address = self.request.getpeername() - return self.server.instance.authenticate(cert, username, - password, client_address) - - def parse_request(self): - """Extends parse_request. - - Optionally check HTTP authentication when parsing. - """ - if not SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.parse_request(self): - return False - try: - if not self.authenticate(): - self.logger.error("Authentication Failure") - self.send_error(401, self.responses[401][0]) - return False - except: # pylint: disable=W0702 - self.logger.error("Unexpected Authentication Failure", exc_info=1) - self.send_error(401, self.responses[401][0]) - return False - return True - - ### need to override do_POST here - def do_POST(self): - try: - max_chunk_size = 10 * 1024 * 1024 - size_remaining = int(self.headers["content-length"]) - L = [] - while size_remaining: - try: - select.select([self.rfile.fileno()], [], [], 3) - except select.error: - print("got select timeout") - raise - chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size).decode('utf-8')) - size_remaining -= len(L[-1]) - data = ''.join(L) - response = self.server._marshaled_dispatch(self.client_address, - data) - if sys.hexversion >= 0x03000000: - response = response.encode('utf-8') - except: # pylint: disable=W0702 - try: - self.send_response(500) - self.end_headers() - except: - (etype, msg) = sys.exc_info()[:2] - self.logger.error("Error sending 500 response (%s): %s" % - (etype.__name__, msg)) - raise - else: - # got a valid XML RPC response - try: - self.send_response(200) - self.send_header("Content-type", "text/xml") - self.send_header("Content-length", str(len(response))) - self.end_headers() - failcount = 0 - while True: - try: - # If we hit SSL3_WRITE_PENDING here try to resend. - self.wfile.write(response) - break - except ssl.SSLError: - e = sys.exc_info()[1] - if str(e).find("SSL3_WRITE_PENDING") < 0: - raise - self.logger.error("SSL3_WRITE_PENDING") - failcount += 1 - if failcount < 5: - continue - raise - except socket.error: - err = sys.exc_info()[1] - if err[0] == 32: - self.logger.warning("Connection dropped from %s" % - self.client_address[0]) - elif err[0] == 104: - self.logger.warning("Connection reset by peer: %s" % - self.client_address[0]) - else: - self.logger.warning("Socket error sending response to %s: " - "%s" % (self.client_address[0], err)) - except ssl.SSLError: - err = sys.exc_info()[1] - self.logger.warning("SSLError handling client %s: %s" % - (self.client_address[0], err)) - except: - etype, err = sys.exc_info()[:2] - self.logger.error("Unknown error sending response to %s: " - "%s (%s)" % - (self.client_address[0], err, - etype.__name__)) - - def finish(self): - # shut down the connection - if not self.wfile.closed: - try: - self.wfile.flush() - self.wfile.close() - except socket.error: - err = sys.exc_info()[1] - self.logger.warning("Error closing connection: %s" % err) - self.rfile.close() - - -class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, - XMLRPCDispatcher, object): - """ Component XMLRPCServer. """ - - def __init__(self, listen_all, server_address, RequestHandlerClass=None, - keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', - timeout=10, logRequests=False, - register=True, allow_none=True, encoding=None): - """ - :param listen_all: Listen on all interfaces - :type listen_all: bool - :param server_address: Address to bind to the server - :param RequestHandlerClass: request handler used by TCP server - :param keyfile: Full path to SSL encryption key file - :type keyfile: string - :param certfile: Full path to SSL certificate file - :type certfile: string - :param ca: Full path to SSL CA that signed the key and cert - :type ca: string - :param logRequests: Log all requests - :type logRequests: bool - :param register: Presence should be reported to service-location - :type register: bool - :param allow_none: Allow None values in XML-RPC - :type allow_non: bool - :param encoding: Encoding to use for XML-RPC - """ - - XMLRPCDispatcher.__init__(self, allow_none, encoding) - - if not RequestHandlerClass: - # pylint: disable=E0102 - class RequestHandlerClass(XMLRPCRequestHandler): - """A subclassed request handler to prevent - class-attribute conflicts.""" - # pylint: enable=E0102 - - SSLServer.__init__(self, - listen_all, - server_address, - RequestHandlerClass, - ca=ca, - timeout=timeout, - keyfile=keyfile, - certfile=certfile, - protocol=protocol) - self.logRequests = logRequests - self.serve = False - self.register = register - self.register_introspection_functions() - self.register_function(self.ping) - self.logger.info("service available at %s" % self.url) - self.timeout = timeout - - def _tasks_thread(self): - try: - while self.serve: - try: - if self.instance and hasattr(self.instance, 'do_tasks'): - self.instance.do_tasks() - except: - self.logger.error("Unexpected task failure", exc_info=1) - time.sleep(self.timeout) - except: - self.logger.error("tasks_thread failed", exc_info=1) - - def server_close(self): - SSLServer.server_close(self) - self.logger.info("server_close()") - - def _get_require_auth(self): - return getattr(self.RequestHandlerClass, "require_auth", False) - - def _set_require_auth(self, value): - self.RequestHandlerClass.require_auth = value - require_auth = property(_get_require_auth, _set_require_auth) - - def _get_credentials(self): - try: - return self.RequestHandlerClass.credentials - except AttributeError: - return dict() - - def _set_credentials(self, value): - self.RequestHandlerClass.credentials = value - credentials = property(_get_credentials, _set_credentials) - - def register_instance(self, instance, *args, **kwargs): - XMLRPCDispatcher.register_instance(self, instance, *args, **kwargs) - try: - name = instance.name - except AttributeError: - name = "unknown" - if hasattr(instance, 'plugins'): - for pname, pinst in list(instance.plugins.items()): - for mname in pinst.__rmi__: - xmname = "%s.%s" % (pname, mname) - fn = getattr(pinst, mname) - self.register_function(fn, name=xmname) - self.logger.info("serving %s at %s" % (name, self.url)) - - def serve_forever(self): - """Serve single requests until (self.serve == False).""" - self.serve = True - self.task_thread = threading.Thread(target=self._tasks_thread) - self.task_thread.start() - self.logger.info("serve_forever() [start]") - signal.signal(signal.SIGINT, self._handle_shutdown_signal) - signal.signal(signal.SIGTERM, self._handle_shutdown_signal) - - try: - while self.serve: - try: - self.handle_request() - except socket.timeout: - pass - except select.error: - pass - except: - self.logger.error("Got unexpected error in handle_request", - exc_info=1) - finally: - self.logger.info("serve_forever() [stop]") - - def shutdown(self): - """Signal that automatic service should stop.""" - self.serve = False - - def _handle_shutdown_signal(self, *_): - self.shutdown() - - def ping(self, *args): - """Echo response.""" - self.logger.info("ping(%s)" % (", ".join([repr(arg) for arg in args]))) - return args diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py index 7448855ce..a7e67c956 100644 --- a/src/lib/Bcfg2/Server/Admin/Perf.py +++ b/src/lib/Bcfg2/Server/Admin/Perf.py @@ -2,7 +2,7 @@ import sys import Bcfg2.Options -import Bcfg2.Proxy +import Bcfg2.Client.Proxy import Bcfg2.Server.Admin @@ -22,13 +22,13 @@ class Perf(Bcfg2.Server.Admin.Mode): opts = sys.argv[1:] opts.remove(self.__class__.__name__.lower()) setup.reparse(argv=opts) - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) + proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], + setup['user'], + setup['password'], + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca'], + timeout=setup['timeout']) data = proxy.get_statistics() for key in sorted(data.keys()): output.append((key, ) + diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index 7f9f32816..13077c7ad 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -2,7 +2,7 @@ import sys import Bcfg2.Options -import Bcfg2.Proxy +import Bcfg2.Client.Proxy import Bcfg2.Server.Admin from Bcfg2.Compat import xmlrpclib @@ -23,14 +23,14 @@ class Xcmd(Bcfg2.Server.Admin.Mode): opts = sys.argv[1:] opts.remove(self.__class__.__name__.lower()) setup.reparse(argv=opts) - Bcfg2.Proxy.RetryMethod.max_retries = 1 - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) + Bcfg2.Client.Proxy.RetryMethod.max_retries = 1 + proxy = Bcfg2.Client.Proxy.ComponentProxy(setup['server'], + setup['user'], + setup['password'], + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca'], + timeout=setup['timeout']) if len(setup['args']) == 0: print("Usage: xcmd ") return @@ -46,7 +46,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): return else: raise - except Bcfg2.Proxy.ProxyError: + except Bcfg2.Client.Proxy.ProxyError: err = sys.exc_info()[1] print("Proxy Error: %s" % err) return diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 14b64ff40..663ee6f92 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -4,10 +4,10 @@ import sys import time import socket import daemon -import Bcfg2.Statistics +import Bcfg2.Server.Statistics from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse -from Bcfg2.SSLServer import XMLRPCServer +from Bcfg2.Server.SSLServer import XMLRPCServer from lockfile import LockFailed # pylint: disable=E0611 @@ -25,8 +25,8 @@ class Core(BaseCore): def __init__(self): BaseCore.__init__(self) - #: The :class:`Bcfg2.SSLServer.XMLRPCServer` instance powering - #: this server core + #: The :class:`Bcfg2.Server.SSLServer.XMLRPCServer` instance + #: powering this server core self.server = None daemon_args = dict(uid=self.setup['daemon_uid'], @@ -68,8 +68,9 @@ class Core(BaseCore): try: return method_func(*args) finally: - Bcfg2.Statistics.stats.add_value(method, - time.time() - method_start) + Bcfg2.Server.Statistics.stats.add_value( + method, + time.time() - method_start) except xmlrpclib.Fault: raise except Exception: diff --git a/src/lib/Bcfg2/Server/Cache.py b/src/lib/Bcfg2/Server/Cache.py new file mode 100644 index 000000000..842098eda --- /dev/null +++ b/src/lib/Bcfg2/Server/Cache.py @@ -0,0 +1,14 @@ +""" An implementation of a simple memory-backed cache. Right now this +doesn't provide many features, but more (time-based expiration, etc.) +can be added as necessary. """ + + +class Cache(dict): + """ an implementation of a simple memory-backed cache """ + + def expire(self, key=None): + """ expire all items, or a specific item, from the cache """ + if key is None: + self.clear() + elif key in self: + del self[key] diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py index 79768df20..936279508 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -3,7 +3,7 @@ server. """ import sys import time -import Bcfg2.Statistics +import Bcfg2.Server.Statistics from Bcfg2.Compat import urlparse, xmlrpclib, b64decode from Bcfg2.Server.Core import BaseCore import cherrypy @@ -96,8 +96,8 @@ class Core(BaseCore): try: body = handler(*rpcparams, **params) finally: - Bcfg2.Statistics.stats.add_value(rpcmethod, - time.time() - method_start) + Bcfg2.Server.Statistics.stats.add_value(rpcmethod, + time.time() - method_start) xmlrpcutil.respond(body, 'utf-8', True) return cherrypy.serving.response.body diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index c7797a711..a2c4ee1cb 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -13,14 +13,13 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Logger import Bcfg2.settings -import Bcfg2.Statistics +import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain -from Bcfg2.Cache import Cache +from Bcfg2.Server.Cache import Cache from Bcfg2.Options import get_option_parser, SERVER_FAM_IGNORE from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError, \ - track_statistics +from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError try: import psyco @@ -292,7 +291,7 @@ class BaseCore(object): #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() - #: A :class:`Bcfg2.Cache.Cache` object for caching client + #: A :class:`Bcfg2.Server.Cache.Cache` object for caching client #: metadata self.metadata_cache = Cache() @@ -429,11 +428,11 @@ class BaseCore(object): self.logger.error("%s: Error invoking hook %s: %s" % (plugin, hook, err)) finally: - Bcfg2.Statistics.stats.add_value("%s:client_run_hook:%s" % + Bcfg2.Server.Statistics.stats.add_value("%s:client_run_hook:%s" % (self.__class__.__name__, hook), time.time() - start) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def validate_structures(self, metadata, data): """ Checks the data structures by calling the :func:`Bcfg2.Server.Plugin.interfaces.StructureValidator.validate_structures` @@ -460,7 +459,7 @@ class BaseCore(object): self.logger.error("Plugin %s: unexpected structure validation " "failure" % plugin.name, exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def validate_goals(self, metadata, data): """ Checks that the config matches the goals enforced by :class:`Bcfg2.Server.Plugin.interfaces.GoalValidator` plugins @@ -485,7 +484,7 @@ class BaseCore(object): self.logger.error("Plugin %s: unexpected goal validation " "failure" % plugin.name, exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def GetStructures(self, metadata): """ Get all structures (i.e., bundles) for the given client @@ -502,7 +501,7 @@ class BaseCore(object): (metadata.hostname, ':'.join(missing))) return structures - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def BindStructures(self, structures, metadata, config): """ Given a list of structures (i.e. bundles), bind all the entries in them and add the structures to the config. @@ -522,7 +521,7 @@ class BaseCore(object): except: self.logger.error("error in BindStructure", exc_info=1) - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def BindStructure(self, structure, metadata): """ Bind all elements in a single structure (i.e., bundle). @@ -595,7 +594,7 @@ class BaseCore(object): raise PluginExecutionError("No matching generator: %s:%s" % (entry.tag, entry.get('name'))) finally: - Bcfg2.Statistics.stats.add_value("%s:Bind:%s" % + Bcfg2.Server.Statistics.stats.add_value("%s:Bind:%s" % (self.__class__.__name__, entry.tag), time.time() - start) @@ -744,7 +743,7 @@ class BaseCore(object): % plugin.name, exc_info=1) return result - @track_statistics() + @Bcfg2.Server.Statistics.track_statistics() def build_metadata(self, client_name): """ Build initial client metadata for a client @@ -1094,11 +1093,11 @@ class BaseCore(object): @exposed def get_statistics(self, _): """ Get current statistics about component execution from - :attr:`Bcfg2.Statistics.stats`. + :attr:`Bcfg2.Server.Statistics.stats`. :returns: dict - The statistics data as returned by - :func:`Bcfg2.Statistics.Statistics.display` """ - return Bcfg2.Statistics.stats.display() + :func:`Bcfg2.Server.Statistics.Statistics.display` """ + return Bcfg2.Server.Statistics.stats.display() @exposed def toggle_debug(self, address): diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py new file mode 100755 index 000000000..c931ed2a7 --- /dev/null +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -0,0 +1,195 @@ +""" Bcfg2.Server.Encryption provides a number of convenience methods +for handling encryption in Bcfg2. See :ref:`server-encryption` for +more details. """ + +import os +import Bcfg2.Options +from M2Crypto import Rand +from M2Crypto.EVP import Cipher, EVPError +from Bcfg2.Compat import StringIO, md5, b64encode, b64decode + +#: Constant representing the encryption operation for +#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This +#: makes our code more readable. +ENCRYPT = 1 + +#: Constant representing the decryption operation for +#: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This +#: makes our code more readable. +DECRYPT = 0 + +#: Default initialization vector. For best security, you should use a +#: unique IV for each message. :func:`ssl_encrypt` does this in an +#: automated fashion. +IV = '\0' * 16 + +#: The config file section encryption options and passphrases are +#: stored in +CFG_SECTION = "encryption" + +#: The config option used to store the algorithm +CFG_ALGORITHM = "algorithm" + +#: Default cipher algorithm. To get a full list of valid algorithms, +#: you can run:: +#: +#: openssl list-cipher-algorithms | grep -v ' => ' | \ +#: tr 'A-Z-' 'a-z_' | sort -u +ALGORITHM = Bcfg2.Options.get_option_parser().cfp.get( # pylint: disable=E1103 + CFG_SECTION, + CFG_ALGORITHM, + default="aes_256_cbc").lower().replace("-", "_") + +Rand.rand_seed(os.urandom(1024)) + + +def _cipher_filter(cipher, instr): + """ M2Crypto reads and writes file-like objects, so this uses + StringIO to pass data through it """ + inbuf = StringIO(instr) + outbuf = StringIO() + while 1: + buf = inbuf.read() + if not buf: + break + outbuf.write(cipher.update(buf)) + outbuf.write(cipher.final()) + rv = outbuf.getvalue() + inbuf.close() + outbuf.close() + return rv + + +def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): + """ Encrypt a string with a key. For a higher-level encryption + interface, see :func:`ssl_encrypt`. + + :param plaintext: The plaintext data to encrypt + :type plaintext: string + :param key: The key to encrypt the data with + :type key: string + :param iv: The initialization vector + :type iv: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :param salt: The salt to use + :type salt: string + :returns: string - The decrypted data + """ + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) + return _cipher_filter(cipher, plaintext) + + +def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): + """ Decrypt a string with a key. For a higher-level decryption + interface, see :func:`ssl_decrypt`. + + :param crypted: The raw binary encrypted data + :type crypted: string + :param key: The encryption key to decrypt with + :type key: string + :param iv: The initialization vector + :type iv: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + """ + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) + return _cipher_filter(cipher, crypted) + + +def ssl_decrypt(data, passwd, algorithm=ALGORITHM): + """ Decrypt openssl-encrypted data. This can decrypt data + encrypted by :func:`ssl_encrypt`, or ``openssl enc``. It performs + a base64 decode first if the data is base64 encoded, and + automatically determines the salt and initialization vector (both + of which are embedded in the encrypted data). + + :param data: The encrypted data (either base64-encoded or raw + binary) to decrypt + :type data: string + :param passwd: The password to use to decrypt the data + :type passwd: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + """ + # base64-decode the data + data = b64decode(data) + salt = data[8:16] + hashes = [md5(passwd + salt).digest()] + for i in range(1, 3): + hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + key = hashes[0] + hashes[1] + iv = hashes[2] + + return str_decrypt(data[16:], key=key, iv=iv, algorithm=algorithm) + + +def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): + """ Encrypt data in a format that is openssl compatible. + + :param plaintext: The plaintext data to encrypt + :type plaintext: string + :param passwd: The password to use to encrypt the data + :type passwd: string + :param algorithm: The cipher algorithm to use + :type algorithm: string + :param salt: The salt to use. If none is provided, one will be + randomly generated. + :type salt: bytes + :returns: string - The base64-encoded, salted, encrypted string. + The string includes a trailing newline to make it fully + compatible with openssl command-line tools. + """ + if salt is None: + salt = Rand.rand_bytes(8) + + hashes = [md5(passwd + salt).digest()] + for i in range(1, 3): + hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + key = hashes[0] + hashes[1] + iv = hashes[2] + + crypted = str_encrypt(plaintext, key=key, salt=salt, iv=iv, + algorithm=algorithm) + return b64encode("Salted__" + salt + crypted) + "\n" + + +def get_passphrases(): + """ Get all candidate encryption passphrases from the config file. + + :returns: dict - a dict of ````: ```` + """ + setup = Bcfg2.Options.get_option_parser() + if setup.cfp.has_section(CFG_SECTION): + return dict([(o, setup.cfp.get(CFG_SECTION, o)) + for o in setup.cfp.options(CFG_SECTION) + if o != CFG_ALGORITHM]) + else: + return dict() + + +def bruteforce_decrypt(crypted, passphrases=None, algorithm=ALGORITHM): + """ Convenience method to decrypt the given encrypted string by + trying the given passphrases or all passphrases (as returned by + :func:`get_passphrases`) sequentially until one is found that + works. + + :param crypted: The data to decrypt + :type crypted: string + :param passphrases: The passphrases to try. + :type passphrases: list + :param algorithm: The cipher algorithm to use + :type algorithm: string + :returns: string - The decrypted data + :raises: :class:`M2Crypto.EVP.EVPError`, if the data cannot be decrypted + """ + if passphrases is None: + passphrases = get_passphrases().values() + for passwd in passphrases: + try: + return ssl_decrypt(crypted, passwd, algorithm=algorithm) + except EVPError: + pass + raise EVPError("Failed to decrypt") diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 871b5eb4e..bc82ee6c2 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -4,14 +4,12 @@ import os import re import sys import copy -import time import genshi import logging import operator import lxml.etree import Bcfg2.Server import Bcfg2.Options -import Bcfg2.Statistics import Bcfg2.Server.FileMonitor from Bcfg2.Compat import CmpMixin, wraps from Bcfg2.Server.Plugin.base import Debuggable, Plugin @@ -20,7 +18,7 @@ from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ PluginExecutionError try: - import Bcfg2.Encryption + import Bcfg2.Server.Encryption HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -94,40 +92,6 @@ def bind_info(entry, metadata, infoxml=None, default=None): entry.set(attr, val) -class track_statistics(object): # pylint: disable=C0103 - """ Decorator that tracks execution time for the given - :class:`Plugin` method with :mod:`Bcfg2.Statistics` for reporting - via ``bcfg2-admin perf`` """ - - def __init__(self, name=None): - """ - :param name: The name under which statistics for this function - will be tracked. By default, the name will be - the name of the function concatenated with the - name of the class the function is a member of. - :type name: string - """ - # if this is None, it will be set later during __call_ - self.name = name - - def __call__(self, func): - if self.name is None: - self.name = func.__name__ - - @wraps(func) - def inner(obj, *args, **kwargs): - """ The decorated function """ - name = "%s:%s" % (obj.__class__.__name__, self.name) - - start = time.time() - try: - return func(obj, *args, **kwargs) - finally: - Bcfg2.Statistics.stats.add_value(name, time.time() - start) - - return inner - - class DatabaseBacked(Plugin): """ Provides capabilities for a plugin to read and write to a database. @@ -620,8 +584,8 @@ class StructFile(XMLFileBacked, Debuggable): if self.encryption and HAS_CRYPTO: strict = self.xdata.get( "decrypt", - self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, "decrypt", - default="strict")) == "strict" + self.setup.cfp.get(Bcfg2.Server.Encryption.CFG_SECTION, + "decrypt", default="strict")) == "strict" for el in self.xdata.xpath("//*[@encrypted]"): try: el.text = self._decrypt(el).encode('ascii', @@ -629,7 +593,7 @@ class StructFile(XMLFileBacked, Debuggable): except UnicodeDecodeError: self.logger.info("%s: Decrypted %s to gibberish, skipping" % (self.name, el.tag)) - except Bcfg2.Encryption.EVPError: + except Bcfg2.Server.Encryption.EVPError: msg = "Failed to decrypt %s element in %s" % (el.tag, self.name) if strict: @@ -642,19 +606,20 @@ class StructFile(XMLFileBacked, Debuggable): """ Decrypt a single encrypted properties file element """ if not element.text or not element.text.strip(): return - passes = Bcfg2.Encryption.get_passphrases() + passes = Bcfg2.Server.Encryption.get_passphrases() try: passphrase = passes[element.get("encrypted")] try: - return Bcfg2.Encryption.ssl_decrypt(element.text, passphrase) - except Bcfg2.Encryption.EVPError: + return Bcfg2.Server.Encryption.ssl_decrypt(element.text, + passphrase) + except Bcfg2.Server.Encryption.EVPError: # error is raised below pass except KeyError: # bruteforce_decrypt raises an EVPError with a sensible # error message, so we just let it propagate up the stack - return Bcfg2.Encryption.bruteforce_decrypt(element.text) - raise Bcfg2.Encryption.EVPError("Failed to decrypt") + return Bcfg2.Server.Encryption.bruteforce_decrypt(element.text) + raise Bcfg2.Server.Encryption.EVPError("Failed to decrypt") def _include_element(self, item, metadata): """ determine if an XML element matches the metadata """ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 3b3b95ff5..516eba2f6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -4,7 +4,7 @@ from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt, EVPError + from Bcfg2.Server.Encryption import bruteforce_decrypt, EVPError HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index 215e4c1f1..a285eecd8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -6,7 +6,7 @@ from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator try: - from Bcfg2.Encryption import bruteforce_decrypt + from Bcfg2.Server.Encryption import bruteforce_decrypt HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -21,7 +21,7 @@ except ImportError: class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt the data on the fly as it's read in using - :func:`Bcfg2.Encryption.bruteforce_decrypt` """ + :func:`Bcfg2.Server.Encryption.bruteforce_decrypt` """ def _instantiate(self, cls, fileobj, filepath, filename, encoding=None): plaintext = StringIO(bruteforce_decrypt(fileobj.read())) return TemplateLoader._instantiate(self, cls, plaintext, filepath, diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 4d6639e4d..1711cc655 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -9,7 +9,7 @@ from Bcfg2.Server.Plugin import StructFile from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator try: - import Bcfg2.Encryption + from Bcfg2.Server.Encryption import get_passphrases, ssl_encrypt HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False @@ -50,9 +50,8 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if (HAS_CRYPTO and self.setup.cfp.has_section("sshkeys") and self.setup.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Encryption.get_passphrases()[self.setup.cfp.get( - "sshkeys", - "passphrase")] + return get_passphrases()[self.setup.cfp.get("sshkeys", + "passphrase")] return None def handle_event(self, event): @@ -198,8 +197,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): privkey = open(filename).read() if HAS_CRYPTO and self.passphrase: self.debug_log("Cfg: Encrypting key data at %s" % filename) - privkey = Bcfg2.Encryption.ssl_encrypt(privkey, - self.passphrase) + privkey = ssl_encrypt(privkey, self.passphrase) specificity['ext'] = '.crypt' self.write_data(privkey, **specificity) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index cf4234ed0..3ad64b242 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -81,6 +81,7 @@ import Bcfg2.Server.Plugin from Bcfg2.Server.FileMonitor import get_fam from Bcfg2.Options import get_option_parser from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Server.Statistics import track_statistics LOGGER = logging.getLogger(__name__) @@ -213,7 +214,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): cachefiles.add(source.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ Given a list of package group names, return a dict of ``: ``. This method is provided @@ -234,7 +235,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): rv[group] = self.get_group(group, ptype) return rv - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_group(self, group, ptype=None): """ Get the list of packages of the given type in a package group. @@ -469,7 +470,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): """ return list(complete.difference(initial)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 876ee6090..782e077bb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -5,6 +5,7 @@ import os import sys import Bcfg2.Server.Plugin from Bcfg2.Options import get_option_parser +from Bcfg2.Server.Statistics import track_statistics from Bcfg2.Server.Plugins.Packages.Source import SourceInitError @@ -109,7 +110,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, load its data. """ return sorted(list(self.parsed)) == sorted(self.extras) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] @@ -122,7 +123,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, ``Index`` is responsible for calling :func:`source_from_xml` for each ``Source`` tag in each file. """ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def source_from_xml(self, xsource): """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 7a8b2827a..5cf90e188 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -54,6 +54,7 @@ from Bcfg2.Options import get_option_parser from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ urlopen, cPickle, md5 +from Bcfg2.Server.Statistics import track_statistics def fetch_url(url): @@ -320,7 +321,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.essentialpkgs), cache, 2) cache.close() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def setup_data(self, force_update=False): """ Perform all data fetching and setup tasks. For most backends, this involves downloading all metadata from the diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index c5d59eb24..4057ed230 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -69,6 +69,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url +from Bcfg2.Server.Statistics import track_statistics LOGGER = logging.getLogger(__name__) @@ -363,7 +364,7 @@ class YumCollection(Collection): cachefiles.add(self.cachefile) return list(cachefiles) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def write_config(self): """ Write the server-side config file to :attr:`cfgfile` based on the data from :func:`get_config`""" @@ -465,7 +466,7 @@ class YumCollection(Collection): return "# This config was generated automatically by the Bcfg2 " \ "Packages plugin\n\n" + buf.getvalue() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def build_extra_structures(self, independent): """ Add additional entries to the ```` section of the final configuration. This adds several kinds of @@ -572,7 +573,7 @@ class YumCollection(Collection): name=self.pulp_cert_set.certpath) self.pulp_cert_set.bind_entry(crt, self.metadata) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _get_pulp_consumer(self, consumerapi=None): """ Get a Pulp consumer object for the client. @@ -601,7 +602,7 @@ class YumCollection(Collection): "%s" % err) return consumer - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _add_gpg_instances(self, keyentry, localkey, remotekey, keydata=None): """ Add GPG keys instances to a ``Package`` entry. This is called from :func:`build_extra_structures` to add GPG keys to @@ -644,7 +645,7 @@ class YumCollection(Collection): self.logger.error("Packages: Could not read GPG key %s: %s" % (localkey, err)) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_groups(self, grouplist): """ If using the yum libraries, given a list of package group names, return a dict of ``: ``. @@ -806,7 +807,7 @@ class YumCollection(Collection): new.append(pkg) return new - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def complete(self, packagelist): """ Build a complete list of all packages and their dependencies. @@ -846,7 +847,7 @@ class YumCollection(Collection): else: return set(), set() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def call_helper(self, command, inputdata=None): """ Make a call to :ref:`bcfg2-yum-helper`. The yum libs have horrific memory leaks, so apparently the right way to get @@ -1067,7 +1068,7 @@ class YumSource(Source): self.file_to_arch[self.escape_url(fullurl)] = arch return urls - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def read_files(self): """ When using the builtin yum parser, read and parse locally downloaded metadata files. This diverges from the stock @@ -1109,7 +1110,7 @@ class YumSource(Source): self.packages[key].difference(self.packages['global']) self.save_state() - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_filelist(self, data, arch): """ parse filelists.xml.gz data """ if arch not in self.filemap: @@ -1123,7 +1124,7 @@ class YumSource(Source): self.filemap[arch][fentry.text] = \ set([pkg.get('name')]) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def parse_primary(self, data, arch): """ parse primary.xml.gz data """ if arch not in self.packages: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 169dcd588..ca7b7c530 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -13,6 +13,7 @@ from Bcfg2.Compat import ConfigParser, urlopen, HTTPError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Statistics import track_statistics #: The default path for generated yum configs YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" @@ -235,7 +236,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True return False - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def validate_structures(self, metadata, structures): """ Do the real work of Packages. This does two things: @@ -270,7 +271,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection.build_extra_structures(indep) structures.append(indep) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def _build_packages(self, metadata, independent, structures, collection=None): """ Perform dependency resolution and build the complete list @@ -341,7 +342,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, newpkgs.sort() collection.packages_to_entry(newpkgs, independent) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Refresh(self): """ Packages.Refresh() => True|False @@ -349,7 +350,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config(force_update=True) return True - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def Reload(self): """ Packages.Refresh() => True|False @@ -445,7 +446,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if kfile not in keyfiles: os.unlink(kfile) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def get_collection(self, metadata): """ Get a :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 7b8ef2209..2ea5088de 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -10,6 +10,7 @@ import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor +from Bcfg2.Server.Statistics import track_statistics try: from django.db import models @@ -190,7 +191,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.load_data() __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def write_data(self, client): """ Write probe data out for use with bcfg2-info """ if self._use_db: @@ -293,12 +294,12 @@ class Probes(Bcfg2.Server.Plugin.Probing, self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def GetProbes(self, meta): return self.probes.get_probe_data(meta) GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ - @Bcfg2.Server.Plugin.track_statistics() + @track_statistics() def ReceiveData(self, client, datalist): if self.core.metadata_cache_mode in ['cautious', 'aggressive']: if client.hostname in self.cgroups: diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index b7ede594c..762f9f8f0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -10,11 +10,6 @@ import lxml.etree from Bcfg2.Options import get_option_parser import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError -try: - import Bcfg2.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False try: import json diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py new file mode 100644 index 000000000..eeaeb9516 --- /dev/null +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -0,0 +1,452 @@ +""" Bcfg2 SSL server used by the builtin server core +(:mod:`Bcfg2.Server.BuiltinCore`). This needs to be documented +better. """ + +import os +import sys +import socket +import select +import signal +import logging +import ssl +import threading +import time +from Bcfg2.Compat import xmlrpclib, SimpleXMLRPCServer, SocketServer, \ + b64decode + + +class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): + """ An XML-RPC dispatcher. """ + + def __init__(self, allow_none, encoding): + try: + SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, + allow_none, + encoding) + except: + # Python 2.4? + SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self) + + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) + self.allow_none = allow_none + self.encoding = encoding + + def _marshaled_dispatch(self, address, data): + params, method = xmlrpclib.loads(data) + try: + if '.' not in method: + params = (address, ) + params + response = self.instance._dispatch(method, params, self.funcs) + # py3k compatibility + if type(response) not in [bool, str, list, dict]: + response = (response.decode('utf-8'), ) + else: + response = (response, ) + raw_response = xmlrpclib.dumps(response, methodresponse=1, + allow_none=self.allow_none, + encoding=self.encoding) + except xmlrpclib.Fault: + fault = sys.exc_info()[1] + raw_response = xmlrpclib.dumps(fault, + allow_none=self.allow_none, + encoding=self.encoding) + except: + self.logger.error("Unexpected handler error", exc_info=1) + # report exception back to server + raw_response = xmlrpclib.dumps( + xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)), + allow_none=self.allow_none, encoding=self.encoding) + return raw_response + + +class SSLServer(SocketServer.TCPServer, object): + """ TCP server supporting SSL encryption. """ + allow_reuse_address = True + + def __init__(self, listen_all, server_address, RequestHandlerClass, + keyfile=None, certfile=None, reqCert=False, ca=None, + timeout=None, protocol='xmlrpc/ssl'): + """ + :param listen_all: Listen on all interfaces + :type listen_all: bool + :param server_address: Address to bind to the server + :param RequestHandlerClass: Request handler used by TCP server + :param keyfile: Full path to SSL encryption key file + :type keyfile: string + :param certfile: Full path to SSL certificate file + :type certfile: string + :param reqCert: Require client to present certificate + :type reqCert: bool + :param ca: Full path to SSL CA that signed the key and cert + :type ca: string + :param timeout: Timeout for non-blocking request handling + :param protocol: The protocol to serve. Supported values are + ``xmlrpc/ssl`` and ``xmlrpc/tlsv1``. + :type protocol: string + """ + # check whether or not we should listen on all interfaces + if listen_all: + listen_address = ('', server_address[1]) + else: + listen_address = (server_address[0], server_address[1]) + + # check for IPv6 address + if ':' in server_address[0]: + self.address_family = socket.AF_INET6 + + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) + + try: + SocketServer.TCPServer.__init__(self, listen_address, + RequestHandlerClass) + except socket.gaierror: + e = sys.exc_info()[1] + self.logger.error("Failed to bind to socket: %s" % e) + raise + except socket.error: + self.logger.error("Failed to bind to socket") + raise + + self.timeout = timeout + self.socket.settimeout(timeout) + self.keyfile = keyfile + if (keyfile is not None and + (keyfile == False or + not os.path.exists(keyfile) or + not os.access(keyfile, os.R_OK))): + msg = "Keyfile %s does not exist or is not readable" % keyfile + self.logger.error(msg) + raise Exception(msg) + self.certfile = certfile + if (certfile is not None and + (certfile == False or + not os.path.exists(certfile) or + not os.access(certfile, os.R_OK))): + msg = "Certfile %s does not exist or is not readable" % certfile + self.logger.error(msg) + raise Exception(msg) + self.ca = ca + if (ca is not None and + (ca == False or + not os.path.exists(ca) or + not os.access(ca, os.R_OK))): + msg = "CA %s does not exist or is not readable" % ca + self.logger.error(msg) + raise Exception(msg) + self.reqCert = reqCert + if ca and certfile: + self.mode = ssl.CERT_OPTIONAL + else: + self.mode = ssl.CERT_NONE + if protocol == 'xmlrpc/ssl': + self.ssl_protocol = ssl.PROTOCOL_SSLv23 + elif protocol == 'xmlrpc/tlsv1': + self.ssl_protocol = ssl.PROTOCOL_TLSv1 + else: + self.logger.error("Unknown protocol %s" % (protocol)) + raise Exception("unknown protocol %s" % protocol) + + def get_request(self): + (sock, sockinfo) = self.socket.accept() + sock.settimeout(self.timeout) # pylint: disable=E1101 + sslsock = ssl.wrap_socket(sock, + server_side=True, + certfile=self.certfile, + keyfile=self.keyfile, + cert_reqs=self.mode, + ca_certs=self.ca, + ssl_version=self.ssl_protocol) + return sslsock, sockinfo + + def close_request(self, request): + try: + request.unwrap() + except: + pass + try: + request.close() + except: + pass + + def _get_url(self): + port = self.socket.getsockname()[1] + hostname = socket.gethostname() + protocol = "https" + return "%s://%s:%i" % (protocol, hostname, port) + url = property(_get_url) + + +class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): + """ XML-RPC request handler. + + Adds support for HTTP authentication. + """ + + def __init__(self, *args, **kwargs): + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.__init__(self, *args, + **kwargs) + self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__, + self.__class__.__name__)) + + def authenticate(self): + try: + header = self.headers['Authorization'] + except KeyError: + self.logger.error("No authentication data presented") + return False + auth_content = b64decode(header.split()[1]) + try: + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(bytes(":", encoding='utf-8')) + password = pw.decode('utf-8') + except ValueError: + username = auth_content + password = "" + cert = self.request.getpeercert() + client_address = self.request.getpeername() + return self.server.instance.authenticate(cert, username, + password, client_address) + + def parse_request(self): + """Extends parse_request. + + Optionally check HTTP authentication when parsing. + """ + if not SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.parse_request(self): + return False + try: + if not self.authenticate(): + self.logger.error("Authentication Failure") + self.send_error(401, self.responses[401][0]) + return False + except: # pylint: disable=W0702 + self.logger.error("Unexpected Authentication Failure", exc_info=1) + self.send_error(401, self.responses[401][0]) + return False + return True + + ### need to override do_POST here + def do_POST(self): + try: + max_chunk_size = 10 * 1024 * 1024 + size_remaining = int(self.headers["content-length"]) + L = [] + while size_remaining: + try: + select.select([self.rfile.fileno()], [], [], 3) + except select.error: + print("got select timeout") + raise + chunk_size = min(size_remaining, max_chunk_size) + L.append(self.rfile.read(chunk_size).decode('utf-8')) + size_remaining -= len(L[-1]) + data = ''.join(L) + response = self.server._marshaled_dispatch(self.client_address, + data) + if sys.hexversion >= 0x03000000: + response = response.encode('utf-8') + except: # pylint: disable=W0702 + try: + self.send_response(500) + self.end_headers() + except: + (etype, msg) = sys.exc_info()[:2] + self.logger.error("Error sending 500 response (%s): %s" % + (etype.__name__, msg)) + raise + else: + # got a valid XML RPC response + try: + self.send_response(200) + self.send_header("Content-type", "text/xml") + self.send_header("Content-length", str(len(response))) + self.end_headers() + failcount = 0 + while True: + try: + # If we hit SSL3_WRITE_PENDING here try to resend. + self.wfile.write(response) + break + except ssl.SSLError: + e = sys.exc_info()[1] + if str(e).find("SSL3_WRITE_PENDING") < 0: + raise + self.logger.error("SSL3_WRITE_PENDING") + failcount += 1 + if failcount < 5: + continue + raise + except socket.error: + err = sys.exc_info()[1] + if err[0] == 32: + self.logger.warning("Connection dropped from %s" % + self.client_address[0]) + elif err[0] == 104: + self.logger.warning("Connection reset by peer: %s" % + self.client_address[0]) + else: + self.logger.warning("Socket error sending response to %s: " + "%s" % (self.client_address[0], err)) + except ssl.SSLError: + err = sys.exc_info()[1] + self.logger.warning("SSLError handling client %s: %s" % + (self.client_address[0], err)) + except: + etype, err = sys.exc_info()[:2] + self.logger.error("Unknown error sending response to %s: " + "%s (%s)" % + (self.client_address[0], err, + etype.__name__)) + + def finish(self): + # shut down the connection + if not self.wfile.closed: + try: + self.wfile.flush() + self.wfile.close() + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) + self.rfile.close() + + +class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, + XMLRPCDispatcher, object): + """ Component XMLRPCServer. """ + + def __init__(self, listen_all, server_address, RequestHandlerClass=None, + keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', + timeout=10, logRequests=False, + register=True, allow_none=True, encoding=None): + """ + :param listen_all: Listen on all interfaces + :type listen_all: bool + :param server_address: Address to bind to the server + :param RequestHandlerClass: request handler used by TCP server + :param keyfile: Full path to SSL encryption key file + :type keyfile: string + :param certfile: Full path to SSL certificate file + :type certfile: string + :param ca: Full path to SSL CA that signed the key and cert + :type ca: string + :param logRequests: Log all requests + :type logRequests: bool + :param register: Presence should be reported to service-location + :type register: bool + :param allow_none: Allow None values in XML-RPC + :type allow_non: bool + :param encoding: Encoding to use for XML-RPC + """ + + XMLRPCDispatcher.__init__(self, allow_none, encoding) + + if not RequestHandlerClass: + # pylint: disable=E0102 + class RequestHandlerClass(XMLRPCRequestHandler): + """A subclassed request handler to prevent + class-attribute conflicts.""" + # pylint: enable=E0102 + + SSLServer.__init__(self, + listen_all, + server_address, + RequestHandlerClass, + ca=ca, + timeout=timeout, + keyfile=keyfile, + certfile=certfile, + protocol=protocol) + self.logRequests = logRequests + self.serve = False + self.register = register + self.register_introspection_functions() + self.register_function(self.ping) + self.logger.info("service available at %s" % self.url) + self.timeout = timeout + + def _tasks_thread(self): + try: + while self.serve: + try: + if self.instance and hasattr(self.instance, 'do_tasks'): + self.instance.do_tasks() + except: + self.logger.error("Unexpected task failure", exc_info=1) + time.sleep(self.timeout) + except: + self.logger.error("tasks_thread failed", exc_info=1) + + def server_close(self): + SSLServer.server_close(self) + self.logger.info("server_close()") + + def _get_require_auth(self): + return getattr(self.RequestHandlerClass, "require_auth", False) + + def _set_require_auth(self, value): + self.RequestHandlerClass.require_auth = value + require_auth = property(_get_require_auth, _set_require_auth) + + def _get_credentials(self): + try: + return self.RequestHandlerClass.credentials + except AttributeError: + return dict() + + def _set_credentials(self, value): + self.RequestHandlerClass.credentials = value + credentials = property(_get_credentials, _set_credentials) + + def register_instance(self, instance, *args, **kwargs): + XMLRPCDispatcher.register_instance(self, instance, *args, **kwargs) + try: + name = instance.name + except AttributeError: + name = "unknown" + if hasattr(instance, 'plugins'): + for pname, pinst in list(instance.plugins.items()): + for mname in pinst.__rmi__: + xmname = "%s.%s" % (pname, mname) + fn = getattr(pinst, mname) + self.register_function(fn, name=xmname) + self.logger.info("serving %s at %s" % (name, self.url)) + + def serve_forever(self): + """Serve single requests until (self.serve == False).""" + self.serve = True + self.task_thread = threading.Thread(target=self._tasks_thread) + self.task_thread.start() + self.logger.info("serve_forever() [start]") + signal.signal(signal.SIGINT, self._handle_shutdown_signal) + signal.signal(signal.SIGTERM, self._handle_shutdown_signal) + + try: + while self.serve: + try: + self.handle_request() + except socket.timeout: + pass + except select.error: + pass + except: + self.logger.error("Got unexpected error in handle_request", + exc_info=1) + finally: + self.logger.info("serve_forever() [stop]") + + def shutdown(self): + """Signal that automatic service should stop.""" + self.serve = False + + def _handle_shutdown_signal(self, *_): + self.shutdown() + + def ping(self, *args): + """Echo response.""" + self.logger.info("ping(%s)" % (", ".join([repr(arg) for arg in args]))) + return args diff --git a/src/lib/Bcfg2/Server/Statistics.py b/src/lib/Bcfg2/Server/Statistics.py new file mode 100644 index 000000000..8a6ff54c4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Statistics.py @@ -0,0 +1,118 @@ +""" Module for tracking execution time statistics from the Bcfg2 +server core. This data is exposed by +:func:`Bcfg2.Server.Core.BaseCore.get_statistics`.""" + +import time +from Bcfg2.Compat import wraps + +class Statistic(object): + """ A single named statistic, tracking minimum, maximum, and + average execution time, and number of invocations. """ + + def __init__(self, name, initial_value): + """ + :param name: The name of this statistic + :type name: string + :param initial_value: The initial value to be added to this + statistic + :type initial_value: int or float + """ + self.name = name + self.min = float(initial_value) + self.max = float(initial_value) + self.ave = float(initial_value) + self.count = 1 + + def add_value(self, value): + """ Add a value to the statistic, recalculating the various + metrics. + + :param value: The value to add to this statistic + :type value: int or float + """ + self.min = min(self.min, value) + self.max = max(self.max, value) + self.ave = (((self.ave * (self.count - 1)) + value) / self.count) + self.count += 1 + + def get_value(self): + """ Get a tuple of all the stats tracked on this named item. + The tuple is in the format:: + + (, (min, max, average, number of values)) + + This makes it very easy to cast to a dict in + :func:`Statistics.display`. + + :returns: tuple + """ + return (self.name, (self.min, self.max, self.ave, self.count)) + + +class Statistics(object): + """ A collection of named :class:`Statistic` objects. """ + + def __init__(self): + self.data = dict() + + def add_value(self, name, value): + """ Add a value to the named :class:`Statistic`. This just + proxies to :func:`Statistic.add_value` or the + :class:`Statistic` constructor as appropriate. + + :param name: The name of the :class:`Statistic` to add the + value to + :type name: string + :param value: The value to add to the Statistic + :type value: int or float + """ + if name not in self.data: + self.data[name] = Statistic(name, value) + else: + self.data[name].add_value(value) + + def display(self): + """ Return a dict of all :class:`Statistic` object values. + Keys are the statistic names, and values are tuples of the + statistic metrics as returned by + :func:`Statistic.get_value`. """ + return dict([value.get_value() for value in list(self.data.values())]) + + +#: A module-level :class:`Statistics` objects used to track all +#: execution time metrics for the server. +stats = Statistics() # pylint: disable=C0103 + + +class track_statistics(object): # pylint: disable=C0103 + """ Decorator that tracks execution time for the given method with + :mod:`Bcfg2.Server.Statistics` for reporting via ``bcfg2-admin + perf`` """ + + def __init__(self, name=None): + """ + :param name: The name under which statistics for this function + will be tracked. By default, the name will be + the name of the function concatenated with the + name of the class the function is a member of. + :type name: string + """ + # if this is None, it will be set later during __call_ + self.name = name + + def __call__(self, func): + if self.name is None: + self.name = func.__name__ + + @wraps(func) + def inner(obj, *args, **kwargs): + """ The decorated function """ + name = "%s:%s" % (obj.__class__.__name__, self.name) + + start = time.time() + try: + return func(obj, *args, **kwargs) + finally: + stats.add_value(name, time.time() - start) + + return inner diff --git a/src/lib/Bcfg2/Statistics.py b/src/lib/Bcfg2/Statistics.py deleted file mode 100644 index a869b03cd..000000000 --- a/src/lib/Bcfg2/Statistics.py +++ /dev/null @@ -1,82 +0,0 @@ -""" Module for tracking execution time statistics from the Bcfg2 -server core. This data is exposed by -:func:`Bcfg2.Server.Core.BaseCore.get_statistics`.""" - - -class Statistic(object): - """ A single named statistic, tracking minimum, maximum, and - average execution time, and number of invocations. """ - - def __init__(self, name, initial_value): - """ - :param name: The name of this statistic - :type name: string - :param initial_value: The initial value to be added to this - statistic - :type initial_value: int or float - """ - self.name = name - self.min = float(initial_value) - self.max = float(initial_value) - self.ave = float(initial_value) - self.count = 1 - - def add_value(self, value): - """ Add a value to the statistic, recalculating the various - metrics. - - :param value: The value to add to this statistic - :type value: int or float - """ - self.min = min(self.min, value) - self.max = max(self.max, value) - self.ave = (((self.ave * (self.count - 1)) + value) / self.count) - self.count += 1 - - def get_value(self): - """ Get a tuple of all the stats tracked on this named item. - The tuple is in the format:: - - (, (min, max, average, number of values)) - - This makes it very easy to cast to a dict in - :func:`Statistics.display`. - - :returns: tuple - """ - return (self.name, (self.min, self.max, self.ave, self.count)) - - -class Statistics(object): - """ A collection of named :class:`Statistic` objects. """ - - def __init__(self): - self.data = dict() - - def add_value(self, name, value): - """ Add a value to the named :class:`Statistic`. This just - proxies to :func:`Statistic.add_value` or the - :class:`Statistic` constructor as appropriate. - - :param name: The name of the :class:`Statistic` to add the - value to - :type name: string - :param value: The value to add to the Statistic - :type value: int or float - """ - if name not in self.data: - self.data[name] = Statistic(name, value) - else: - self.data[name].add_value(value) - - def display(self): - """ Return a dict of all :class:`Statistic` object values. - Keys are the statistic names, and values are tuples of the - statistic metrics as returned by - :func:`Statistic.get_value`. """ - return dict([value.get_value() for value in list(self.data.values())]) - - -#: A module-level :class:`Statistics` objects used to track all -#: execution time metrics for the server. -stats = Statistics() # pylint: disable=C0103 diff --git a/src/lib/Bcfg2/__init__.py b/src/lib/Bcfg2/__init__.py deleted file mode 100644 index 3fe2a0d75..000000000 --- a/src/lib/Bcfg2/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Base modules definition.""" - -__all__ = ['Server', 'Client', 'Logger', 'Options', 'Proxy', 'Statistics'] -- cgit v1.2.3-1-g7c22