From 91ad257efa9dd1a0a0db5f68500621529bf02179 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 20 Aug 2012 15:44:23 -0400 Subject: fixed broken XML-RPC retries, made delay configurable --- src/lib/Bcfg2/Options.py | 7 +++++ src/lib/Bcfg2/Proxy.py | 67 +++++++++++++++++++++++++++--------------------- src/sbin/bcfg2 | 19 ++++++++------ 3 files changed, 56 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 028e878a0..1883bc222 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -559,6 +559,12 @@ CLIENT_RETRIES = \ cmd='-R', odesc='', cf=('communication', 'retries')) +CLIENT_RETRY_DELAY = \ + Option('The time in seconds to wait between retries', + default='1', + cmd='-y', + odesc='', + cf=('communication', 'retry_delay')) CLIENT_DRYRUN = \ Option('Do not actually change the system', default=False, @@ -983,6 +989,7 @@ CLIENT_COMMON_OPTIONS = \ user=CLIENT_USER, password=SERVER_PASSWORD, retries=CLIENT_RETRIES, + retry_delay=CLIENT_RETRY_DELAY, kevlar=CLIENT_KEVLAR, omit_lock_check=OMIT_LOCK_CHECK, decision=CLIENT_DLIST, diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index ddeef4570..220b89b5f 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -59,50 +59,58 @@ class CertificateError(Exception): return ("Got unallowed commonName %s from server" % self.commonName) +_orig_Method = xmlrpclib._Method class RetryMethod(xmlrpclib._Method): """Method with error handling and retries built in.""" log = logging.getLogger('xmlrpc') - max_retries = 4 + max_retries = 3 + retry_delay = 1 def __call__(self, *args): for retry in range(self.max_retries): + if retry >= self.max_retries - 1: + final = True + else: + final = False + msg = None try: - return xmlrpclib._Method.__call__(self, *args) + return _orig_Method.__call__(self, *args) except xmlrpclib.ProtocolError: err = sys.exc_info()[1] - self.log.error("Server failure: Protocol Error: %s %s" % \ - (err.errcode, err.errmsg)) - raise xmlrpclib.Fault(20, "Server Failure") + msg = "Server failure: Protocol Error: %s %s" % \ + (err.errcode, err.errmsg) except xmlrpclib.Fault: - raise + msg = sys.exc_info()[1] except socket.error: err = sys.exc_info()[1] if hasattr(err, 'errno') and err.errno == 336265218: - self.log.error("SSL Key error") - break - if hasattr(err, 'errno') and err.errno == 185090050: - self.log.error("SSL CA error") - break - if retry == 3: - self.log.error("Server failure: %s" % err) - raise xmlrpclib.Fault(20, err) + msg = "SSL Key error: %s" % err + elif hasattr(err, 'errno') and err.errno == 185090050: + msg = "SSL CA error: %s" % err + elif final: + msg = "Server failure: %s" % err except CertificateError: - ce = sys.exc_info()[1] - self.log.error("Got unallowed commonName %s from server" \ - % ce.commonName) - break + err = sys.exc_info()[1] + msg = "Got unallowed commonName %s from server" % err.commonName except KeyError: - self.log.error("Server disallowed connection") - break + err = sys.exc_info()[1] + msg = "Server disallowed connection: %s" % err + except ProxyError: + err = sys.exc_info()[1] + msg = err except: - self.log.error("Unknown failure", exc_info=1) - break - time.sleep(0.5) - raise xmlrpclib.Fault(20, "Server Failure") + err = sys.exc_info()[1] + msg = "Unknown failure: %s" % err + if msg: + if final: + self.log.error(msg) + raise ProxyError(msg) + else: + self.log.info(msg) + time.sleep(self.retry_delay) -# sorry jon -_Method = RetryMethod +xmlrpclib._Method = RetryMethod class SSLHTTPConnection(httplib.HTTPConnection): @@ -345,9 +353,8 @@ class XMLRPCTransport(xmlrpclib.Transport): return u.close() -def ComponentProxy(url, user=None, password=None, - key=None, cert=None, ca=None, - allowedServerCNs=None, timeout=90): +def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, + allowedServerCNs=None, timeout=90, retries=3, delay=1): """Constructs proxies to components. @@ -357,6 +364,8 @@ def ComponentProxy(url, user=None, password=None, Additional arguments are passed to the ServerProxy constructor. """ + xmlrpclib._Method.max_retries = retries + xmlrpclib._Method.retry_delay = delay if user and password: method, path = urlparse(url)[:2] diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index d757ca08a..b3b20eb78 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -140,14 +140,17 @@ class Client: return(1) else: # retrieve config from server - proxy = Bcfg2.Proxy.ComponentProxy(self.setup['server'], - self.setup['user'], - self.setup['password'], - key=self.setup['key'], - cert=self.setup['certificate'], - ca=self.setup['ca'], - allowedServerCNs=self.setup['serverCN'], - timeout=self.setup['timeout']) + proxy = \ + Bcfg2.Proxy.ComponentProxy(self.setup['server'], + self.setup['user'], + self.setup['password'], + key=self.setup['key'], + cert=self.setup['certificate'], + ca=self.setup['ca'], + allowedServerCNs=self.setup['serverCN'], + timeout=self.setup['timeout'], + retries=int(self.setup['retries']), + delay=int(self.setup['retry_delay'])) if self.setup['profile']: try: -- cgit v1.2.3-1-g7c22