From b37920c96e64a4c93e1dab45b8990eb986bcbab0 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 1 Nov 2013 22:44:41 -0400 Subject: Proxy: Catch SSL errors and retry The SSL library can throw an exception during the XMLRPC method call. There is no explicit except statement for this type of exception so they get raised up and abort the client's processing. This patch catches the exception so that normal retry processing occurs. Typical traceback: Traceback (most recent call last): File "/usr/sbin/bcfg2", line 29, in ? sys.exit(main()) File "/usr/sbin/bcfg2", line 26, in main return Client(setup).run() File "/usr/lib/python2.4/site-packages/Bcfg2/Client/Client.py", line 256, in run rawconfig = self.get_config(times=times).decode('utf-8') File "/usr/lib/python2.4/site-packages/Bcfg2/Client/Client.py", line 225, in get_config self.run_probes(times=times) File "/usr/lib/python2.4/site-packages/Bcfg2/Client/Client.py", line 151, in run_probes probes = Bcfg2.Client.XML.XML(str(self.proxy.GetProbes())) File "/usr/lib/python2.4/site-packages/Bcfg2/Proxy.py", line 81, in __call__ return _orig_Method.__call__(self, *args) File "/usr/lib64/python2.4/xmlrpclib.py", line 1096, in __call__ return self.__send(self.__name, args) File "/usr/lib64/python2.4/xmlrpclib.py", line 1383, in __request verbose=self.__verbose File "/usr/lib/python2.4/site-packages/Bcfg2/Proxy.py", line 331, in request return self.parse_response(response) File "/usr/lib64/python2.4/xmlrpclib.py", line 1255, in parse_response return self._parse_response(file, None) File "/usr/lib64/python2.4/xmlrpclib.py", line 1276, in _parse_response response = file.read(1024) File "/usr/lib64/python2.4/httplib.py", line 480, in read s = self.fp.read(amt) File "/usr/lib64/python2.4/socket.py", line 303, in read data = self._sock.recv(recv_size) File "/usr/lib64/python2.4/site-packages/M2Crypto/SSL/Connection.py", line 217, in read return self._read_bio(size) File "/usr/lib64/python2.4/site-packages/M2Crypto/SSL/Connection.py", line 202, in _read_bio return m2.ssl_read(self.ssl, size, self._timeout) M2Crypto.SSL.SSLError: (104, 'Connection reset by peer') --- src/lib/Bcfg2/Proxy.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index f6db66a93..83c5107d1 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -103,6 +103,9 @@ class RetryMethod(xmlrpclib._Method): except ProxyError: err = sys.exc_info()[1] msg = err + except SSL_ERROR: + err = sys.exc_info()[1] + msg = err except: raise etype, err = sys.exc_info()[:2] -- cgit v1.2.3-1-g7c22 From 593c22c23b2cfbd988ed824a5adc961880bbe7c4 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Sat, 2 Nov 2013 09:43:17 -0400 Subject: Don't re-raise unknown exceptions per stpierre --- src/lib/Bcfg2/Proxy.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index 83c5107d1..34080da6b 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -103,11 +103,7 @@ class RetryMethod(xmlrpclib._Method): except ProxyError: err = sys.exc_info()[1] msg = err - except SSL_ERROR: - err = sys.exc_info()[1] - msg = err except: - raise etype, err = sys.exc_info()[:2] msg = "Unknown failure: %s (%s)" % (err, etype.__name__) if msg: -- cgit v1.2.3-1-g7c22 From 639c7b07cac8c0a5fcf55d457a1b5a18c02eb2a3 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Tue, 5 Nov 2013 10:19:10 -0500 Subject: Admin: Don't call dead Mode.__call__ code In 9eb3db84, Bcfg2.Server.Admin.Mode.__call__() was changed from pass to raise a NotImplementedError. This causes bcfg2-admin compare and bcfg2-admin snapshots to fail because they call Bcfg2.Server.Admin.Mode.__call__() right away. Since that method didn't do anything anyway, it seems expediant to just avoid the call altogether. --- src/lib/Bcfg2/Server/Admin/Compare.py | 1 - src/lib/Bcfg2/Server/Admin/Snapshots.py | 1 - 2 files changed, 2 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index e3648a6d0..d7285284a 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -115,7 +115,6 @@ class Compare(Bcfg2.Server.Admin.Mode): return identical def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0: self.errExit("No argument specified.\n" "Please see bcfg2-admin compare help for usage.") diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index c2d279391..fcb240352 100644 --- a/src/lib/Bcfg2/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -27,7 +27,6 @@ class Snapshots(Bcfg2.Server.Admin.Mode): self.cfile = self.configfile def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0 or args[0] == '-h': print(self.__usage__) raise SystemExit(0) -- cgit v1.2.3-1-g7c22 From 8a8b10fe738e4cf90f5601b05992f0a4e77502a8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 5 Nov 2013 22:14:43 -0500 Subject: Multiprocessing: don't reuse child RPC listeners --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 33 ++++++----------------------- 1 file changed, 7 insertions(+), 26 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 3cc308b1c..e74bf492e 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -57,8 +57,7 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): Bcfg2.Server.Plugin.Debuggable.__init__(self) self._terminate = threading.Event() self._queues = dict() - self._available_listeners = Queue() - self._blocking_listeners = [] + self._listeners = [] def add_subscriber(self, name): """ Add a subscriber to the queue. This returns the @@ -79,23 +78,14 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): :class:`multiprocessing.connection.Listener` and passes the Listener address to the child as part of the RPC call, so that the child can connect to the Listener to submit its results. - - Listeners are reused when possible to minimize overhead. """ - try: - listener = self._available_listeners.get_nowait() - self.logger.debug("Reusing existing RPC listener at %s" % - listener.address) - except Empty: - listener = Listener() - self.logger.debug("Created new RPC listener at %s" % - listener.address) - self._blocking_listeners.append(listener) + listener = Listener() + self.logger.debug("Created new RPC listener at %s" % listener.address) + self._listeners.append(listener) try: self._queues[dest].put((listener.address, (method, args or [], kwargs or dict()))) conn = listener.accept() - self._blocking_listeners.remove(listener) try: while not self._terminate.is_set(): if conn.poll(self.poll_wait): @@ -103,7 +93,8 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): finally: conn.close() finally: - self._available_listeners.put(listener) + listener.close() + self._listeners.remove(listener) def close(self): """ Close queues and connections. """ @@ -115,21 +106,11 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): # close any listeners that are waiting for connections self.logger.debug("Closing RPC connections") - for listener in self._blocking_listeners: + for listener in self._listeners: self.logger.debug("Closing RPC connection at %s" % listener.address) listener.close() - self.logger.debug("Closing RPC listeners") - try: - while True: - listener = self._available_listeners.get_nowait() - self.logger.debug("Closing RPC listener at %s" % - listener.address) - listener.close() - except Empty: - pass - class DualEvent(object): """ DualEvent is a clone of :class:`threading.Event` that -- cgit v1.2.3-1-g7c22 From 801baeb391dc3711d735b51c4ef8d447592559dc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 6 Nov 2013 09:19:20 -0500 Subject: Multiprocessing: removed unused import --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index e74bf492e..6d41bbcbb 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -18,7 +18,7 @@ import multiprocessing import Bcfg2.Server.Plugin from itertools import cycle from Bcfg2.Cache import Cache -from Bcfg2.Compat import Queue, Empty, wraps +from Bcfg2.Compat import Empty, wraps from Bcfg2.Server.Core import BaseCore, exposed from Bcfg2.Server.BuiltinCore import Core as BuiltinCore from multiprocessing.connection import Listener, Client -- cgit v1.2.3-1-g7c22 From 7838d063e33246ec21cf58060ced35e3771b5e82 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 7 Nov 2013 08:09:57 -0600 Subject: Version bump to 1.3.3 Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/templates/base.html | 2 +- src/lib/Bcfg2/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 0b2b7dd36..7edf3a949 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5
diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 140fb6937..35d4cfa0a 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.2" +__version__ = "1.3.3" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 -- cgit v1.2.3-1-g7c22 From 9dc84e1ec1f7ffc74f878e16d48637eac660e803 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 11 Nov 2013 08:29:36 -0500 Subject: Probes: automatically remove duplicate probe data --- src/lib/Bcfg2/Server/Plugins/Probes.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index fd6fd3bd1..84e1638d6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -252,9 +252,16 @@ class Probes(Bcfg2.Server.Plugin.Probing, def _write_data_db(self, client): """ Write received probe data to the database """ for probe, data in self.probedata[client.hostname].items(): - pdata = \ - ProbesDataModel.objects.get_or_create(hostname=client.hostname, - probe=probe)[0] + try: + pdata = ProbesDataModel.objects.get_or_create( + hostname=client.hostname, + probe=probe)[0] + except MultipleObjectsReturned: + ProbesDataModel.objects.filter(hostname=client.hostname, + probe=probe).delete() + ProbesDataModel.objects.get_or_create( + hostname=client.hostname, + probe=probe) if pdata.data != data: pdata.data = data pdata.save() -- cgit v1.2.3-1-g7c22