From 0d4c72b7543ba1af56d2b2bd9414c4bf0f0c20c4 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sat, 29 Jun 2013 18:45:22 +0200 Subject: bcfg2-reports: update to use current models --- src/sbin/bcfg2-reports | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index bb45e0009..b0c170b1b 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -53,15 +53,15 @@ def print_fields(fields, client, fmt, extra=None): else: fdata.append("dirty") elif field == 'total': - fdata.append(client.current_interaction.totalcount) + fdata.append(client.current_interaction.total_count) elif field == 'good': - fdata.append(client.current_interaction.goodcount) + fdata.append(client.current_interaction.good_count) elif field == 'modified': - fdata.append(client.current_interaction.modified_entry_count()) + fdata.append(client.current_interaction.modified_count) elif field == 'extra': - fdata.append(client.current_interaction.extra_entry_count()) + fdata.append(client.current_interaction.extra_count) elif field == 'bad': - fdata.append((client.current_interaction.badcount())) + fdata.append(client.current_interaction.bad_count) else: try: fdata.append(getattr(client, field)) -- cgit v1.2.3-1-g7c22 From b37920c96e64a4c93e1dab45b8990eb986bcbab0 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 1 Nov 2013 22:44:41 -0400 Subject: Proxy: Catch SSL errors and retry The SSL library can throw an exception during the XMLRPC method call. There is no explicit except statement for this type of exception so they get raised up and abort the client's processing. This patch catches the exception so that normal retry processing occurs. Typical traceback: Traceback (most recent call last): File "/usr/sbin/bcfg2", line 29, in ? sys.exit(main()) File "/usr/sbin/bcfg2", line 26, in main return Client(setup).run() File "/usr/lib/python2.4/site-packages/Bcfg2/Client/Client.py", line 256, in run rawconfig = self.get_config(times=times).decode('utf-8') File "/usr/lib/python2.4/site-packages/Bcfg2/Client/Client.py", line 225, in get_config self.run_probes(times=times) File "/usr/lib/python2.4/site-packages/Bcfg2/Client/Client.py", line 151, in run_probes probes = Bcfg2.Client.XML.XML(str(self.proxy.GetProbes())) File "/usr/lib/python2.4/site-packages/Bcfg2/Proxy.py", line 81, in __call__ return _orig_Method.__call__(self, *args) File "/usr/lib64/python2.4/xmlrpclib.py", line 1096, in __call__ return self.__send(self.__name, args) File "/usr/lib64/python2.4/xmlrpclib.py", line 1383, in __request verbose=self.__verbose File "/usr/lib/python2.4/site-packages/Bcfg2/Proxy.py", line 331, in request return self.parse_response(response) File "/usr/lib64/python2.4/xmlrpclib.py", line 1255, in parse_response return self._parse_response(file, None) File "/usr/lib64/python2.4/xmlrpclib.py", line 1276, in _parse_response response = file.read(1024) File "/usr/lib64/python2.4/httplib.py", line 480, in read s = self.fp.read(amt) File "/usr/lib64/python2.4/socket.py", line 303, in read data = self._sock.recv(recv_size) File "/usr/lib64/python2.4/site-packages/M2Crypto/SSL/Connection.py", line 217, in read return self._read_bio(size) File "/usr/lib64/python2.4/site-packages/M2Crypto/SSL/Connection.py", line 202, in _read_bio return m2.ssl_read(self.ssl, size, self._timeout) M2Crypto.SSL.SSLError: (104, 'Connection reset by peer') --- src/lib/Bcfg2/Proxy.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index f6db66a93..83c5107d1 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -103,6 +103,9 @@ class RetryMethod(xmlrpclib._Method): except ProxyError: err = sys.exc_info()[1] msg = err + except SSL_ERROR: + err = sys.exc_info()[1] + msg = err except: raise etype, err = sys.exc_info()[:2] -- cgit v1.2.3-1-g7c22 From 593c22c23b2cfbd988ed824a5adc961880bbe7c4 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Sat, 2 Nov 2013 09:43:17 -0400 Subject: Don't re-raise unknown exceptions per stpierre --- src/lib/Bcfg2/Proxy.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index 83c5107d1..34080da6b 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -103,11 +103,7 @@ class RetryMethod(xmlrpclib._Method): except ProxyError: err = sys.exc_info()[1] msg = err - except SSL_ERROR: - err = sys.exc_info()[1] - msg = err except: - raise etype, err = sys.exc_info()[:2] msg = "Unknown failure: %s (%s)" % (err, etype.__name__) if msg: -- cgit v1.2.3-1-g7c22 From 639c7b07cac8c0a5fcf55d457a1b5a18c02eb2a3 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Tue, 5 Nov 2013 10:19:10 -0500 Subject: Admin: Don't call dead Mode.__call__ code In 9eb3db84, Bcfg2.Server.Admin.Mode.__call__() was changed from pass to raise a NotImplementedError. This causes bcfg2-admin compare and bcfg2-admin snapshots to fail because they call Bcfg2.Server.Admin.Mode.__call__() right away. Since that method didn't do anything anyway, it seems expediant to just avoid the call altogether. --- src/lib/Bcfg2/Server/Admin/Compare.py | 1 - src/lib/Bcfg2/Server/Admin/Snapshots.py | 1 - 2 files changed, 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index e3648a6d0..d7285284a 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -115,7 +115,6 @@ class Compare(Bcfg2.Server.Admin.Mode): return identical def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0: self.errExit("No argument specified.\n" "Please see bcfg2-admin compare help for usage.") diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index c2d279391..fcb240352 100644 --- a/src/lib/Bcfg2/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -27,7 +27,6 @@ class Snapshots(Bcfg2.Server.Admin.Mode): self.cfile = self.configfile def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0 or args[0] == '-h': print(self.__usage__) raise SystemExit(0) -- cgit v1.2.3-1-g7c22 From 8a8b10fe738e4cf90f5601b05992f0a4e77502a8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 5 Nov 2013 22:14:43 -0500 Subject: Multiprocessing: don't reuse child RPC listeners --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 33 ++++++----------------------- 1 file changed, 7 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 3cc308b1c..e74bf492e 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -57,8 +57,7 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): Bcfg2.Server.Plugin.Debuggable.__init__(self) self._terminate = threading.Event() self._queues = dict() - self._available_listeners = Queue() - self._blocking_listeners = [] + self._listeners = [] def add_subscriber(self, name): """ Add a subscriber to the queue. This returns the @@ -79,23 +78,14 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): :class:`multiprocessing.connection.Listener` and passes the Listener address to the child as part of the RPC call, so that the child can connect to the Listener to submit its results. - - Listeners are reused when possible to minimize overhead. """ - try: - listener = self._available_listeners.get_nowait() - self.logger.debug("Reusing existing RPC listener at %s" % - listener.address) - except Empty: - listener = Listener() - self.logger.debug("Created new RPC listener at %s" % - listener.address) - self._blocking_listeners.append(listener) + listener = Listener() + self.logger.debug("Created new RPC listener at %s" % listener.address) + self._listeners.append(listener) try: self._queues[dest].put((listener.address, (method, args or [], kwargs or dict()))) conn = listener.accept() - self._blocking_listeners.remove(listener) try: while not self._terminate.is_set(): if conn.poll(self.poll_wait): @@ -103,7 +93,8 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): finally: conn.close() finally: - self._available_listeners.put(listener) + listener.close() + self._listeners.remove(listener) def close(self): """ Close queues and connections. """ @@ -115,21 +106,11 @@ class RPCQueue(Bcfg2.Server.Plugin.Debuggable): # close any listeners that are waiting for connections self.logger.debug("Closing RPC connections") - for listener in self._blocking_listeners: + for listener in self._listeners: self.logger.debug("Closing RPC connection at %s" % listener.address) listener.close() - self.logger.debug("Closing RPC listeners") - try: - while True: - listener = self._available_listeners.get_nowait() - self.logger.debug("Closing RPC listener at %s" % - listener.address) - listener.close() - except Empty: - pass - class DualEvent(object): """ DualEvent is a clone of :class:`threading.Event` that -- cgit v1.2.3-1-g7c22 From 538b4cb83ca4224f4c9e993149b3cd83e227d61f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 5 Nov 2013 22:25:53 -0500 Subject: bcfg2-crypt: fixed deprecation warning --- src/sbin/bcfg2-crypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 0ba84fa0a..ba4d0f1d9 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -418,7 +418,7 @@ def main(): # pylint: disable=R0912,R0915 if data is None: data = getattr(tool, mode)() - if not data: + if data is not None: logger.error("Failed to %s %s, skipping" % (mode, fname)) continue if setup['crypt_stdout']: -- cgit v1.2.3-1-g7c22 From 801baeb391dc3711d735b51c4ef8d447592559dc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 6 Nov 2013 09:19:20 -0500 Subject: Multiprocessing: removed unused import --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index e74bf492e..6d41bbcbb 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -18,7 +18,7 @@ import multiprocessing import Bcfg2.Server.Plugin from itertools import cycle from Bcfg2.Cache import Cache -from Bcfg2.Compat import Queue, Empty, wraps +from Bcfg2.Compat import Empty, wraps from Bcfg2.Server.Core import BaseCore, exposed from Bcfg2.Server.BuiltinCore import Core as BuiltinCore from multiprocessing.connection import Listener, Client -- cgit v1.2.3-1-g7c22 From 7838d063e33246ec21cf58060ced35e3771b5e82 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 7 Nov 2013 08:09:57 -0600 Subject: Version bump to 1.3.3 Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/templates/base.html | 2 +- src/lib/Bcfg2/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 0b2b7dd36..7edf3a949 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5
diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 140fb6937..35d4cfa0a 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.2" +__version__ = "1.3.3" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 -- cgit v1.2.3-1-g7c22 From 9dc84e1ec1f7ffc74f878e16d48637eac660e803 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 11 Nov 2013 08:29:36 -0500 Subject: Probes: automatically remove duplicate probe data --- src/lib/Bcfg2/Server/Plugins/Probes.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index fd6fd3bd1..84e1638d6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -252,9 +252,16 @@ class Probes(Bcfg2.Server.Plugin.Probing, def _write_data_db(self, client): """ Write received probe data to the database """ for probe, data in self.probedata[client.hostname].items(): - pdata = \ - ProbesDataModel.objects.get_or_create(hostname=client.hostname, - probe=probe)[0] + try: + pdata = ProbesDataModel.objects.get_or_create( + hostname=client.hostname, + probe=probe)[0] + except MultipleObjectsReturned: + ProbesDataModel.objects.filter(hostname=client.hostname, + probe=probe).delete() + ProbesDataModel.objects.get_or_create( + hostname=client.hostname, + probe=probe) if pdata.data != data: pdata.data = data pdata.save() -- cgit v1.2.3-1-g7c22 From 16d4a64be162ed6555a7ad497b148b3accb83af0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 11 Nov 2013 13:22:33 -0500 Subject: bcfg2-crypt: fixed completely backwards logic --- src/sbin/bcfg2-crypt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index ba4d0f1d9..4567bc512 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -418,7 +418,7 @@ def main(): # pylint: disable=R0912,R0915 if data is None: data = getattr(tool, mode)() - if data is not None: + if data is None: logger.error("Failed to %s %s, skipping" % (mode, fname)) continue if setup['crypt_stdout']: -- cgit v1.2.3-1-g7c22 From ba22b9e3f8d993d7e0c4c762fac338c8684e3f81 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 11 Nov 2013 13:44:24 -0500 Subject: bcfg2-crypt: better debugging, error handling with Properties files --- src/sbin/bcfg2-crypt | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index 4567bc512..c67334738 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -18,6 +18,33 @@ except ImportError: raise SystemExit(1) +def print_xml(element, keep_text=False): + """ Render an XML element for error output. This prefixes the + line number and removes children for nicer display. + + :param element: The element to render + :type element: lxml.etree._Element + :param keep_text: Do not discard text content from the element for + display + :type keep_text: boolean + """ + xml = None + if len(element) or element.text: + el = copy.copy(element) + if el.text and not keep_text: + el.text = '...' + for child in el.iterchildren(): + el.remove(child) + xml = lxml.etree.tostring( + el, + xml_declaration=False).decode("UTF-8").strip() + else: + xml = lxml.etree.tostring( + element, + xml_declaration=False).decode("UTF-8").strip() + return "%s (line %s)" % (xml, element.sourceline) + + class PassphraseError(Exception): """ Exception raised when there's a problem determining the passphrase to encrypt or decrypt with """ @@ -245,6 +272,7 @@ class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): except PassphraseError: self.logger.error(str(sys.exc_info()[1])) return False + self.logger.debug("Encrypting %s" % print_xml(elt)) elt.text = Bcfg2.Encryption.ssl_encrypt( elt.text, passphrase, Bcfg2.Encryption.get_algorithm(self.setup)).strip() @@ -267,9 +295,15 @@ class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): except PassphraseError: self.logger.error(str(sys.exc_info()[1])) return False - decrypted = Bcfg2.Encryption.ssl_decrypt( - elt.text, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() + self.logger.debug("Decrypting %s" % print_xml(elt)) + try: + decrypted = Bcfg2.Encryption.ssl_decrypt( + elt.text, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)).strip() + except EVPError: + + self.logger.error("Could not decrypt %s, skipping" % + print_xml(elt)) try: elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') elt.set("encrypted", pname) -- cgit v1.2.3-1-g7c22 From 103b1b5198828876fa0684296900769018075f1b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 11 Nov 2013 14:08:51 -0500 Subject: bcfg2-crypt: fixed missing EVPError --- src/sbin/bcfg2-crypt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index c67334738..98a1ca4b0 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -300,8 +300,7 @@ class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): decrypted = Bcfg2.Encryption.ssl_decrypt( elt.text, passphrase, Bcfg2.Encryption.get_algorithm(self.setup)).strip() - except EVPError: - + except Bcfg2.Encryption.EVPError: self.logger.error("Could not decrypt %s, skipping" % print_xml(elt)) try: -- cgit v1.2.3-1-g7c22