From 73979d1482a73da24104950be4bad8f3d50c0754 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 10 Jan 2014 11:59:14 -0500 Subject: Reporting: occasionally reap child threads I noticed that, at least on Python 2.4, memory for threads doesn't get freed until the threads are joined. This patch causes the collector to periodically go through and reap those threads. Tested in production for ~1 month, no reported issues. --- src/lib/Bcfg2/Reporting/Collector.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'src/lib/Bcfg2/Reporting/Collector.py') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index b42364d8d..52700f917 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -63,6 +63,8 @@ class ReportingCollector(object): self.encoding = setup['encoding'] self.terminate = None self.context = None + self.children = [] + self.cleanup_threshold = 25 if setup['debug']: level = logging.DEBUG @@ -110,6 +112,7 @@ class ReportingCollector(object): self.terminate = threading.Event() atexit.register(self.shutdown) self.context = daemon.DaemonContext(detach_process=True) + iter = 0 if self.setup['daemon']: self.logger.debug("Daemonizing") @@ -133,6 +136,13 @@ class ReportingCollector(object): store_thread = ReportingStoreThread(interaction, self.storage) store_thread.start() + self.children.append(store_thread) + + iter += 1 + if iter >= self.cleanup_threshold: + self.reap_children() + iter = 0 + except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() @@ -152,3 +162,16 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() + + def reap_children(self): + """Join any non-live threads""" + newlist = [] + + self.logger.debug("Starting reap_children") + for child in self.children: + if child.isAlive(): + newlist.append(child) + else: + child.join() + self.logger.debug("Joined child thread %s" % child.getName()) + self.children = newlist -- cgit v1.2.3-1-g7c22 From 2de76a7b44c148f2ce9e851060c16513581174ff Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 18 Feb 2014 08:53:35 -0500 Subject: ensure that DB connections are always closed at thread/process exit --- src/lib/Bcfg2/Reporting/Collector.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'src/lib/Bcfg2/Reporting/Collector.py') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 52700f917..0493c907b 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -56,7 +56,7 @@ class ReportingCollector(object): """The collecting process for reports""" def __init__(self, setup): - """Setup the collector. This may be called by the daemon or though + """Setup the collector. This may be called by the daemon or though bcfg2-admin""" self.setup = setup self.datastore = setup['repo'] @@ -99,12 +99,12 @@ class ReportingCollector(object): raise ReportingError try: - self.logger.debug("Validating storage %s" % + self.logger.debug("Validating storage %s" % self.storage.__class__.__name__) self.storage.validate() except: self.logger.error("Storage backed %s failed to validate: %s" % - (self.storage.__class__.__name__, + (self.storage.__class__.__name__, traceback.format_exc().splitlines()[-1])) def run(self): @@ -162,6 +162,9 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() + from django import db + self.logger.info("%s: Closing database connection" % self.name) + db.close_connection() def reap_children(self): """Join any non-live threads""" -- cgit v1.2.3-1-g7c22 From e7d8c70958e80924408c0e31fa06f1ad06607f99 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 28 Feb 2014 11:25:40 -0500 Subject: Reporting: properly close db connection Close the db connection at the end of each DjangoORM import, not when the reporting collector shuts down. The collector may not have even opened a connection, in the case of a storage backend other than DjangoORM. Fixes #157 --- src/lib/Bcfg2/Reporting/Collector.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/lib/Bcfg2/Reporting/Collector.py') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 0493c907b..2859eca28 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -162,9 +162,6 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() - from django import db - self.logger.info("%s: Closing database connection" % self.name) - db.close_connection() def reap_children(self): """Join any non-live threads""" -- cgit v1.2.3-1-g7c22 From f76ee4bae4d8a45cc2050b8219cdf7f2e8c6533d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 4 Mar 2014 07:46:46 -0500 Subject: bcfg2-report-collector: better error messages when failing to daemonize --- src/lib/Bcfg2/Reporting/Collector.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/lib/Bcfg2/Reporting/Collector.py') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 2859eca28..8ca145f16 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import sys import atexit import daemon import logging @@ -6,6 +7,7 @@ import traceback import threading # pylint: disable=E0611 +from lockfile import LockFailed, LockTimeout try: from lockfile.pidlockfile import PIDLockFile from lockfile import Error as PIDFileError @@ -119,6 +121,17 @@ class ReportingCollector(object): try: self.context.pidfile = PIDLockFile(self.setup['daemon']) self.context.open() + except LockFailed: + self.logger.error("Failed to daemonize: %s" % + sys.exc_info()[1]) + self.shutdown() + return + except LockTimeout: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + self.setup['daemon']) + self.shutdown() + return except PIDFileError: self.logger.error("Error writing pid file: %s" % traceback.format_exc().splitlines()[-1]) -- cgit v1.2.3-1-g7c22