Attempt to break the pid lock during startup.

This commit attempts to break the pidfilelock during startup in cases where the
process may have exited without successfully cleaning up the lockfile. It also
attempts to grab the lock before opening the context. Also applied to the
Collector module, which may have been looking for the wrong exception since it
does not rely on a timeout.

2 files changed, 41 insertions, 15 deletions

diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py
index 8ca145f16..50bb49f78 100644
--- a/src/lib/Bcfg2/Reporting/Collector.py
+++ b/src/lib/Bcfg2/Reporting/Collector.py
@@ -7,7 +7,7 @@ import traceback
 import threading
 
 # pylint: disable=E0611
-from lockfile import LockFailed, LockTimeout
+from lockfile import LockFailed, LockTimeout, AlreadyLocked
 try:
     from lockfile.pidlockfile import PIDLockFile
     from lockfile import Error as PIDFileError
@@ -118,25 +118,43 @@ class ReportingCollector(object):
 
         if self.setup['daemon']:
             self.logger.debug("Daemonizing")
+            self.context.pidfile = PIDLockFile(self.setup['daemon'])
             try:
-                self.context.pidfile = PIDLockFile(self.setup['daemon'])
-                self.context.open()
+                self.context.pidfile.acquire()
             except LockFailed:
                 self.logger.error("Failed to daemonize: %s" %
                                   sys.exc_info()[1])
                 self.shutdown()
                 return
+            except AlreadyLocked:
+                try: # attempt to break the lock
+                    os.kill(self.context.pidfile.read_pid(), 0)
+                except OSError: # No process with locked PID
+                    self.context.pidfile.break_lock()
+                else:
+                    self.logger.error("Failed to daemonize: "
+                                    "Failed to acquire lock on %s" %
+                                    self.setup['daemon'])
+                    self.shutdown()
+                    return
             except LockTimeout:
-                self.logger.error("Failed to daemonize: "
-                                  "Failed to acquire lock on %s" %
-                                  self.setup['daemon'])
-                self.shutdown()
-                return
+                try: # attempt to break the lock
+                    os.kill(self.context.pidfile.read_pid(), 0)
+                except OSError: # No process with locked PID
+                    self.context.pidfile.break_lock()
+                else:
+                    self.logger.error("Failed to daemonize: "
+                                    "Failed to acquire lock on %s" %
+                                    self.setup['daemon'])
+                    self.shutdown()
+                    return
             except PIDFileError:
                 self.logger.error("Error writing pid file: %s" %
                     traceback.format_exc().splitlines()[-1])
                 self.shutdown()
                 return
+
+            self.context.open()
             self.logger.info("Starting daemon")
 
         self.transport.start_monitor(self)
diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py
index 93da767c7..aeac161dd 100644
--- a/src/lib/Bcfg2/Server/BuiltinCore.py
+++ b/src/lib/Bcfg2/Server/BuiltinCore.py
@@ -1,5 +1,6 @@
 """ The core of the builtin Bcfg2 server. """
 
+import os
 import sys
 import time
 import socket
@@ -84,18 +85,25 @@ class Core(BaseCore):
         """ Open :attr:`context` to drop privileges, write the PID
         file, and daemonize the server core. """
         try:
-            self.context.open()
-            self.logger.info("%s daemonized" % self.name)
-            return True
+            self.context.pidfile.acquire()
         except LockFailed:
             err = sys.exc_info()[1]
             self.logger.error("Failed to daemonize %s: %s" % (self.name, err))
             return False
         except LockTimeout:
-            err = sys.exc_info()[1]
-            self.logger.error("Failed to daemonize %s: Failed to acquire lock "
-                              "on %s" % (self.name, self.setup['daemon']))
-            return False
+            try:
+                os.kill(self.context.pidfile.read_pid(), 0)
+            except OSError: # No process with locked PID
+                self.context.pidfile.break_lock()
+            else:
+                err = sys.exc_info()[1]
+                self.logger.error("Failed to daemonize %s: Failed to acquire lock "
+                                "on %s" % (self.name, self.setup['daemon']))
+                return False
+
+        self.context.open()
+        self.logger.info("%s daemonized" % self.name)
+        return True
 
     def _run(self):
         """ Create :attr:`server` to start the server listening. """