From 14dc1773ce6cd110869d4957b2b9d4e3c2afd965 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 18 Jul 2012 15:08:28 -0400 Subject: made inotify FAM work. i think. --- src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 95 ++++++++++++++++++++++------ src/lib/Bcfg2/Server/FileMonitor/Pseudo.py | 4 +- src/lib/Bcfg2/Server/FileMonitor/__init__.py | 14 ++-- src/lib/Bcfg2/Server/Plugins/Metadata.py | 26 ++++---- 4 files changed, 99 insertions(+), 40 deletions(-) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index 50c724279..bf540a1c6 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -1,7 +1,7 @@ """ Inotify driver for file alteration events """ import os -import stat +import sys import logging import operator import pyinotify @@ -12,16 +12,20 @@ logger = logging.getLogger(__name__) class Inotify(Pseudo, pyinotify.ProcessEvent): __priority__ = 1 - mask = pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY action_map = {pyinotify.IN_CREATE: 'created', pyinotify.IN_DELETE: 'deleted', - pyinotify.IN_MODIFY: 'changed'} + pyinotify.IN_MODIFY: 'changed', + pyinotify.IN_MOVED_FROM: 'deleted', + pyinotify.IN_MOVED_TO: 'created'} + mask = reduce(lambda x, y: x | y, action_map.keys()) def __init__(self, ignore=None, debug=False): Pseudo.__init__(self, ignore=ignore, debug=debug) self.wm = pyinotify.WatchManager() self.notifier = pyinotify.ThreadedNotifier(self.wm, self) self.notifier.start() + self.event_filter = dict() + self.watches_by_path = dict() def fileno(self): return self.wm.get_fd() @@ -32,33 +36,88 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): if ievent.mask & amask: action = aname break + try: + watch = self.wm.watches[ievent.wd] + except KeyError: + err = sys.exc_info()[1] + logger.error("Error handling event for %s: Watch %s not found" % + (ievent.pathname, ievent.wd)) + return # FAM-style file monitors return the full path to the parent # directory that is being watched, relative paths to anything - # contained within the directory - watch = self.wm.watches[ievent.wd] - if watch.path == ievent.pathname: + # contained within the directory. since we can't use inotify + # to watch files directly, we have to sort of guess at whether + # this watch was actually added on a file (and thus is in + # self.event_filter because we're filtering out other events + # on the directory) or was added directly on a directory. + if (watch.path == ievent.pathname or ievent.wd in self.event_filter): path = ievent.pathname else: # relative path path = os.path.basename(ievent.pathname) - evt = Event(ievent.wd, path, action) - self.events.append(evt) + # figure out the handleID. start with the path of the event; + # that should catch events on files that are watched directly. + # (we have to watch the directory that a file is in, so this + # lets us handle events on different files in the same + # directory -- and thus under the same watch -- with different + # objects.) If the path to the event doesn't have a handler, + # use the path of the watch itself. + handleID = ievent.pathname + if handleID not in self.handles: + handleID = watch.path + evt = Event(handleID, path, action) + + if (ievent.wd not in self.event_filter or + ievent.pathname in self.event_filter[ievent.wd]): + self.events.append(evt) def AddMonitor(self, path, obj): - res = self.wm.add_watch(path, self.mask, quiet=False) - if not res: - # if we didn't get a return, but we also didn't get an - # exception, we're already watching this directory, so we - # need to find the watch descriptor for it - for wd, watch in self.wm.watches.items(): - if watch.path == path: - wd = watch.wd + if not os.path.isdir(path): + # inotify is a little wonky about watching files. for + # instance, if you watch /tmp/foo, and then do 'mv + # /tmp/bar /tmp/foo', it processes that as a deletion of + # /tmp/foo (which it technically _is_, but that's rather + # useless -- we care that /tmp/foo changed, not that it + # was first deleted and then created). In order to + # effectively watch a file, we have to watch the directory + # it's in, and filter out events for other files in the + # same directory that are not similarly watched. + # watch_transient_file requires a Processor _class_, not + # an object, so we can't have this object handle events, + # which is Wrong, so we can't use that function. + watch_path = os.path.dirname(path) + is_dir = False else: - wd = res[path] + watch_path = path + is_dir = True + + # see if this path is already being watched + try: + wd = self.watches_by_path[watch_path] + except KeyError: + wd = self.wm.add_watch(watch_path, self.mask, + quiet=False)[watch_path] + self.watches_by_path[watch_path] = wd + + produce_exists = True + if not is_dir: + if wd not in self.event_filter: + self.event_filter[wd] = [path] + elif path not in self.event_filter[wd]: + self.event_filter[wd].append(path) + else: + # we've been asked to watch a file that we're already + # watching, so we don't need to produce 'exists' + # events + produce_exists = False # inotify doesn't produce initial 'exists' events, so we # inherit from Pseudo to produce those - return Pseudo.AddMonitor(self, path, obj, handleID=wd) + if produce_exists: + return Pseudo.AddMonitor(self, path, obj, handleID=path) + else: + self.handles[path] = obj + return path def shutdown(self): self.notifier.stop() diff --git a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py index baff871d0..089d4cf0f 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py @@ -1,7 +1,6 @@ """ Pseudo provides static monitor support for file alteration events """ import os -import stat import logging from Bcfg2.Server.FileMonitor import FileMonitor, Event @@ -14,9 +13,8 @@ class Pseudo(FileMonitor): """add a monitor to path, installing a callback to obj.HandleEvent""" if handleID is None: handleID = len(list(self.handles.keys())) - mode = os.stat(path)[stat.ST_MODE] self.events.append(Event(handleID, path, 'exists')) - if stat.S_ISDIR(mode): + if os.path.isdir(path): dirList = os.listdir(path) for includedFile in dirList: self.events.append(Event(handleID, includedFile, 'exists')) diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 8bd63e18d..c490acc81 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -44,12 +44,15 @@ class FileMonitor(object): def __repr__(self): return "%s (%s events, fd %s)" % (str(self), len(self.events), self.fileno) + def debug_log(self, msg): + if self.debug: + logger.info(msg) + def should_ignore(self, event): for pattern in self.ignore: if (fnmatch.fnmatch(event.filename, pattern) or fnmatch.fnmatch(os.path.split(event.filename)[-1], pattern)): - if self.debug: - logger.info("Ignoring %s" % event) + self.debug_log("Ignoring %s" % event) return True return False @@ -69,10 +72,9 @@ class FileMonitor(object): logger.info("Got event for unexpected id %s, file %s" % (event.requestID, event.filename)) return - if self.debug: - logger.info("Dispatching event %s %s to obj %s" % - (event.code2str(), event.filename, - self.handles[event.requestID])) + self.debug_log("Dispatching event %s %s to obj %s" % + (event.code2str(), event.filename, + self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) except: diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 77e433ab1..2abc96cc3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -44,10 +44,10 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): # value, so that XIinclude'd files get properly watched fpath = os.path.join(metadata.data, basefile) Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, - fam=metadata.core.fam) + fam=metadata.core.fam, + should_monitor=False) self.should_monitor = watch_clients self.metadata = metadata - self.fam = metadata.core.fam self.basefile = basefile self.data = None self.basedata = None @@ -157,7 +157,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): def HandleEvent(self, event): """Handle fam events""" - filename = event.filename.split('/')[-1] + filename = os.path.basename(event.filename) if filename in self.extras: if event.code2str() == 'exists': return False @@ -233,20 +233,20 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Metadata.__init__(self) Bcfg2.Server.Plugin.Statistics.__init__(self) + self.states = dict() if watch_clients: - try: - core.fam.AddMonitor(os.path.join(self.data, "groups.xml"), self) - core.fam.AddMonitor(os.path.join(self.data, "clients.xml"), self) - except: - print("Unable to add file monitor for groups.xml or clients.xml") - raise Bcfg2.Server.Plugin.PluginInitError + for fname in ["groups.xml", "clients.xml"]: + self.states[fname] = False + try: + core.fam.AddMonitor(os.path.join(self.data, fname), self) + except: + err = sys.exc_info()[1] + msg = "Unable to add file monitor for %s: %s" % (fname, err) + print(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) self.clients_xml = XMLMetadataConfig(self, watch_clients, 'clients.xml') self.groups_xml = XMLMetadataConfig(self, watch_clients, 'groups.xml') - self.states = {} - if watch_clients: - self.states = {"groups.xml": False, - "clients.xml": False} self.addresses = {} self.auth = dict() self.clients = {} -- cgit v1.2.3-1-g7c22