From e496fb95eaf9200f78248106f9fd7ec6b7d9e530 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Fri, 22 Jul 2011 14:26:35 -0500 Subject: Fixed to accommodate changes made to Plugin.py in changeset 3291a875339a7e5569d4. The changes to the INode.Match() function in changeset 3291a875339a7e5569d4 caused breakage in the Deps plugin, as it inherits from INode. This commit adjusts the definition of the predicate function in Deps.py to mirror the changes made to INode in Plugin.py, eliminating the error about the wrong number of parameters being passed to the lambda function. --- src/lib/Server/Plugins/Deps.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index b186258cb..389645232 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -8,18 +8,19 @@ import Bcfg2.Server.Plugin class DNode(Bcfg2.Server.Plugin.INode): """DNode provides supports for single predicate types for dependencies.""" - raw = {'Group': "lambda x:'%s' in x.groups and predicate(x)"} + raw = {'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)"} containers = ['Group'] def __init__(self, data, idict, parent=None): self.data = data self.contents = {} if parent == None: - self.predicate = lambda x: True + self.predicate = lambda x, d: True else: predicate = parent.predicate if data.tag in list(self.raw.keys()): - self.predicate = eval(self.raw[data.tag] % (data.get('name')), + self.predicate = eval(self.raw[data.tag] % + {'name': data.get('name')}, {'predicate': predicate}) else: raise Exception -- cgit v1.2.3-1-g7c22 From 42199a9835a548a0a78f48fd6c5f11c3153371dd Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Fri, 22 Jul 2011 23:17:14 -0500 Subject: Created a new method to return a sorted list of plugins by type. Replaced many list comprehensions in Core.py with a new method named plugins_by_type(), which does the same thing with an added twist: this new method also sorts the list of plugins by a new field named sort_order. It also uses the name of the plugin where the sort_order values are the same. This lets us control the processing sequence of plugins that need to build on the results of a plugin that runs prior to them. The immediate example is Deps, which should run after Packages has generated the full list of packages to be installed. Prior to this commit, it was impossible to control the order in which they ran. A future commit will (hopefully) take advantage of this capability. This commit also splits the Core.validate_data() method into two: one for validate_structures() and one for validate_goals(), instead of passing in a base class and using if logic. This approach seemed a little clearer to me. --- src/lib/Server/Core.py | 95 ++++++++++++++++++++++---------------- src/lib/Server/Plugin.py | 5 ++ src/lib/Server/Plugins/Metadata.py | 1 + 3 files changed, 60 insertions(+), 41 deletions(-) diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 5adfa5381..91b6a3555 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -90,42 +90,49 @@ class Core(Component): "Unloading %s" % (p, bl)) for plug in bl: del self.plugins[plug] - # This section loads the experimental plugins + # This section logs the experimental plugins expl = [plug for (name, plug) in list(self.plugins.items()) if plug.experimental] if expl: logger.info("Loading experimental plugin(s): %s" % \ (" ".join([x.name for x in expl]))) logger.info("NOTE: Interfaces subject to change") + # This section logs the deprecated plugins depr = [plug for (name, plug) in list(self.plugins.items()) if plug.deprecated] - # This section loads the deprecated plugins if depr: logger.info("Loading deprecated plugin(s): %s" % \ (" ".join([x.name for x in depr]))) - mlist = [p for p in list(self.plugins.values()) if \ - isinstance(p, Bcfg2.Server.Plugin.Metadata)] + mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) if len(mlist) == 1: self.metadata = mlist[0] else: logger.error("No Metadata Plugin loaded; failed to instantiate Core") raise CoreInitError("No Metadata Plugin") - self.statistics = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Statistics)] - self.pull_sources = [plugin for plugin in self.statistics - if isinstance(plugin, Bcfg2.Server.Plugin.PullSource)] - self.generators = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Generator)] - self.structures = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Structure)] - self.connectors = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Connector)] + self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) + self.pull_sources = self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) + self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) + self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) + self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) self.ca = ca self.fam_thread = threading.Thread(target=self._file_monitor_thread) if start_fam_thread: self.fam_thread.start() + def plugins_by_type(self, base_cls): + """Return a list of loaded plugins that match the passed type. + + The returned list is sorted in ascending order by the Plugins' + sort_order value. The sort_order defaults to 500 in Plugin.py, + but can be overridden by individual plugins. Plugins with the + same numerical sort_order value are sorted in alphabetical + order by their name. + """ + return sorted([plugin for plugin in self.plugins.values() + if isinstance(plugin, base_cls)], + key=lambda p: (p.sort_order, p.name)) + def _file_monitor_thread(self): """The thread for monitor the files.""" famfd = self.fam.fileno() @@ -141,9 +148,8 @@ class Core(Component): except: continue # VCS plugin periodic updates - for plugin in list(self.plugins.values()): - if isinstance(plugin, Bcfg2.Server.Plugin.Version): - self.revision = plugin.get_revision() + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): + self.revision = plugin.get_revision() def init_plugins(self, plugin): """Handling for the plugins.""" @@ -176,23 +182,33 @@ class Core(Component): for plugin in list(self.plugins.values()): plugin.shutdown() - def validate_data(self, metadata, data, base_cls): + def validate_structures(self, metadata, data): """Checks the data structure.""" - for plugin in list(self.plugins.values()): - if isinstance(plugin, base_cls): - try: - if base_cls == Bcfg2.Server.Plugin.StructureValidator: - plugin.validate_structures(metadata, data) - elif base_cls == Bcfg2.Server.Plugin.GoalValidator: - plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: - err = sys.exc_info()[1] - logger.error("Plugin %s structure validation failed: %s" \ - % (plugin.name, err.message)) - raise - except: - logger.error("Plugin %s: unexpected structure validation failure" \ - % (plugin.name), exc_info=1) + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): + try: + plugin.validate_structures(metadata, data) + except Bcfg2.Server.Plugin.ValidationError: + err = sys.exc_info()[1] + logger.error("Plugin %s structure validation failed: %s" \ + % (plugin.name, err.message)) + raise + except: + logger.error("Plugin %s: unexpected structure validation failure" \ + % (plugin.name), exc_info=1) + + def validate_goals(self, metadata, data): + """Checks that the config matches the goals enforced by the plugins.""" + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): + try: + plugin.validate_goals(metadata, data) + except Bcfg2.Server.Plugin.ValidationError: + err = sys.exc_info()[1] + logger.error("Plugin %s goal validation failed: %s" \ + % (plugin.name, err.message)) + raise + except: + logger.error("Plugin %s: unexpected goal validation failure" \ + % (plugin.name), exc_info=1) def GetStructures(self, metadata): """Get all structures for client specified by metadata.""" @@ -276,8 +292,7 @@ class Core(Component): logger.error("error in GetStructures", exc_info=1) return lxml.etree.Element("error", type='structure error') - self.validate_data(meta, structures, - Bcfg2.Server.Plugin.StructureValidator) + self.validate_structures(meta, structures) # Perform altsrc consistency checking esrcs = {} @@ -297,7 +312,7 @@ class Core(Component): config.append(astruct) except: logger.error("error in BindStructure", exc_info=1) - self.validate_data(meta, config, Bcfg2.Server.Plugin.GoalValidator) + self.validate_goals(meta, config) logger.info("Generated config for %s in %.03fs" % \ (client, time.time() - start)) return config @@ -305,10 +320,9 @@ class Core(Component): def GetDecisions(self, metadata, mode): """Get data for the decision list.""" result = [] - for plugin in list(self.plugins.values()): + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): try: - if isinstance(plugin, Bcfg2.Server.Plugin.Decision): - result += plugin.GetDecisions(metadata, mode) + result += plugin.GetDecisions(metadata, mode) except: logger.error("Plugin: %s failed to generate decision list" \ % plugin.name, exc_info=1) @@ -354,8 +368,7 @@ class Core(Component): name = self.metadata.resolve_client(address) meta = self.build_metadata(name) - for plugin in [p for p in list(self.plugins.values()) \ - if isinstance(p, Bcfg2.Server.Plugin.Probing)]: + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): for probe in plugin.GetProbes(meta): resp.append(probe) return lxml.etree.tostring(resp, encoding='UTF-8', diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 17547be13..108e2d89f 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -81,6 +81,11 @@ class Plugin(object): deprecated = False conflicts = [] + # Default sort_order to 500. Plugins of the same type are + # processed in order of ascending sort_order value. Plugins with + # the same sort_order are sorted alphabetically by their name. + sort_order = 500 + def __init__(self, core, datastore): object.__init__(self) self.Entries = {} diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index 7fc34f178..bfe1ac053 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -222,6 +222,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' name = "Metadata" + sort_order = 500 def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) -- cgit v1.2.3-1-g7c22 From 71b2daf28752af0f4f91e10139bfac8bce10b041 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Sat, 23 Jul 2011 00:36:42 -0500 Subject: Refactor the validate_structure() method into two. New method named calculate_prereqs() handles pre-req calculation. Existing validate_structure() method now only handles memoization of the prereqs and use of the prereqs to update the configuration. Also added a sort_order value to cause this plugin to run after Packages, so we can use Deps to add dependencies to implicitly added Packages. NOTE: This doesn't work yet, as Packages adds BoundPackage entries, and Deps doesn't yet realize they are equivalent to the Package entries it knows about. This will be fixed in a future commit. --- src/lib/Server/Plugins/Deps.py | 70 +++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index 389645232..cedee4d8b 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -49,6 +49,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = DepXMLSrc + # Override the default sort_order (of 500) so that this plugin + # gets handled after others running at the default. In particular, + # we want to run after Packages, so we can see the final set of + # packages that will be installed on the client. + sort_order = 750 + def __init__(self, core, datastore): Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) @@ -59,8 +65,10 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event) def validate_structures(self, metadata, structures): + """Examine the passed structures and append any additional + prerequisite entries as defined by the files in Deps. + """ entries = [] - prereqs = [] for structure in structures: for entry in structure.getchildren(): if (entry.tag, entry.get('name')) not in entries \ @@ -71,33 +79,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, gdata = list(metadata.groups) gdata.sort() gdata = tuple(gdata) + + # Check to see if we have cached the prereqs already if (entries, gdata) in self.cache: prereqs = self.cache[(entries, gdata)] else: - [src.Cache(metadata) for src in list(self.entries.values())] - - toexamine = list(entries[:]) - while toexamine: - entry = toexamine.pop() - matching = [src for src in list(self.entries.values()) - if src.cache and entry[0] in src.cache[1] - and entry[1] in src.cache[1][entry[0]]] - if len(matching) > 1: - prio = [int(src.priority) for src in matching] - if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % - (entry[0].lower(), metadata.hostname, entry[1])) - raise Bcfg2.Server.Plugin.PluginExecutionError - index = prio.index(max(prio)) - matching = [matching[index]] - - if not matching: - continue - elif len(matching) == 1: - for prq in matching[0].cache[1][entry[0]][entry[1]]: - if prq not in prereqs and prq not in entries: - toexamine.append(prq) - prereqs.append(prq) + prereqs = self.calculate_prereqs(metadata, entries) self.cache[(entries, gdata)] = prereqs newstruct = lxml.etree.Element("Independent") @@ -107,3 +94,36 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, except: self.logger.error("Failed to add dep entry for %s:%s" % (tag, name)) structures.append(newstruct) + + + def calculate_prereqs(self, metadata, entries): + """Calculate the prerequisites defined in Deps for the passed + set of entries. + """ + prereqs = [] + [src.Cache(metadata) for src in list(self.entries.values())] + + import ipdb; ipdb.set_trace() + toexamine = list(entries[:]) + while toexamine: + entry = toexamine.pop() + matching = [src for src in list(self.entries.values()) + if src.cache and entry[0] in src.cache[1] + and entry[1] in src.cache[1][entry[0]]] + if len(matching) > 1: + prio = [int(src.priority) for src in matching] + if prio.count(max(prio)) > 1: + self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % + (entry[0].lower(), metadata.hostname, entry[1])) + raise Bcfg2.Server.Plugin.PluginExecutionError + index = prio.index(max(prio)) + matching = [matching[index]] + elif len(matching) == 1: + for prq in matching[0].cache[1][entry[0]][entry[1]]: + if prq not in prereqs and prq not in entries: + toexamine.append(prq) + prereqs.append(prq) + else: + continue + + return prereqs -- cgit v1.2.3-1-g7c22 From 0b69a35bb21341d006a1463f8683d249eae3229a Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Sat, 23 Jul 2011 10:25:26 -0500 Subject: Treat Bound entries like unbound entries for calculating prerequisites in Deps. --- src/lib/Server/Plugins/Deps.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index cedee4d8b..3a538292e 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -71,9 +71,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, entries = [] for structure in structures: for entry in structure.getchildren(): - if (entry.tag, entry.get('name')) not in entries \ - and not isinstance(entry, lxml.etree._Comment): - entries.append((entry.tag, entry.get('name'))) + tag = entry.tag + if tag.startswith('Bound'): + tag = tag[5:] + if (tag, entry.get('name')) not in entries \ + and not isinstance(entry, lxml.etree._Comment): + entries.append((tag, entry.get('name'))) entries.sort() entries = tuple(entries) gdata = list(metadata.groups) @@ -101,9 +104,8 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, set of entries. """ prereqs = [] - [src.Cache(metadata) for src in list(self.entries.values())] + [src.Cache(metadata) for src in self.entries.values()] - import ipdb; ipdb.set_trace() toexamine = list(entries[:]) while toexamine: entry = toexamine.pop() -- cgit v1.2.3-1-g7c22 From 7e06e71fb12413b5179a151a99fcfea94ee004d7 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Sat, 23 Jul 2011 23:51:13 -0500 Subject: Added docstring to explain what 'datastore' argument contains. --- src/lib/Server/Plugin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 108e2d89f..70ab64df0 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -87,6 +87,11 @@ class Plugin(object): sort_order = 500 def __init__(self, core, datastore): + """Initialize the plugin. + + :param core: the Bcfg2.Server.Core initializing the plugin + :param datastore: the filesystem path of Bcfg2's repository + """ object.__init__(self) self.Entries = {} self.core = core -- cgit v1.2.3-1-g7c22 From 17d801958e683c5ff4a9122f9fd172f0be67f5a5 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Mon, 25 Jul 2011 17:23:46 -0500 Subject: Update to Deps to eliminate "Failed to add dep entry for :None" warning. Comments in the Deps file (inside a Package element) will cause the above error. In generating the prerequisites, we can eliminate this by looking for callable() items and excluding them. --- src/lib/Server/Plugins/Deps.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index 3a538292e..482d457af 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -122,7 +122,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, matching = [matching[index]] elif len(matching) == 1: for prq in matching[0].cache[1][entry[0]][entry[1]]: - if prq not in prereqs and prq not in entries: + # XML comments seem to show up in the cache as a + # tuple with item 0 being callable. The logic + # below filters them out. Would be better to + # exclude them when we load the cache in the first + # place. + if prq not in prereqs and prq not in entries and not callable(prq[0]): toexamine.append(prq) prereqs.append(prq) else: -- cgit v1.2.3-1-g7c22 From bda0f01adda437a6bac1c5a890063bccbd1f5b78 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Tue, 26 Jul 2011 09:34:39 -0500 Subject: Rewrote DirectoryBacked so it handles files contained in an arbitrary directory layout. Previously, DirectoryBacked (and as a result Bundler, Deps, Rules, Base, Pkgmgr, and others) only recognized XML files contained in the top-level plugin directory, for example: Deps/foo.xml Deps/subdir/foo.xml # <--- Ignored Bundler/bar.xml Bundler/subdir/baz.xml # <--- Ignored Now it can support the following as well: Deps/debian-lenny/foo.xml Deps/debian-squeeze/foo.xml Bundler/group-a/bar.xml Bundler/group-b/baz.xml Note that the directories and filenames do not factor into the semantic meaning of the configuration specification. The contents of foo.xml must stand alone, as if they were in the same single-level directory as before. In the case of Deps, Rules, Pkgmgr, and Svcmgr, you must use Groups and priorities within the XML files as needed to ensure that Bcfg2 can correctly resolve the configuration for your hosts. For example, prior to this change you would use a single file like the following: Deps/foo.xml: Now you can use a pair of files in separate directories like the following. Note how the groups within each file prevent there from being two sources for a single package: Deps/debian-lenny/foo.xml: Deps/debian-squeeze/foo.xml: In the case of Bundler, individual filenames must remain unique throughout the directory hierarchy, and they must match the bundle name. --- src/lib/Server/Plugin.py | 144 ++++++++++++++++++++++++++++---------- src/lib/Server/Plugins/Bundler.py | 5 +- 2 files changed, 110 insertions(+), 39 deletions(-) diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 70ab64df0..a7ab9feab 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -36,7 +36,7 @@ mdata_setup = Bcfg2.Options.OptionParser(opts) mdata_setup.parse([]) del mdata_setup['args'] -logger = logging.getLogger('Bcfg2.Plugin') +logger = logging.getLogger('Bcfg2.Server.Plugin') default_file_metadata = mdata_setup @@ -380,9 +380,22 @@ class DirectoryBacked(object): object.__init__(self) self.name = name self.fam = fam + + # self.entries contains information about the files monitored + # by this object.... The keys of the dict are the relative + # paths to the files. The values are the objects (of type + # __child__) that handle their contents. self.entries = {} - self.inventory = False - fam.AddMonitor(name, self) + + # self.handles contains information about the directories + # monitored by this object. The keys of the dict are the + # values returned by the initial fam.AddMonitor() call (which + # appear to be integers). The values are the relative paths of + # the directories. + self.handles = {} + + # Monitor everything in the plugin's directory + self.add_directory_monitor('') def __getitem__(self, key): return self.entries[key] @@ -390,46 +403,103 @@ class DirectoryBacked(object): def __iter__(self): return iter(list(self.entries.items())) - def AddEntry(self, name): - """Add new entry to data structures upon file creation.""" - if name == '': - logger.info("got add for empty name") - elif name in self.entries: - self.entries[name].HandleEvent() - else: - if ((name[-1] == '~') or - (name[:2] == '.#') or - (name[-4:] == '.swp') or - (name in ['SCCS', '.svn'])): - return - if not self.patterns.match(name): + def add_directory_monitor(self, relative): + """Add a new directory to FAM structures for monitoring. + + :param relative: Path name to monitor. This must be relative + to the plugin's directory. An empty string value ("") will + cause the plugin directory itself to be monitored. + """ + dirpathname = os.path.join(self.data, relative) + if relative not in self.handles.values(): + if not posixpath.isdir(dirpathname): + logger.error("Failed to open directory %s" % (dirpathname)) return - self.entries[name] = self.__child__('%s/%s' % (self.name, name)) - self.entries[name].HandleEvent() + reqid = self.fam.AddMonitor(dirpathname, self) + self.handles[reqid] = relative def HandleEvent(self, event): - """Propagate fam events to underlying objects.""" + """Handle FAM/Gamin events. + + This method is invoked by FAM/Gamin when it detects a change + to a filesystem object we have requsted to be monitored. + + This method manages the lifecycle of events related to the + monitored objects, adding them to our indiciess and creating + objects of type __child__ that actually do the domain-specific + processing. When appropriate, it propogates events those + objects by invoking their HandleEvent in turn. + """ action = event.code2str() - if event.filename == '': - logger.info("Got event for blank filename") + + # Exclude events for actions and filesystem paths we don't + # care about + if action == 'endExist': return - if action == 'exists': - if event.filename != self.name: - self.AddEntry(event.filename) - elif action == 'created': - self.AddEntry(event.filename) - elif action == 'changed': - if event.filename in self.entries: - self.entries[event.filename].HandleEvent(event) - elif action == 'deleted': - if event.filename in self.entries: - del self.entries[event.filename] - elif action in ['endExist']: - pass + elif os.path.isabs(event.filename[0]): + # After AddDirectoryMonitor calls, we receive an 'exists' + # event with the just-added directory and its absolute + # path name. Ignore these. + return + elif event.filename == '': + logger.warning("Got event for blank filename") + return + + # Calculate the absolute and relative paths this event refers to + abspath = os.path.join(self.data, self.handles[event.requestID], + event.filename) + relpath = os.path.join(self.handles[event.requestID], event.filename) + + if action == 'deleted': + for key in self.entries.keys(): + if key.startswith(relpath): + del self.entries[key] + for handle in self.handles.keys(): + if self.handles[handle].startswith(relpath): + del self.handles[handle] + elif posixpath.isdir(abspath): + # Deal with events for directories + if action in ['exists', 'created']: + self.add_directory_monitor(relpath) + elif action == 'changed' and relpath in self.entries: + # Ownerships, permissions or timestamps changed on the + # directory. None of these should affect the contents + # of the files, though it could change our ability to + # access them. + # + # It seems like the right thing to do is to cancel + # monitoring the directory and then begin monitoring + # it again. But the current FileMonitor class doesn't + # support canceling, so at least let the user know + # that a restart might be a good idea. + logger.warn("Directory properties for %s changed, please " + + " consider restarting the server" % (abspath)) + else: + logger.warn("Got unknown dir event %s %s %s" % (event.requestID, + event.code2str(), + abspath)) else: - print("Got unknown event %s %s %s" % (event.requestID, - event.code2str(), - event.filename)) + # Deal with events for non-directories + if action in ['exists', 'created']: + if ((event.filename[-1] == '~') or + (event.filename[:2] == '.#') or + (event.filename[-4:] == '.swp') or + (event.filename in ['SCCS', '.svn'])): + return + if not self.patterns.match(event.filename): + return + self.entries[relpath] = self.__child__('%s/%s' % (self.name, + relpath)) + self.entries[relpath].HandleEvent(event) + elif action == 'changed': + if relpath in self.entries: + self.entries[relpath].HandleEvent(event) + else: + logger.warn("Got %s event for unexpected path %s" % (action, abspath)) + else: + logger.warn("Got unknown file event %s %s %s" % (event.requestID, + event.code2str(), + abspath)) class XMLFileBacked(FileBacked): diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 01ad3c78b..bf0c42416 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -3,6 +3,7 @@ __revision__ = '$Revision$' import copy import lxml.etree +import os import re import sys @@ -74,8 +75,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, """Build all structures for client (metadata).""" bundleset = [] for bundlename in metadata.bundles: - entries = [item for (key, item) in list(self.entries.items()) if \ - self.patterns.match(key).group('name') == bundlename] + entries = [item for (key, item) in self.entries.items() if \ + self.patterns.match(os.path.basename(key)).group('name') == bundlename] if len(entries) == 0: continue elif len(entries) == 1: -- cgit v1.2.3-1-g7c22