From 36b2aa66627a4cc147f982d03688ae9df14bbe08 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 24 Jul 2015 00:57:55 +0200 Subject: TemplateHelper: Expire metadata cache on FileMonitor event --- src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 12 ++++++++++-- .../Testlib/TestServer/TestPlugins/TestTemplateHelper.py | 7 +++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index cec2de297..09f3130ca 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -22,9 +22,10 @@ def safe_module_name(module): class HelperModule(Debuggable): """ Representation of a TemplateHelper module """ - def __init__(self, name): + def __init__(self, name, core): Debuggable.__init__(self) self.name = name + self.core = core #: The name of the module as used by get_additional_data(). #: the name of the file with .py stripped off. @@ -51,6 +52,10 @@ class HelperModule(Debuggable): if event and event.code2str() not in ['exists', 'changed', 'created']: return + # expire the metadata cache, because the module might have changed + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + try: module = imp.load_source(safe_module_name(self._module_name), self.name) @@ -107,7 +112,6 @@ class TemplateHelper(Plugin, Connector, DirectoryBacked, TemplateDataProvider): __author__ = 'chris.a.st.pierre@gmail.com' ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.py[co])$') patterns = MODULE_RE - __child__ = HelperModule def __init__(self, core): Plugin.__init__(self, core) @@ -115,6 +119,10 @@ class TemplateHelper(Plugin, Connector, DirectoryBacked, TemplateDataProvider): DirectoryBacked.__init__(self, self.data) TemplateDataProvider.__init__(self) + # The HelperModule needs access to the core, so we have to construct + # it manually and add the custom argument. + self.__child__ = lambda fname: HelperModule(fname, core) + def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 for h in self.entries.values()]) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py index 128d6cae5..6f5ee18ba 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestTemplateHelper.py @@ -22,10 +22,13 @@ class TestHelperModule(Bcfg2TestCase): test_obj = HelperModule path = os.path.join(datastore, "test.py") - def get_obj(self, path=None): + def get_obj(self, path=None, core=None): if path is None: path = self.path - return self.test_obj(path) + if core is None: + core = Mock() + core.metadata_cache_mode = 'none' + return self.test_obj(path, core) def test__init(self): hm = self.get_obj() -- cgit v1.2.3-1-g7c22 From f4d9a230d237a9b412e8056629ec1dee1ce478c3 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 24 Jul 2015 01:06:21 +0200 Subject: Ohai: Expire metadata cache, if ohai data changes --- src/lib/Bcfg2/Server/Plugins/Ohai.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 461be9ba8..b314e60a0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -94,7 +94,12 @@ class Ohai(Bcfg2.Server.Plugin.Plugin, return [self.probe] def ReceiveData(self, meta, datalist): - self.cache[meta.hostname] = datalist[0].text + if meta.hostname not in self.cache or \ + self.cache[meta.hostname] != datalist[0].text: + self.cache[meta.hostname] = datalist[0].text + + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire(meta.hostname) def get_additional_data(self, meta): if meta.hostname in self.cache: -- cgit v1.2.3-1-g7c22 From f2acd697d69351be036ffb089b8244accfe8ae06 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 24 Jul 2015 01:13:47 +0200 Subject: SSHbase: Expire metadata cache on pubkey event SSHbase supply the public keyfiles as additional metadata, so we need to expire the metadata cache if we get an event for such file. We would only need to expire the metadata cache for hosts, that reference that exact file (host or group specific) but we cannot get this information, so that we simply expire the metadata cache for all hosts. --- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 7736bd050..7f20e72eb 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -286,6 +286,10 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.debug_log("New public key %s; invalidating " "ssh_known_hosts cache" % event.filename) self.skn = False + + if self.core.metadata_cache_mode in ['cautious', + 'aggressive']: + self.core.metadata_cache.expire() return if event.filename == 'info.xml': -- cgit v1.2.3-1-g7c22 From d657ae4a8c2e1e60ff98c95f7939c3ec22ffe5d1 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 24 Jul 2015 01:37:47 +0200 Subject: GroupLogic: Expire metadata cache on config file changes --- src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index b60f60e65..184b362f9 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -13,6 +13,17 @@ class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): create = lxml.etree.Element("GroupLogic", nsmap=dict(py="http://genshi.edgewall.org/")) + def __init__(self, filename, core): + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, + should_monitor=True) + self.core = core + + def Index(self): + Bcfg2.Server.Plugin.StructFile.Index(self) + + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + def _match(self, item, metadata, *args): if item.tag == 'Group' and not len(item.getchildren()): return [item] @@ -39,7 +50,7 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Plugin.__init__(self, core) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), - should_monitor=True) + core=core) self._local = local() def get_additional_groups(self, metadata): -- cgit v1.2.3-1-g7c22 From 6af6e2de346b37b92a9ffc3f2a7b874c42332397 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 24 Jul 2015 02:20:41 +0200 Subject: PuppetENC: Fix cache expiration This was maybe missing in 38f3cfcfd. --- src/lib/Bcfg2/Server/Plugins/PuppetENC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 59fbe6f03..e2d8a058f 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -117,7 +117,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.warning("PuppetENC is incompatible with aggressive " "client metadata caching, try 'cautious' or " "'initial' instead") - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.core.metadata_cache.expire() def end_statistics(self, metadata): self.end_client_run(self, metadata) -- cgit v1.2.3-1-g7c22 From e32136ac6e2d96f1e1dd36a16db628b0f686ac7c Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 24 Jul 2015 02:46:36 +0200 Subject: AWSTags: Expire metadata cache at start of client run AWSTags cannot know whether the tags change, so it cannot expire the metadata cache correctly. Instead we have to expire the cache at beginning of each client run. This practically disables agressive client metadata caching, so we print a warning. --- src/lib/Bcfg2/Server/Plugins/AWSTags.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py index 0d6eefaaa..556805bde 100644 --- a/src/lib/Bcfg2/Server/Plugins/AWSTags.py +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -172,6 +172,11 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, def start_client_run(self, metadata): self.expire_cache(key=metadata.hostname) + if self.core.metadata_cache_mode == 'aggressive': + self.logger.warning("AWSTags is incompatible with aggressive " + "client metadata caching, try 'cautious' " + "or 'initial'") + self.core.metadata_cache.expire(metadata.hostname) def get_additional_data(self, metadata): return self.get_tags(metadata) -- cgit v1.2.3-1-g7c22 From ecca2c39b9ace4d46624edba347fe5d77954dc2c Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 24 Jul 2015 07:12:58 +0200 Subject: Probes: Use core.metadata_cache instead of Bcfg2.Server.Cache --- src/lib/Bcfg2/Server/Plugins/Probes.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 76aab69b5..573c9af71 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -74,6 +74,7 @@ class ProbeStore(Debuggable): def __init__(self, core, datadir): # pylint: disable=W0613 Debuggable.__init__(self) + self.core = core self._groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") self._datacache = Bcfg2.Server.Cache.Cache("Probes", "probedata") @@ -134,7 +135,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) groupdata = ProbesGroupsModel.objects.filter(hostname=hostname) self._groupcache[hostname] = list(set(r.group for r in groupdata)) - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock def set_groups(self, hostname, groups): @@ -155,7 +156,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): ProbesGroupsModel.objects.filter( hostname=hostname).exclude(group__in=groups).delete() if olddata != groups: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) def _load_data(self, hostname): Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) @@ -168,7 +169,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): time.mktime(pdata.timestamp.timetuple()) ts_set = True self._datacache[hostname][pdata.probe] = ProbeData(pdata.data) - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock def set_data(self, hostname, data): @@ -198,7 +199,7 @@ class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): qset.delete() expire_metadata = True if expire_metadata: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) class XMLProbeStore(ProbeStore): @@ -234,7 +235,7 @@ class XMLProbeStore(ProbeStore): self._groupcache[client.get('name')].append( pdata.get('name')) - Bcfg2.Server.Cache.expire("Metadata") + self.core.metadata_cache.expire() def _load_groups(self, hostname): self._load_data(hostname) @@ -274,7 +275,7 @@ class XMLProbeStore(ProbeStore): olddata = self._groupcache.get(hostname, []) self._groupcache[hostname] = groups if olddata != groups: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) def set_data(self, hostname, data): Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) @@ -285,7 +286,7 @@ class XMLProbeStore(ProbeStore): self._datacache[hostname][probe] = pdata expire_metadata |= olddata != data if expire_metadata: - Bcfg2.Server.Cache.expire("Metadata", hostname) + self.core.metadata_cache.expire(hostname) class ClientProbeDataSet(dict): -- cgit v1.2.3-1-g7c22 From 08a61d59c985f917dcf485e090b7f2fadbb42f70 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 27 Jul 2015 14:11:17 +0200 Subject: Properties: Expire the metadata cache during reload of the files --- src/lib/Bcfg2/Server/Plugins/Properties.py | 41 ++++++++++++++++------ .../TestServer/TestPlugins/TestProperties.py | 15 ++++++-- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index c4dd75e60..e6549b714 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -35,13 +35,17 @@ LOGGER = logging.getLogger(__name__) class PropertyFile(object): """ Base Properties file handler """ - def __init__(self, name): + def __init__(self, name, core): """ :param name: The filename of this properties file. + :type name: string + :param core: The Bcfg2.Server.Core initializing the Properties plugin + :type core: Bcfg2.Server.Core .. automethod:: _write """ self.name = name + self.core = core def write(self): """ Write the data in this data structure back to the property @@ -69,6 +73,12 @@ class PropertyFile(object): file. """ raise NotImplementedError + def _expire_metadata_cache(self): + """ Expires the metadata cache, if it is required by the caching + mode. """ + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.metadata_cache.expire() + def validate_data(self): """ Verify that the data in this file is valid. """ raise NotImplementedError @@ -81,9 +91,9 @@ class PropertyFile(object): class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """Handle JSON Properties files.""" - def __init__(self, name): + def __init__(self, name, core): Bcfg2.Server.Plugin.FileBacked.__init__(self, name) - PropertyFile.__init__(self, name) + PropertyFile.__init__(self, name, core) self.json = None def Index(self): @@ -93,10 +103,13 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): err = sys.exc_info()[1] raise PluginExecutionError("Could not load JSON data from %s: %s" % (self.name, err)) + self._expire_metadata_cache() + Index.__doc__ = Bcfg2.Server.Plugin.FileBacked.Index.__doc__ def _write(self): json.dump(self.json, open(self.name, 'wb')) return True + _write.__doc__ = PropertyFile._write.__doc__ def validate_data(self): try: @@ -105,6 +118,7 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): err = sys.exc_info()[1] raise PluginExecutionError("Data for %s cannot be dumped to JSON: " "%s" % (self.name, err)) + validate_data.__doc__ = PropertyFile.validate_data.__doc__ def __str__(self): return str(self.json) @@ -116,11 +130,10 @@ class JSONPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): """ Handle YAML Properties files. """ - def __init__(self, name): + def __init__(self, name, core): Bcfg2.Server.Plugin.FileBacked.__init__(self, name) - PropertyFile.__init__(self, name) + PropertyFile.__init__(self, name, core) self.yaml = None - __init__.__doc__ = Bcfg2.Server.Plugin.FileBacked.__init__.__doc__ def Index(self): try: @@ -129,6 +142,7 @@ class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): err = sys.exc_info()[1] raise PluginExecutionError("Could not load YAML data from %s: %s" % (self.name, err)) + self._expire_metadata_cache() Index.__doc__ = Bcfg2.Server.Plugin.FileBacked.Index.__doc__ def _write(self): @@ -155,10 +169,15 @@ class YAMLPropertyFile(Bcfg2.Server.Plugin.FileBacked, PropertyFile): class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): """ Handle XML Properties files. """ - def __init__(self, name, should_monitor=False): + def __init__(self, name, core, should_monitor=False): Bcfg2.Server.Plugin.StructFile.__init__(self, name, should_monitor=should_monitor) - PropertyFile.__init__(self, name) + PropertyFile.__init__(self, name, core) + + def Index(self): + Bcfg2.Server.Plugin.StructFile.Index(self) + self._expire_metadata_cache() + Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ def _write(self): open(self.name, "wb").write( @@ -258,11 +277,11 @@ class Properties(Bcfg2.Server.Plugin.Plugin, :class:`PropertyFile` """ if fname.endswith(".xml"): - return XMLPropertyFile(fname) + return XMLPropertyFile(fname, self.core) elif HAS_JSON and fname.endswith(".json"): - return JSONPropertyFile(fname) + return JSONPropertyFile(fname, self.core) elif HAS_YAML and (fname.endswith(".yaml") or fname.endswith(".yml")): - return YAMLPropertyFile(fname) + return YAMLPropertyFile(fname, self.core) else: raise Bcfg2.Server.Plugin.PluginExecutionError( "Properties: Unknown extension %s" % fname) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py index 36baee899..8c7f3c5d7 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py @@ -29,11 +29,14 @@ class TestPropertyFile(Bcfg2TestCase): test_obj = PropertyFile path = os.path.join(datastore, "test") - def get_obj(self, path=None): + def get_obj(self, path=None, core=None, *args, **kwargs): set_setup_default("writes_enabled", False) if path is None: path = self.path - return self.test_obj(path) + if core is None: + core = Mock() + core.metadata_cache_mode = 'none' + return self.test_obj(path, core, *args, **kwargs) def test_write(self): pf = self.get_obj() @@ -97,6 +100,9 @@ class TestJSONPropertyFile(TestFileBacked, TestPropertyFile): TestFileBacked.setUp(self) TestPropertyFile.setUp(self) + def get_obj(self, *args, **kwargs): + return TestPropertyFile.get_obj(self, *args, **kwargs) + @patch("%s.loads" % JSON) def test_Index(self, mock_loads): pf = self.get_obj() @@ -137,6 +143,9 @@ class TestYAMLPropertyFile(TestFileBacked, TestPropertyFile): TestFileBacked.setUp(self) TestPropertyFile.setUp(self) + def get_obj(self, *args, **kwargs): + return TestPropertyFile.get_obj(self, *args, **kwargs) + @patch("yaml.load") def test_Index(self, mock_load): pf = self.get_obj() @@ -179,7 +188,7 @@ class TestXMLPropertyFile(TestPropertyFile, TestStructFile): set_setup_default("automatch", False) def get_obj(self, *args, **kwargs): - return TestStructFile.get_obj(self, *args, **kwargs) + return TestPropertyFile.get_obj(self, *args, **kwargs) @patch("%s.open" % builtins) def test__write(self, mock_open): -- cgit v1.2.3-1-g7c22 From 3d2fca1e8573d60e67bb2fe642fb78c517d5ccc2 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 27 Jul 2015 14:15:42 +0200 Subject: doc: Fix docs about caching The data returned by Connector.get_additional_data is also cached, so the plugin needs to invalidate the caches, even if it does not implement Connector.get_additional_groups. --- doc/development/plugins.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/development/plugins.txt b/doc/development/plugins.txt index d292c9dd7..0d524973e 100644 --- a/doc/development/plugins.txt +++ b/doc/development/plugins.txt @@ -118,7 +118,9 @@ Invalidating Caches In Bcfg2 1.3.0, some limited :ref:`server-caching` was introduced. If you are writing a :class:`Bcfg2.Server.Plugin.interfaces.Connector` plugin that implements -:func:`Bcfg2.Server.Plugin.interfaces.Connector.get_additional_groups`, +:func:`Bcfg2.Server.Plugin.interfaces.Connector.get_additional_groups` +or +:func:`Bcfg2.Server.Plugin.interfaces.Connector.get_additional_data`, then you need to be able to invalidate the server metadata cache in order to be compatible with the ``cautious`` or ``aggressive`` caching modes. @@ -140,9 +142,9 @@ called with one string argument, it expires cached data for the named client. It's important, therefore, that your Connector plugin can either track -when changes are made to the group membership it reports, and expire -cached data appropriately when in ``cautious`` or ``aggressive`` mode; -or prudently flag an incompatibility with those two modes. +when changes are made to the data or group membership it reports, and +expire cached data appropriately when in ``cautious`` or ``aggressive`` +mode; or prudently flag an incompatibility with those two modes. For examples, see: -- cgit v1.2.3-1-g7c22