From be564316c7c4deaad090bfc0bc79c460965cb1d6 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 25 Nov 2014 00:51:23 +0100 Subject: Bundler: add modification support to Bundle dependencies Bundle dependencies are now realized with RequiredBundle and support inheritance of the modification flag. This requires new client support and will only work with clients >= 1.4.0pre2. --- doc/server/plugins/structures/bundler/bcfg2.txt | 2 +- doc/server/plugins/structures/bundler/index.txt | 24 +++++---- schemas/bundle.xsd | 36 +++++++++++++ src/lib/Bcfg2/Client/Tools/BundleDeps.py | 34 +++++++++++++ src/lib/Bcfg2/Client/__init__.py | 59 +++++++++++++++++----- src/lib/Bcfg2/Server/Plugins/Bundler.py | 27 ++++++++-- .../Testlib/TestServer/TestPlugins/TestBundler.py | 2 +- 7 files changed, 156 insertions(+), 28 deletions(-) create mode 100644 src/lib/Bcfg2/Client/Tools/BundleDeps.py diff --git a/doc/server/plugins/structures/bundler/bcfg2.txt b/doc/server/plugins/structures/bundler/bcfg2.txt index 0fd0a3fdf..6d1159ae0 100644 --- a/doc/server/plugins/structures/bundler/bcfg2.txt +++ b/doc/server/plugins/structures/bundler/bcfg2.txt @@ -16,7 +16,7 @@ entries between Bundler and Rules. .. code-block:: xml - + diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 31faeaf17..e6a9cf345 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -151,20 +151,20 @@ See :ref:`bcfg2-info ` for more details. Dependencies ============ -Dependencies on other bundles can be specified by adding an empty -bundle tag that adds another bundle by name, e.g.: +Dependencies on other bundles can be specified by adding an +RequiredBundle tag that adds another bundle by name, e.g.: .. code-block:: xml - + ... The dependent bundle is added to the list of bundles sent to the -client, *not* to the parent bundle itself. In other words, if an -entry in the dependent bundle changes, Services are restarted and -Actions are run in the dependent bundle *only*. An example: +client, *not* to the parent bundle itself. If you want to propagate +the modification flag from the required bundle, you can add +``modification="inherit"`` to the RequiredBundle tag. An example: ``nfs-client.xml``: @@ -182,7 +182,7 @@ Actions are run in the dependent bundle *only*. An example: .. code-block:: xml - + @@ -193,9 +193,13 @@ Actions are run in the dependent bundle *only*. An example: If a new ``nfs-utils`` package was installed, the ``nfslock``, ``rpcbind``, and ``nfs`` services would be restarted, but *not* the -``autofs`` service. Similarly, if a new ``/etc/auto.misc`` file was -sent out, the ``autofs`` service would be restarted, but the -``nfslock``, ``rpcbind``, and ``nfs`` services would not be restarted. +``autofs`` service. If you would add ``modification="inherit`` to +the RequiredBundle tag, you would ensure the propagation of the +modification flag and the ``autofs`` service would be restarted, +too. But if a new ``/etc/auto.misc`` file was sent out, *only* the +``autofs`` service would be restarted, but the ``nfslock``, +``rpcbind``, and ``nfs`` services would not be restarted +(independent of the ``modification`` flag). Altsrc ====== diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd index aeacd0517..b6f9e00af 100644 --- a/schemas/bundle.xsd +++ b/schemas/bundle.xsd @@ -263,6 +263,13 @@ + + + + Nesting Bundle tags to specify dependencies to other bundles. + + + @@ -300,6 +307,35 @@ + + + + + + + + + + + + The name of the required bundle. + + + + + + + Specify how to handle modifications in the required + bundle. You can either ignore the modifications (this + is the default) or you can inherit the modifications + so that Services in the current Bundle are restarted + if the required Bundle is modified. + + + + + + diff --git a/src/lib/Bcfg2/Client/Tools/BundleDeps.py b/src/lib/Bcfg2/Client/Tools/BundleDeps.py new file mode 100644 index 000000000..aaa090633 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/BundleDeps.py @@ -0,0 +1,34 @@ +""" Bundle dependency support """ + +import Bcfg2.Client.Tools + + +class BundleDeps(Bcfg2.Client.Tools.Tool): + """Bundle dependency helper for Bcfg2. It handles Bundle tags inside the + bundles that references the required other bundles that should change the + modification status if the referenced bundles is modified.""" + + name = 'Bundle' + __handles__ = [('Bundle', None)] + __req__ = {'Bundle': ['name']} + + def InstallBundle(self, _): + """Simple no-op because we only need the BundleUpdated hook.""" + return dict() + + def VerifyBundle(self, entry, _): # pylint: disable=W0613 + """Simple no-op because we only need the BundleUpdated hook.""" + return True + + def BundleUpdated(self, entry): + """This handles the dependencies on this bundle. It searches all + Bundle tags in other bundles that references the current bundle name + and marks those tags as modified to trigger the modification hook on + the other bundles.""" + + bundle_name = entry.get('name') + for bundle in self.config.findall('./Bundle/Bundle'): + if bundle.get('name') == bundle_name and \ + bundle not in self.modified: + self.modified.append(bundle) + return dict() diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 5f4f15dcc..d834576c9 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -768,27 +768,27 @@ class Client(object): if not Bcfg2.Options.setup.interactive: self.DispatchInstallCalls(clobbered) - for bundle in self.config.findall('.//Bundle'): + all_bundles = self.config.findall('./Bundle') + mbundles.extend(self._get_all_modified_bundles(mbundles, all_bundles)) + + for bundle in all_bundles: if (Bcfg2.Options.setup.only_bundles and bundle.get('name') not in Bcfg2.Options.setup.only_bundles): # prune out unspecified bundles when running with -b continue if bundle in mbundles: - self.logger.debug("Bundle %s was modified" % - bundle.get('name')) - func = "BundleUpdated" - else: - self.logger.debug("Bundle %s was not modified" % - bundle.get('name')) - func = "BundleNotUpdated" + continue + + self.logger.debug("Bundle %s was not modified" % + bundle.get('name')) for tool in self.tools: try: - self.states.update(getattr(tool, func)(bundle)) + self.states.update(tool.BundleNotUpdated(bundle)) except: # pylint: disable=W0702 - self.logger.error("%s.%s(%s:%s) call failed:" % - (tool.name, func, bundle.tag, - bundle.get("name")), exc_info=1) + self.logger.error('%s.BundleNotUpdated(%s:%s) call failed:' + % (tool.name, bundle.tag, + bundle.get('name')), exc_info=1) for indep in self.config.findall('.//Independent'): for tool in self.tools: @@ -799,6 +799,41 @@ class Client(object): % (tool.name, indep.tag, indep.get("name")), exc_info=1) + def _get_all_modified_bundles(self, mbundles, all_bundles): + """This gets all modified bundles by calling BundleUpdated until no + new bundles get added to the modification list.""" + new_mbundles = mbundles + add_mbundles = [] + + while new_mbundles: + for bundle in self.config.findall('./Bundle'): + if (Bcfg2.Options.setup.only_bundles and + bundle.get('name') not in + Bcfg2.Options.setup.only_bundles): + # prune out unspecified bundles when running with -b + continue + if bundle not in new_mbundles: + continue + + self.logger.debug('Bundle %s was modified' % + bundle.get('name')) + for tool in self.tools: + try: + self.states.update(tool.BundleUpdated(bundle)) + except: # pylint: disable=W0702 + self.logger.error('%s.BundleUpdated(%s:%s) call ' + 'failed:' % (tool.name, bundle.tag, + bundle.get("name")), + exc_info=1) + + mods = self.modified + new_mbundles = [struct for struct in all_bundles + if any(True for mod in mods if mod in struct) + and struct not in mbundles + add_mbundles] + add_mbundles.extend(new_mbundles) + + return add_mbundles + def Remove(self): """Remove extra entries.""" for tool in self.tools: diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 41ee57b6d..4945bf85b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -8,6 +8,7 @@ import fnmatch import lxml.etree from Bcfg2.Server.Plugin import StructFile, Plugin, Structure, \ StructureValidator, XMLDirectoryBacked, Generator +from Bcfg2.version import Bcfg2VersionInfo from genshi.template import TemplateError @@ -116,17 +117,35 @@ class Bundler(Plugin, for el in child.getchildren(): data.append(el) data.remove(child) - elif child.get("name"): + else: + # no children -- wat + self.logger.warning("Bundler: Useless empty Bundle tag " + "in %s" % self.name) + data.remove(child) + + for child in data.findall('RequiredBundle'): + if child.get("name"): # dependent bundle -- add it to the list of # bundles for this client if child.get("name") not in bundles_added: bundles.append(child.get("name")) bundles_added.add(child.get("name")) + if child.get('modification', 'ignore') == 'inherit': + if metadata.version_info >= \ + Bcfg2VersionInfo('1.4.0pre2'): + lxml.etree.SubElement(data, 'Bundle', + name=child.get('name')) + else: + self.logger.warning( + 'Bundler: modification="inherit" is only ' + 'supported for clients starting 1.4.0pre2') data.remove(child) else: - # neither name or children -- wat - self.logger.warning("Bundler: Useless empty Bundle tag " - "in %s" % self.name) + # no name -- wat + self.logger.warning('Bundler: Missing required name in ' + 'RequiredBundle tag in %s' % + self.name) data.remove(child) + bundleset.append(data) return bundleset diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py index cfb379c40..dbed50ddb 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py @@ -74,7 +74,7 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked): lxml.etree.SubElement(expected['xinclude'], "Path", name="/test") has_dep = lxml.etree.Element("Bundle") - lxml.etree.SubElement(has_dep, "Bundle", name="is_dep") + lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_dep") lxml.etree.SubElement(has_dep, "Package", name="foo") b.bundles['has_dep'].XMLMatch.return_value = has_dep expected['has_dep'] = lxml.etree.Element("Bundle", name="has_dep") -- cgit v1.2.3-1-g7c22 From 84b180e2fddcb5563adca33ed6feb3fd991dfde2 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 25 Nov 2014 16:14:43 +0100 Subject: testsuite: add test for modification inheritance --- .../Testlib/TestServer/TestPlugins/TestBundler.py | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py index dbed50ddb..db2a91227 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py @@ -3,6 +3,7 @@ import sys import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Bundler import * +from Bcfg2.version import Bcfg2VersionInfo # add all parent testsuite directories to sys.path to allow (most) # relative imports in python 2.4 @@ -75,10 +76,14 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked): has_dep = lxml.etree.Element("Bundle") lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_dep") + lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_mod_dep", + modification="inherit") lxml.etree.SubElement(has_dep, "Package", name="foo") b.bundles['has_dep'].XMLMatch.return_value = has_dep expected['has_dep'] = lxml.etree.Element("Bundle", name="has_dep") lxml.etree.SubElement(expected['has_dep'], "Package", name="foo") + lxml.etree.SubElement(expected['has_dep'], "Bundle", + name="is_mod_dep") is_dep = lxml.etree.Element("Bundle") lxml.etree.SubElement(is_dep, "Package", name="bar") @@ -94,6 +99,7 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked): metadata = Mock() metadata.bundles = ["error", "xinclude", "has_dep", "indep"] + metadata.version_info = Bcfg2VersionInfo('1.4.0') rv = b.BuildStructures(metadata) self.assertEqual(len(rv), 4) @@ -109,3 +115,33 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked): b.bundles['error'].XMLMatch.assert_called_with(metadata) self.assertFalse(b.bundles['skip'].XMLMatch.called) + + def test_BuildStructuresOldClient(self): + b = self.get_obj() + b.bundles = dict(has_dep=Mock()) + expected = dict() + + has_dep = lxml.etree.Element("Bundle") + lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_dep") + lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_mod_dep", + modification="inherit") + lxml.etree.SubElement(has_dep, "Package", name="foo") + b.bundles['has_dep'].XMLMatch.return_value = has_dep + expected['has_dep'] = lxml.etree.Element("Bundle", name="has_dep") + lxml.etree.SubElement(expected['has_dep'], "Package", name="foo") + + metadata = Mock() + metadata.bundles = ["has_dep"] + metadata.version_info = Bcfg2VersionInfo('1.3.0') + + rv = b.BuildStructures(metadata) + self.assertEqual(len(rv), len(metadata.bundles)) + for bundle in rv: + name = bundle.get("name") + self.assertIsNotNone(name, + "Bundle %s was not built" % name) + self.assertIn(name, expected, + "Unexpected bundle %s was built" % name) + self.assertXMLEqual(bundle, expected[name], + "Bundle %s was not built correctly" % name) + b.bundles[name].XMLMatch.assert_called_with(metadata) -- cgit v1.2.3-1-g7c22 From d4ae5e04739d9a8e0732dd35ee28c14b0ff96957 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 25 Nov 2014 17:56:16 +0100 Subject: Bundler: modification is now the boolean inherit_modification --- doc/server/plugins/structures/bundler/index.txt | 9 +++++---- schemas/bundle.xsd | 9 +-------- src/lib/Bcfg2/Server/Plugins/Bundler.py | 7 ++++--- testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py | 4 ++-- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index e6a9cf345..e8f609e09 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -164,7 +164,8 @@ RequiredBundle tag that adds another bundle by name, e.g.: The dependent bundle is added to the list of bundles sent to the client, *not* to the parent bundle itself. If you want to propagate the modification flag from the required bundle, you can add -``modification="inherit"`` to the RequiredBundle tag. An example: +``inherit_modification="true"`` to the RequiredBundle tag. +An example: ``nfs-client.xml``: @@ -193,13 +194,13 @@ the modification flag from the required bundle, you can add If a new ``nfs-utils`` package was installed, the ``nfslock``, ``rpcbind``, and ``nfs`` services would be restarted, but *not* the -``autofs`` service. If you would add ``modification="inherit`` to -the RequiredBundle tag, you would ensure the propagation of the +``autofs`` service. If you would add ``inherit_modification="true"`` +to the RequiredBundle tag, you would ensure the propagation of the modification flag and the ``autofs`` service would be restarted, too. But if a new ``/etc/auto.misc`` file was sent out, *only* the ``autofs`` service would be restarted, but the ``nfslock``, ``rpcbind``, and ``nfs`` services would not be restarted -(independent of the ``modification`` flag). +(independent of the ``inherit_modification`` flag). Altsrc ====== diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd index b6f9e00af..4a11a1d1b 100644 --- a/schemas/bundle.xsd +++ b/schemas/bundle.xsd @@ -307,13 +307,6 @@ - - - - - - - @@ -322,7 +315,7 @@ - + Specify how to handle modifications in the required diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 4945bf85b..6c35ada59 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -130,15 +130,16 @@ class Bundler(Plugin, if child.get("name") not in bundles_added: bundles.append(child.get("name")) bundles_added.add(child.get("name")) - if child.get('modification', 'ignore') == 'inherit': + if child.get('inherit_modification', 'false') == 'true': if metadata.version_info >= \ Bcfg2VersionInfo('1.4.0pre2'): lxml.etree.SubElement(data, 'Bundle', name=child.get('name')) else: self.logger.warning( - 'Bundler: modification="inherit" is only ' - 'supported for clients starting 1.4.0pre2') + 'Bundler: inherit_modification="true" is ' + 'only supported for clients starting ' + '1.4.0pre2') data.remove(child) else: # no name -- wat diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py index db2a91227..1bf208c3e 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py @@ -77,7 +77,7 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked): has_dep = lxml.etree.Element("Bundle") lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_dep") lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_mod_dep", - modification="inherit") + inherit_modification="true") lxml.etree.SubElement(has_dep, "Package", name="foo") b.bundles['has_dep'].XMLMatch.return_value = has_dep expected['has_dep'] = lxml.etree.Element("Bundle", name="has_dep") @@ -124,7 +124,7 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked): has_dep = lxml.etree.Element("Bundle") lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_dep") lxml.etree.SubElement(has_dep, "RequiredBundle", name="is_mod_dep", - modification="inherit") + inherit_modification="true") lxml.etree.SubElement(has_dep, "Package", name="foo") b.bundles['has_dep'].XMLMatch.return_value = has_dep expected['has_dep'] = lxml.etree.Element("Bundle", name="has_dep") -- cgit v1.2.3-1-g7c22