Bundler: added support for independent bundles

8 files changed, 270 insertions, 195 deletions

diff --git a/doc/server/plugins/structures/base.txt b/doc/server/plugins/structures/base.txt
deleted file mode 100644
index 03eae0573..000000000
--- a/doc/server/plugins/structures/base.txt
+++ /dev/null
@@ -1,83 +0,0 @@
-.. -*- mode: rst -*-
-
-.. _server-plugins-structures-base:
-
-====
-Base
-====
-
-.. deprecated:: 1.2.0
-
-.. warning::
-
-    The Base plugin no longer receives new features/functionality.
-    Please use :ref:`server-plugins-structures-bundler-index` instead.
-
-The Base plugin is a structure plugin that provides the ability to add
-lists of unrelated entries into client configuration entry inventories.
-
-Base works much like Bundler in its file format. The main difference
-between Base and Bundler is that Base files are included in all clients'
-configuration whereas bundles must be included explicitly in your
-Metadata. See the :ref:`server-plugins-structures-bundler-index` page
-for details.
-
-If you have lots of unconnected items (for instance: software packages
-whose configuration wasn't modified, and that are also not depended
-on by other packages; or single directories or files not belonging
-to a package), using Bundles in Metadata would clutter or enlarge
-your ``Metadata/groups.xml`` file, because they all would need to be
-explicitly specified. ``Base/`` on the other hand is the perfect place
-to put these items.
-
-Without using Base, you would be forced to put them directly
-into your group definitions in ``groups.xml``, either as many
-small bundles (substantially enlarging it) or into something like
-``Bundler/unrelated-entries.xml``. Using the latter is especially bad
-if you mix packages and services in your Bundle, since for any updated
-package in that bundle, the now-related services would be restarted.
-
-The Base entries can still be assigned based on group membership, but when
-they aren't part of a group, each and every client gets the entry. So Base is
-also a great place to put entries that a large number of your clients will
-get.
-
-For example, you could have a file ``Base/packages.xml``
-
-.. code-block:: xml
-
-    <Base>
-        <Package name='acpid'/>
-        <Package name='auditd'/>
-        [...]
-        <Group name='openSUSE11.2'>
-            <Package name='syslog-ng'/>
-        </Group>
-        <Group name='openSUSE11.3'>
-            <Package name='rsyslog'/>
-        </Group>
-        [...]
-        <Package name='zlib'/>
-    </Base>
-
-.. note::
-
-    You don't have to reference to the files in Base from anywhere. As long
-    as you include ``Base`` in your ``plugins = ...`` line in ``bcfg2.conf``,
-    these are included automatically.
-
-.. note::
-
-    Your Base files have to match the pattern ``Base/*.xml`` to be included.
-
-
-The decision when to use Base and when to use Bundler depends on the
-configuration entry in question, and what you are trying to achieve.
-
-Base is mainly used for cases where you don't want/need to explicitly
-include particular configuration items. Let's say all your machines are
-various linux distributions. In this case, you may want to manage the
-``/etc/hosts`` file using Base instead of Bundler since you will not have
-to include any Bundles in your Metadata. However, you could alternatively
-have a base 'linux' group that all the clients inherit which includes a
-*linux* Bundle with the ``/etc/hosts`` configuration entry.
diff --git a/doc/server/plugins/structures/bundler/bcfg2.txt b/doc/server/plugins/structures/bundler/bcfg2.txt
new file mode 100644
index 000000000..7465f15cb
--- /dev/null
+++ b/doc/server/plugins/structures/bundler/bcfg2.txt
@@ -0,0 +1,88 @@
+.. -*- mode: rst -*-
+
+.. _server-plugins-structures-bundler-bcfg2-server:
+
+Bcfg2 Server
+============
+
+These two bundles split out the entries that do require a restart of
+``bcfg2-server`` from those that don't.
+
+These bundles also demonstrate use of bound entries to avoid splitting
+entries between Bundler and Rules.
+
+``Bundler/bcfg2-server.xml``:
+
+.. code-block:: xml
+
+  <Bundle>
+    <Bundle name="bcfg2-server-base.xml"/>
+
+    <Path name="/etc/pki/tls/private/bcfg2.key"/>
+    <Path name="/etc/sysconfig/bcfg2-server"/>
+    <Path name="/etc/bcfg2.conf"/>
+
+    <BoundPath name="/var/lib/bcfg2/Packages/cache" type="directory"
+               owner="bcfg2" group="bcfg2" mode="0755"/>
+    <BoundPath name="/var/lib/bcfg2" type="symlink"
+               to="/var/lib/bcfg2-vcs/bcfg2/public"/>
+    <BoundPath name="/var/lib/bcfg2/etc/bcfg2.sqlite" type="permissions"
+               owner="bcfg2" group="apache" mode="0660"/>
+
+    <BoundService name="bcfg2-server" type="chkconfig" status="on"/>
+
+    <Package name="bcfg2-server"/>
+    <Package name="python-genshi"/>
+    <Package name="python-inotify"/>
+    <Package name="Django"/>
+    <Package name="Django-south"/>
+    <Package name="m2crypto"/>
+    <Package name="GitPython"/>
+  </Bundle>
+
+``Bundler/bcfg2-server-base.xml``:
+
+.. code-block:: xml
+
+  <Bundle>
+    <Path name="/etc/bcfg2-web.conf"/>
+    <Path name="/etc/cron.daily/bcfg2_cleanup_db"/>
+
+    <BoundPOSIXGroup name='bcfg2'/>
+    <BoundPOSIXUser name='bcfg2' shell='/sbin/nologin' gecos='Bcfg2 User'/>
+    <Path name="/home/bcfg2/.ssh/id_rsa"/>
+
+    <!-- SSLCA setup -->
+    <BoundPath name="/etc/pki/CA" type="directory" important="true"
+               owner="bcfg2" group="bcfg2" mode="755"/>
+    <BoundPath name="/etc/pki/CA/crl" type="directory" owner="bcfg2"
+               group="bcfg2" mode="755"/>
+    <BoundPath name="/etc/pki/CA/certs" type="directory" owner="bcfg2"
+               group="bcfg2" mode="755"/>
+    <BoundPath name="/etc/pki/CA/newcerts" type="directory" owner="bcfg2"
+               group="bcfg2" mode="755"/>
+    <BoundPath name="/etc/pki/CA/private" type="directory" owner="bcfg2"
+               group="bcfg2" mode="755"/>
+    <Path name="/etc/pki/CA/openssl.cnf" altsrc="/etc/pki/CA/openssl.cnf"/>
+    <Path name="/etc/pki/CA/index.txt.attr"/>
+    <Path name="/etc/pki/CA/CA.crt"/>
+    <Path name="/etc/pki/CA/CA.key"/>
+    <Path name="/etc/pki/CA/CA.pem"/>
+    <Path name="/etc/pki/tls/certs/server-chain.crt"/>
+    <BoundPath name="/etc/pki/CA/serial" type="permissions" owner="bcfg2"
+               group="bcfg2" mode="0600"/>
+    <BoundPath name="/etc/pki/CA/index.txt" type="permissions" owner="bcfg2"
+               group="bcfg2" mode="0600"/>
+    <BoundPath name="/etc/pki/CA/crlnumber" type="permissions" owner="bcfg2"
+               group="bcfg2" mode="0644"/>
+    <BoundAction
+        name="create-CA-serial" timing="post" when="always" status="check"
+        command="[ -e /etc/pki/CA/serial ] || echo '01' > /etc/pki/CA/serial"/>
+    <BoundAction
+        name="create-CA-index" timing="post" when="always" status="check"
+        command="[ -e /etc/pki/CA/index.txt ] || touch /etc/pki/CA/index.txt"/>
+    <BoundAction
+        name="create-CA-crlnumber" timing="post" when="always" status="check"
+        command="[ -e /etc/pki/CA/crlnumber ] || touch /etc/pki/CA/crlnumber"/>
+  </Bundle>
+
diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt
index 8f767ba03..c7dde193f 100644
--- a/doc/server/plugins/structures/bundler/index.txt
+++ b/doc/server/plugins/structures/bundler/index.txt
@@ -20,91 +20,94 @@ Group and Client tags can be used inside of bundles to differentiate
 which entries particular clients will recieve; this is useful for the
 case where entries are named differently across systems; for example,
 one Linux distro may have a package called ``openssh`` while another
-uses the name ``ssh``. See :ref:`xml-group-client-tags` for details.
+uses the name ``ssh``. See :ref:`xml-group-client-tags` for details
+and a longer example.
 
-The following is an annotated copy of a bundle:
+A brief example:
 
 .. code-block:: xml
 
     <Bundle>
-      <Path name='/etc/ssh/ssh_host_dsa_key'/>
-      <Path name='/etc/ssh/ssh_host_rsa_key'/>
-      <Path name='/etc/ssh/ssh_host_dsa_key.pub'/>
-      <Path name='/etc/ssh/ssh_host_rsa_key.pub'/>
-      <Path name='/etc/ssh/ssh_host_key'/>
-      <Path name='/etc/ssh/ssh_host_key.pub'/>
-      <Path name='/etc/ssh/sshd_config'/>
       <Path name='/etc/ssh/ssh_config'/>
-      <Path name='/etc/ssh/ssh_known_hosts'/>
       <Group name='rpm'>
-        <Package name='openssh'/>
-        <Package name='openssh-askpass'/>
         <Service name='sshd'/>
-        <Group name='fedora' >
-           <Group name='fc14' negate='true'>
-             <Package name='openssh-clients'/>
-           </Group>
-           <Package name='openssh-server'/>
-        </Group>
+        <Package name='openssh-server'/>
       </Group>
       <Group name='deb'>
         <Package name='ssh'/>
         <Service name='ssh'/>
       </Group>
-      <Client name='trust.example.com'>
-        <Path name='/etc/ssh/shosts.equiv'/>
-      </Client>
     </Bundle>
 
-In this bundle, most of the entries are common to all systems. Clients
-in group **deb** get one extra package and service, while clients in
-group **rpm** get two extra packages and an extra service. In
-addition, clients in group **fedora** *and* group **rpm** get one
-extra package entries, unless they are not in the **fc14** group, in
-which case, they get an extra package. The client
-**trust.example.com** gets one extra file that is not distributed to
-any other clients.  Notice that this file doesn't describe which
-versions of these entries that clients should get, only that they
-should get them. (Admittedly, this example is slightly contrived, but
-demonstrates how group entries can be used in bundles)
-
-+----------------------------+-------------------------------+
-| Group/Hostname             | Entry                         |
-+============================+===============================+
-| all                        | /etc/ssh/ssh_host_dsa_key     |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/ssh_host_rsa_key     |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/ssh_host_dsa_key.pub |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/ssh_host_rsa_key.pub |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/ssh_host_key         |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/ssh_host_key.pub     |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/sshd_config          |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/ssh_config           |
-+----------------------------+-------------------------------+
-| all                        | /etc/ssh/ssh_known_hosts      |
-+----------------------------+-------------------------------+
-| rpm                        | Package openssh               |
-+----------------------------+-------------------------------+
-| rpm                        | Package openssh-askpass       |
-+----------------------------+-------------------------------+
-| rpm                        | Service sshd                  |
-+----------------------------+-------------------------------+
-| rpm and fedora             | Package openssh-server        |
-+----------------------------+-------------------------------+
-| rpm and fedora and not fc4 | Package openssh-clients       |
-+----------------------------+-------------------------------+
-| deb                        | Package ssh                   |
-+----------------------------+-------------------------------+
-| deb                        | Service ssh                   |
-+----------------------------+-------------------------------+
-| trust.example.com          | /etc/ssh/shosts.equiv         |
-+----------------------------+-------------------------------+
+Note that we do not specify *how* a given entry should be managed,
+only that it should be.  The concrete specification of each entry will
+be provided by a different plugin such as
+:ref:`server-plugins-generators-cfg`,
+:ref:`server-plugins-generators-rules`, or
+:ref:`server-plugins-generators-packages`.
+
+Alternatively, you can use fully-bound entries in Bundler, which has
+various uses.  For instance:
+
+    <Bundle>
+      <Path name='/etc/ssh/ssh_config'/>
+      <Group name='rpm'>
+        <BoundService name='sshd' type="chkconfig" status="on"/>
+        <BoundPackage name='openssh-server' version='5.8p2' type="yum" />
+      </Group>
+      <Group name='deb'>
+        <Package name='ssh'/>
+        <BoundService name='ssh' type="chkconfig" status="on"/>
+      </Group>
+    </Bundle>
+
+In this example, both Service tags and one Package tag are fully bound
+-- i.e., all information required by the client to manage those
+entries is provided in the bundle itself.
+
+.. _server-plugins-structures-bundler-magic:
+
+Bundle "Magic"
+==============
+
+Bundles are collections of *related* entries.  That point is very,
+very important, because a bundle performs certain "magic" actions when
+one or more entries in it are modified:
+
+* :xml:type:`Service <ServiceType>` entries whose ``restart``
+  attribute is ``true`` (the default) will be restarted.
+* :xml:type:`Action <ActionType>` entries whose ``when`` attribute is
+  ``modified`` will be run.
+
+Because of these two magic actions, it's extremely important to
+structure your bundles around Service and Action entries, rather than
+around some loose idea of which entries are related.  For instance, in
+order to manage a Bcfg2 server, a number of packages, paths, services,
+etc. must be managed.  But not all of these entries would require
+``bcfg2-server`` to be restarted, so to limit restarts it's wise to
+split these entries into two bundles.  See
+:ref:`server-plugins-structures-bundler-bcfg2-server` for an example
+of this.
+
+Disabling Magic
+---------------
+
+Disabling magic bundler actions can be done in one of two ways:
+
+* On a per-entry basis.  Set ``restart="false"`` on a Service to
+  prevent it from being restarted when the bundle is modified.  Set
+  ``when="always"`` on an Action to cause it to run every time,
+  regardless of whether or not the bundle was modified.
+* On a per-bundle basis.  Set ``independent="true"`` on the top-level
+  ``Bundle`` tag to signify that the bundle is a collection of
+  independent (i.e., unrelated) entries, and to prevent any magic
+  actions from being performed.  (This is similar to the ``Base``
+  plugin in older versions of Bcfg2.)  This was added in Bcfg2 1.4.
+
+Service entries in independent bundles are never restarted, and Action
+entries in independent bundles are only executed if ``when="always"``.
+(I.e., an Action entry in an independent bundle with
+``when="modified"`` is useless.)
 
 Genshi templates
 ================
diff --git a/doc/server/xml-common.txt b/doc/server/xml-common.txt
index c2c3f34d4..96644571b 100644
--- a/doc/server/xml-common.txt
+++ b/doc/server/xml-common.txt
@@ -75,6 +75,89 @@ Or, more compactly:
     Else
         Manage the abstract service "bar"
 
+As an example, consider the following :ref:`bundle
+<server-plugins-structures-bundler-index>`:
+
+.. code-block:: xml
+
+    <Bundle>
+      <Path name='/etc/ssh/ssh_host_dsa_key'/>
+      <Path name='/etc/ssh/ssh_host_rsa_key'/>
+      <Path name='/etc/ssh/ssh_host_dsa_key.pub'/>
+      <Path name='/etc/ssh/ssh_host_rsa_key.pub'/>
+      <Path name='/etc/ssh/ssh_host_key'/>
+      <Path name='/etc/ssh/ssh_host_key.pub'/>
+      <Path name='/etc/ssh/sshd_config'/>
+      <Path name='/etc/ssh/ssh_config'/>
+      <Path name='/etc/ssh/ssh_known_hosts'/>
+      <Group name='rpm'>
+        <Package name='openssh'/>
+        <Package name='openssh-askpass'/>
+        <Service name='sshd'/>
+        <Group name='fedora' >
+           <Group name='fedora14' negate='true'>
+             <Package name='openssh-clients'/>
+           </Group>
+           <Package name='openssh-server'/>
+        </Group>
+      </Group>
+      <Group name='deb'>
+        <Package name='ssh'/>
+        <Service name='ssh'/>
+      </Group>
+      <Client name='trust.example.com'>
+        <Path name='/etc/ssh/shosts.equiv'/>
+      </Client>
+    </Bundle>
+
+In this bundle, most of the entries are common to all systems. Clients
+in group ``deb`` get one extra package and service, while clients in
+group ``rpm`` get two extra packages and an extra service. In
+addition, clients in group ``fedora`` *and* group ``rpm`` get one
+extra package entries, unless they are not in the ``fedora14`` group,
+in which case, they get an extra package. The client
+``trust.example.com`` gets one extra file that is not distributed to
+any other clients.
+
++------------------------+-----------------------------------+
+| Group/Hostname         | Entry                             |
++========================+===================================+
+| all                    | ``/etc/ssh/ssh_host_dsa_key``     |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/ssh_host_rsa_key``     |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/ssh_host_dsa_key.pub`` |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/ssh_host_rsa_key.pub`` |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/ssh_host_key``         |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/ssh_host_key.pub``     |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/sshd_config``          |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/ssh_config``           |
++------------------------+-----------------------------------+
+| all                    | ``/etc/ssh/ssh_known_hosts``      |
++------------------------+-----------------------------------+
+| ``rpm``                | Package ``openssh``               |
++------------------------+-----------------------------------+
+| ``rpm``                | Package ``openssh-askpass``       |
++------------------------+-----------------------------------+
+| ``rpm``                | Service ``sshd``                  |
++------------------------+-----------------------------------+
+| ``rpm`` AND ``fedora`` | Package ``openssh-server``        |
++------------------------+-----------------------------------+
+| ``rpm`` AND ``fedora`` | Package ``openssh-clients``       |
+| AND NOT ``fedora14``   |                                   |
++------------------------+-----------------------------------+
+| ``deb``                | Package ``ssh``                   |
++------------------------+-----------------------------------+
+| ``deb``                | Service ``ssh``                   |
++------------------------+-----------------------------------+
+| ``trust.example.com``  | ``/etc/ssh/shosts.equiv``         |
++------------------------+-----------------------------------+
+
 .. _xml-genshi-templating:
 
 Genshi templating
diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd
index 1b997ede3..e180d265a 100644
--- a/schemas/bundle.xsd
+++ b/schemas/bundle.xsd
@@ -256,12 +256,22 @@
         </xsd:documentation>
       </xsd:annotation>
     </xsd:attribute>
+    <xsd:attribute type='xsd:boolean' name='independent'>
+      <xsd:annotation>
+        <xsd:documentation>
+          If set to ``true``, indicates that the bundle is a
+          collection of independent entries, and that service restarts
+          and modified actions should not be performed.  See
+          :ref:`server-plugins-structures-bundler-magic` for more.
+        </xsd:documentation>
+      </xsd:annotation>
+    </xsd:attribute>
     <xsd:attribute type='xsd:string' name='name'>
       <xsd:annotation>
         <xsd:documentation>
-          The name of the bundle.  If present, this must match the
-          bundle filename, minus the extension.  Specifying the name
-          explicitly is deprecated.
+          **Deprecated.** The name of the bundle.  If present, this
+          must match the bundle filename, minus the extension.
+          Specifying the name explicitly is deprecated.
         </xsd:documentation>
       </xsd:annotation>
     </xsd:attribute>
diff --git a/src/lib/Bcfg2/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py
deleted file mode 100644
index db003f729..000000000
--- a/src/lib/Bcfg2/Server/Plugins/Base.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""This module sets up a base list of configuration entries."""
-
-import copy
-import lxml.etree
-import Bcfg2.Server.Plugin
-from itertools import chain
-
-
-class Base(Bcfg2.Server.Plugin.Plugin,
-           Bcfg2.Server.Plugin.Structure,
-           Bcfg2.Server.Plugin.XMLDirectoryBacked):
-    """This Structure is good for the pile of independent configs
-    needed for most actual systems.
-    """
-    name = 'Base'
-    __author__ = 'bcfg-dev@mcs.anl.gov'
-    __child__ = Bcfg2.Server.Plugin.StructFile
-    deprecated = True
-
-    def __init__(self, core, datastore):
-        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
-        Bcfg2.Server.Plugin.Structure.__init__(self)
-        try:
-            Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data)
-        except OSError:
-            self.logger.error("Failed to load Base repository")
-            raise Bcfg2.Server.Plugin.PluginInitError
-
-    def BuildStructures(self, metadata):
-        """Build structures for client described by metadata."""
-        ret = lxml.etree.Element("Independent", version='2.0')
-        fragments = list(chain(*[base.Match(metadata)
-                                 for base in list(self.entries.values())]))
-        for frag in fragments:
-            ret.append(copy.copy(frag))
-        return [ret]
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index 5a6767755..d8290d844 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -87,6 +87,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
                                   bundlename, exc_info=1)
                 continue
 
+            if data.get("independent", "false").lower() == "true":
+                data.tag = "Independent"
+                del data.attrib['independent']
+
             data.set("name", bundlename)
 
             for child in data.findall("Bundle"):
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py
index f5250ed85..cfb379c40 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestBundler.py
@@ -61,7 +61,7 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked):
     def test_BuildStructures(self):
         b = self.get_obj()
         b.bundles = dict(error=Mock(), skip=Mock(), xinclude=Mock(),
-                         has_dep=Mock(), is_dep=Mock())
+                         has_dep=Mock(), is_dep=Mock(), indep=Mock())
         expected = dict()
 
         b.bundles['error'].XMLMatch.side_effect = TemplateError(None)
@@ -86,11 +86,17 @@ class TestBundler(TestPlugin, TestStructure, TestXMLDirectoryBacked):
         expected['is_dep'] = lxml.etree.Element("Bundle", name="is_dep")
         lxml.etree.SubElement(expected['is_dep'], "Package", name="bar")
 
+        indep = lxml.etree.Element("Bundle", independent="true")
+        lxml.etree.SubElement(indep, "Service", name="baz")
+        b.bundles['indep'].XMLMatch.return_value = indep
+        expected['indep'] = lxml.etree.Element("Independent", name="indep")
+        lxml.etree.SubElement(expected['indep'], "Service", name="baz")
+
         metadata = Mock()
-        metadata.bundles = ["error", "xinclude", "has_dep"]
+        metadata.bundles = ["error", "xinclude", "has_dep", "indep"]
 
         rv = b.BuildStructures(metadata)
-        self.assertEqual(len(rv), 3)
+        self.assertEqual(len(rv), 4)
         for bundle in rv:
             name = bundle.get("name")
             self.assertIsNotNone(name,