From a9a7c84703eb7250012fd68af3e4ca77eae07029 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 14 Aug 2013 10:22:44 -0400 Subject: doc: minor fixes --- doc/appendix/guides/converging_rhel5.txt | 2 +- doc/development/caching.txt | 73 +++++++++++++++++++++++ doc/development/cfg.txt | 6 +- doc/development/core.txt | 2 +- doc/server/caching.txt | 2 +- doc/server/plugins/generators/cfg.txt | 7 ++- doc/server/plugins/generators/rules.txt | 4 +- doc/server/plugins/misc/acl.txt | 2 +- doc/server/plugins/structures/bundler/index.txt | 1 + doc/server/xml-common.txt | 2 +- schemas/sslca-cert.xsd | 7 ++- schemas/sslca-key.xsd | 3 +- testsuite/Testsrc/Testlib/TestServer/TestCache.py | 54 +++++++++++++++++ 13 files changed, 146 insertions(+), 19 deletions(-) create mode 100644 doc/development/caching.txt create mode 100644 testsuite/Testsrc/Testlib/TestServer/TestCache.py diff --git a/doc/appendix/guides/converging_rhel5.txt b/doc/appendix/guides/converging_rhel5.txt index 4ad5756b9..38d8761cb 100644 --- a/doc/appendix/guides/converging_rhel5.txt +++ b/doc/appendix/guides/converging_rhel5.txt @@ -25,7 +25,7 @@ Unmanaged entries sudo yum remove PACKAGE #. Otherwise, add ```` to the Bundler - configuration. + configuration. * Package (dependency) diff --git a/doc/development/caching.txt b/doc/development/caching.txt new file mode 100644 index 000000000..47d627278 --- /dev/null +++ b/doc/development/caching.txt @@ -0,0 +1,73 @@ +.. -*- mode: rst -*- + +.. _development-cache: + +============================ + Server-side Caching System +============================ + +.. versionadded:: 1.4.0 + +Bcfg2 caches two kinds of data: + +* The contents of all files that it reads in, including (often) an + optimized representation. E.g., XML files are cached both in their + raw (text) format, and also as :class:`lxml.etree._Element` objects. +* Arbitrary data, in the server-side caching system documented on this + page. + +The caching system keeps a single unified cache with all cache data in +it. Each individual datum stored in the cache is associated with any +number of "tags" -- simple terms that uniquely identify the datum. +This lets you very easily expire related data from multiple caches at +once; for isntance, for expiring all data related to a host: + +.. code-block:: python + + Bcfg2.Server.Cache.expire("foo.example.com") + +This would expire *all* data related to ``foo.example.com``, +regardless of which plugin cached it, and so on. + +This permits a high level of interoperation between different plugins +and the cache, which is necessary due to the wide distribution of data +in Bcfg2 and the many different data sources that can be incorported. +More technical details about writing code that uses the caches is below. + +Currently known caches are: + +.. currentmodule:: Bcfg2.Server.Plugins.Packages.Collection + ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Tags | Key(s) | Values | Use | ++=============+=======================================+=================================================+======================================================+ +| Metadata | Hostname | :class:`ClientMetadata | The :ref:`Metadata cache ` | +| | | ` | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Probes, | Hostname | ``list`` of group names | Groups set by :ref:`server-plugins-probes-index` | +| probegroups | | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Probes, | Hostname | ``dict`` of ````: | Other data set by :ref:`server-plugins-probes-index` | +| probedata | | :class:`ProbeData | | +| | | ` | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | :class:`Collection` | Kept by :ref:`server-plugins-generators-packages` in | +| collections | ` | | order to expire repository metadata cached on disk | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | Hostname | :attr:`Packages Collection cache key | Used by the Packages plugin to return Collection | +| clients | | ` | objects for clients. This is cross-referenced with | +| | | | the ``Packages, collections`` cache | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | ``set`` of package names | Cached results from looking up | +| pkg_groups | `, | | ```` entries | +| | hash of the selected package groups | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | ``set`` of package names | Cached results from resolving complete package sets | +| pkg_sets | `, | | for clients | +| | hash of the initial package selection | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ + +These are enumerated so that they can be expired as needed by other +plugins or other code points. + +.. automodule:: Bcfg2.Server.Cache diff --git a/doc/development/cfg.txt b/doc/development/cfg.txt index a4360559f..f93bb42c7 100644 --- a/doc/development/cfg.txt +++ b/doc/development/cfg.txt @@ -55,11 +55,6 @@ exceptions: .. autoexception:: Bcfg2.Server.Plugin.exceptions.PluginInitError :noindex: -Global Variables -================ - -.. autodata:: Bcfg2.Server.Plugins.Cfg.CFG - Existing Cfg Handlers ===================== @@ -99,3 +94,4 @@ included for completeness. .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEntrySet .. autoclass:: Bcfg2.Server.Plugins.Cfg.Cfg +.. automethod:: Bcfg2.Server.Plugins.Cfg.get_cfg diff --git a/doc/development/core.txt b/doc/development/core.txt index ecbcbebd3..f5cc7de67 100644 --- a/doc/development/core.txt +++ b/doc/development/core.txt @@ -86,4 +86,4 @@ Multiprocessing Core CherryPy Core ------------- -.. automodule:: Bcfg2.Server.CherryPyCore +.. automodule:: Bcfg2.Server.CherrypyCore diff --git a/doc/server/caching.txt b/doc/server/caching.txt index 51245bd08..32be684db 100644 --- a/doc/server/caching.txt +++ b/doc/server/caching.txt @@ -13,7 +13,7 @@ Metadata Caching Caching (or, rather, cache expiration) is always a difficult problem, but it's particularly vexing in Bcfg2 due to the number of different -data sources incorporated. In 1.3.0, we introduce some limited +data sources incorporated. In 1.3.0, we introduced some limited caching of client metadata objects. Since a client metadata object can be generated anywhere from 7 to dozens of times per client run (depending on your templates), and since client metadata generation diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 56804db99..8220fb21b 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -560,7 +560,6 @@ Example Hopefully, the performance concerns can be resolved in a future release and these features can be added. -.. _server-plugins-generators-sslca: .. _server-plugins-generators-cfg-ssl-certificates: SSL Keys and Certificates @@ -640,7 +639,7 @@ paths. `sslkey.xml`_ for details on how to change the key type and size.) #. Similarly, create `sslcert.xml`_ in - ``Cfg/etc/pki/tls/certs/localhost.cfg/``, containing the following: + ``Cfg/etc/pki/tls/certs/localhost.cfg/``, containing the following: .. code-block:: xml @@ -772,6 +771,8 @@ File permissions for entries handled by Cfg are controlled via the use of :ref:`server-info` files. Note that you **cannot** use both a Permissions entry and a Path entry to handle the same file. +.. _server-plugins-generators-cfg-configuration: + Cfg Configuration ================= @@ -785,7 +786,7 @@ influenced by several options in the ``[sshkeys]`` section of +-------------+----------------+---------------------------------------------------------+-----------------------+------------+ | Section | Option | Description | Values | Default | -+================+=========================================================+=======================+============+ ++=============+================+=========================================================+=======================+============+ | ``cfg`` | ``passphrase`` | Use the named passphrase to encrypt created data on the | String | None | | | | filesystem. (E.g., SSH and SSL keys.) The passphrase | | | | | | must be defined in the ``[encryption]`` section. | | | diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index a95d4a2a4..64dbc8597 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -479,8 +479,8 @@ If you wish, you can configure the Rules plugin to support regular expressions. This entails a small performance and memory usage penalty. To do so, add the following setting to ``bcfg2.conf``:: - [rules] - regex = yes + [rules] + regex = yes With regular expressions enabled, you can use a regex in the ``name`` attribute to match multiple abstract configuration entries. diff --git a/doc/server/plugins/misc/acl.txt b/doc/server/plugins/misc/acl.txt index 73f99bf85..45780bef8 100644 --- a/doc/server/plugins/misc/acl.txt +++ b/doc/server/plugins/misc/acl.txt @@ -189,7 +189,7 @@ The ACL descriptions allow you to use '*' as a wildcard for any number of characters *other than* ``.``. That is: * ``*`` would match ``DeclareVersion`` and ``GetProbes``, but would - *not* match ``Git.Update`. + *not* match ``Git.Update``. * ``*.*`` would match ``Git.Update``, but not ``DeclareVersion`` or ``GetProbes``. diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 25134cb89..0b6b8eb50 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -293,6 +293,7 @@ more complex example Bundles. .. toctree:: :maxdepth: 1 + bcfg2 kernel moab nagios diff --git a/doc/server/xml-common.txt b/doc/server/xml-common.txt index fad054213..3aacfd468 100644 --- a/doc/server/xml-common.txt +++ b/doc/server/xml-common.txt @@ -299,7 +299,7 @@ such an included file to conform to the schema, although in general the included files should be structure exactly like the parent file. Wildcard XInclude -~~~~~~~~~~~~~~~~~ +----------------- .. versionadded:: 1.3.1 diff --git a/schemas/sslca-cert.xsd b/schemas/sslca-cert.xsd index 7a9fb5683..7330ca0ff 100644 --- a/schemas/sslca-cert.xsd +++ b/schemas/sslca-cert.xsd @@ -2,7 +2,8 @@ xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> - Schema for :ref:`server-plugins-generators-sslca` ``sslcert.xml`` + Schema for :ref:`server-plugins-generators-cfg-ssl-certificates` + ``sslcert.xml`` @@ -92,8 +93,8 @@ The name of the CA (from :ref:`bcfg2.conf - <sslca-configuration>`) to use to generate this - certificate. + <server-plugins-generators-cfg-configuration>`) to use + to generate this certificate. diff --git a/schemas/sslca-key.xsd b/schemas/sslca-key.xsd index 3523a0c60..496da859f 100644 --- a/schemas/sslca-key.xsd +++ b/schemas/sslca-key.xsd @@ -2,7 +2,8 @@ xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> - Schema for :ref:`server-plugins-generators-sslca` ``sslkey.xml`` + Schema for :ref:`server-plugins-generators-cfg-ssl-certificates` + ``sslkey.xml`` diff --git a/testsuite/Testsrc/Testlib/TestServer/TestCache.py b/testsuite/Testsrc/Testlib/TestServer/TestCache.py new file mode 100644 index 000000000..7c26e52b8 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestCache.py @@ -0,0 +1,54 @@ +import os +import sys + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +from Bcfg2.Server.Cache import * + + +class TestCache(Bcfg2TestCase): + def test_cache(self): + md_cache = Cache("Metadata") + md_cache['foo.example.com'] = 'foo metadata' + md_cache['bar.example.com'] = 'bar metadata' + self.assertItemsEqual(list(iter(md_cache)), + ["foo.example.com", "bar.example.com"]) + + probe_cache = Cache("Probes", "data") + probe_cache['foo.example.com'] = 'foo probe data' + probe_cache['bar.example.com'] = 'bar probe data' + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + md_cache.expire("foo.example.com") + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + probe_cache.expire("bar.example.com") + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com"]) + + probe_cache['bar.example.com'] = 'bar probe data' + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + expire("bar.example.com") + self.assertEqual(len(md_cache), 0) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com"]) + + probe_cache2 = Cache("Probes", "data") + self.assertItemsEqual(list(iter(probe_cache)), + list(iter(probe_cache2))) -- cgit v1.2.3-1-g7c22