diff options
92 files changed, 3288 insertions, 3007 deletions
diff --git a/doc/appendix/guides/converging_rhel5.txt b/doc/appendix/guides/converging_rhel5.txt index 4ad5756b9..38d8761cb 100644 --- a/doc/appendix/guides/converging_rhel5.txt +++ b/doc/appendix/guides/converging_rhel5.txt @@ -25,7 +25,7 @@ Unmanaged entries sudo yum remove PACKAGE #. Otherwise, add ``<Package name="PACKAGE" />`` to the Bundler - configuration. + configuration. * Package (dependency) diff --git a/doc/appendix/guides/sslca_howto.txt b/doc/appendix/guides/sslca_howto.txt new file mode 100644 index 000000000..9c939dcd3 --- /dev/null +++ b/doc/appendix/guides/sslca_howto.txt @@ -0,0 +1,183 @@ +.. -*- mode: rst -*- + +.. _appendix-guides-sslca_howto: + +==================================== + Automated Bcfg2 SSL Authentication +==================================== + +This how-to describes one possible scenario for automating SSL +certificate generation and distribution for bcfg2 client/server +communication using the :ref:`SSL CA feature +<server-plugins-generators-cfg-ssl-certificates>` of +:ref:`server-plugins-generators-cfg`. The process involves configuring +a certificate authority (CA), generating the CA cert and key pair, +configuring the Cfg SSL CA feature and a Bundle to use the generated +certs to authenticate the Bcfg2 client and server. + +OpenSSL CA +========== + +If you already have a SSL CA available you can skip this section, +otherwise you can easily build one on the server using openssl. The +paths should be adjusted to suite your preferences. + +#. Prepare the directories and files:: + + mkdir -p /etc/pki/CA/newcerts + mkdir /etc/pki/CA/crl + echo '01' > /etc/pki/CA/serial + touch /etc/pki/CA/index.txt + touch /etc/pki/CA/crlnumber + +#. Edit the ``openssl.cnf`` config file, and in the **[ CA_default ]** + section adjust the following parameters:: + + dir = /etc/pki # Where everything is kept + certs = /etc/pki/CA/certs # Where the issued certs are kept + database = /etc/pki/CA/index.txt # database index file. + new_certs_dir = /etc/pki/CA/newcerts # default place for new certs. + certificate = /etc/pki/CA/certs/bcfg2ca.crt # The CA certificate + serial = /etc/pki/CA/serial # The current serial number + crl_dir = /etc/pki/CA/crl # Where the issued crl are kept + crlnumber = /etc/pki/CA/crlnumber # the current crl number + crl = /etc/pki/CA/crl.pem # The current CRL + private_key = /etc/pki/CA/private/bcfg2ca.key # The private key + +#. Create the CA root certificate and key pair. You'll be asked to + supply a passphrase, and some organizational info. The most + important bit is **Common Name** which you should set to be the + hostname of your bcfg2 server that your clients will see when doing + a reverse DNS query on it's ip address.:: + + openssl req -new -x509 -extensions v3_ca -keyout bcfg2ca.key \ + -out bcfg2ca.crt -days 3650 + +#. Move the generated cert and key to the locations specified in + ``openssl.cnf``:: + + mv bcfg2ca.key /etc/pki/CA/private/ + mv bcfg2ca.crt /etc/pki/CA/certs/ + +Your self-signing CA is now ready to use. + +Bcfg2 +===== + +SSL CA Feature +-------------- + +The SSL CA feature of Cfg was not designed specifically to manage +Bcfg2 client/server communication, though it is certainly able to +provide certificate generation and management services for that +purpose. You'll need to configure Cfg as described in +:ref:`server-plugins-generators-cfg-ssl-certificates`, including: + +* Configuring a ``[sslca_default]`` section in ``bcfg2.conf`` that + describes the CA you created above; +* Creating ``Cfg/etc/pki/tls/certs/bcfg2client.crt/sslcert.xml`` and + ``Cfg/etc/pki/tls/private/bcfg2client.key/sslkey.xml`` to describe + the key and cert you want generated. + +In general, the defaults in ``sslcert.xml`` and ``sslkey.xml`` should +be fine, so those files can look like this: + +``Cfg/etc/pki/tls/certs/bcfg2client.crt/sslcert.xml``: + +.. code-block:: xml + + <CertInfo> + <Cert key="/etc/pki/tls/private/bcfg2client.key"/> + </CertInfo> + +``Cfg/etc/pki/tls/private/bcfg2client.key/sslkey.xml``: + +.. code-block:: xml + + <KeyInfo/> + +Client Bundle +------------- + +To automate the process of generating and distributing certs to the +clients we need define at least the cert and key paths created by Cfg, +as well as the CA certificate path in a Bundle. For example: + +.. code-block:: xml + + <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> + <Path name='/etc/pki/tls/bcfg2client.crt'/> + <Path name='/etc/pki/tls/private/bcfg2client.key'/> + +Here's a more complete example bcfg2-client bundle: + +.. code-block:: xml + + <Bundle> + <Path name='/etc/bcfg2.conf'/> + <Path name='/etc/cron.d/bcfg2-client'/> + <Package name='bcfg2'/> + <Service name='bcfg2'/> + <Group name='rpm'> + <Path name='/etc/sysconfig/bcfg2'/> + <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> + <Path name='/etc/pki/tls/certs/bcfg2client.crt'/> + <Path name='/etc/pki/tls/private/bcfg2client.key'/> + </Group> + <Group name='deb'> + <Path name='/etc/default/bcfg2' altsrc='/etc/sysconfig/bcfg2'/> + <Path name='/etc/ssl/certs/bcfg2ca.crt' altsrc='/etc/pki/tls/certs/bcfg2ca.crt'/> + <Path name='/etc/ssl/certs/bcfg2client.crt' altsrc='/etc/pki/tls/certs/bcfg2client.crt'/> + <Path name='/etc/ssl/private/bcfg2client.key' altsrc='/etc/pki/tls/private/bcfg2client.key'/> + </Group> + </Bundle> + +The ``bcfg2.conf`` client config needs at least 5 parameters set for +SSL auth. + +#. ``key`` : This is the host specific key that Cfg will create. +#. ``certificate`` : This is the host specific cert that Cfg will + create. +#. ``ca`` : This is a copy of your CA certificate. Not generated by + Cfg. +#. ``password`` : Set to arbitrary string when using certificate + auth. This also *shouldn't* be required. See: + http://trac.mcs.anl.gov/projects/bcfg2/ticket/1019 + +Here's what a functional **[communication]** section in a +``bcfg2.conf`` genshi template for clients might look like.:: + + [communication] + protocol = xmlrpc/ssl + {% if metadata.uuid != None %}\ + user = ${metadata.uuid} + {% end %}\ + password = DUMMYPASSWORDFORCERTAUTH + {% choose %}\ + {% when 'rpm' in metadata.groups %}\ + certificate = /etc/pki/tls/certs/bcfg2client.crt + key = /etc/pki/tls/private/bcfg2client.key + ca = /etc/pki/tls/certs/bcfg2ca.crt + {% end %}\ + {% when 'deb' in metadata.groups %}\ + certificate = /etc/ssl/certs/bcfg2client.crt + key = /etc/ssl/private/bcfg2client.key + ca = /etc/ssl/certs/bcfg2ca.crt + {% end %}\ + {% end %}\ + +As a client will not be able to authenticate with certificates it does +not yet posses we need to overcome the chicken and egg scenario the +first time we try to connect such a client to the server. We can do so +using password based auth to bootstrap the client manually specifying +all the relevant auth parameters like so:: + + bcfg2 -qv -S https://fqdn.of.bcfg2-server:6789 -u fqdn.of.client \ + -x SUPER_SECRET_PASSWORD + +If all goes well the client should recieve a freshly generated key and +cert and you should be able to run ``bcfg2`` again without specifying +the connection parameters. + +If you do run into problems you may want to review +:ref:`appendix-guides-authentication`. diff --git a/doc/client/tools/actions.txt b/doc/client/tools/actions.txt index 61bb8854b..52d07eb4f 100644 --- a/doc/client/tools/actions.txt +++ b/doc/client/tools/actions.txt @@ -30,10 +30,11 @@ return status, causing failures to still not be centrally reported. If central reporting of action failure is desired, set this attribute to 'check'. -Actions are not completely defined inside of a bundle; they are an -abstract entry. The Rules plugin can bind these entries. For example -to include the above action in a bundle, first the Action entry must -be included in the bundle: +Actions may be completely defined inside of a bundle with the use of +:ref:`server-configurationentries`, much like Packages, Services or Paths. +The Rules plugin can also bind these entries. For example to include the +above action in a bundle, first the Action entry must be included in the +bundle: .. code-block:: xml @@ -79,3 +80,18 @@ requires this key. <Action timing='post' name='apt-key-update' command='apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu 0C5A2783' when='modified' status='check'/> </Group> </Rules> + +Example BoundAction (add RPM GPG keys) +====================================== + +This example will add the RPM-GPG-KEY-redhat-release key to the RPM +GPG keyring **before** Package entries are handled on the client run. + +.. code-block:: xml + + <Bundle name="rpm-gpg-keys"> + <Group name='rhel'> + <Path name="/etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release"/> + <BoundAction timing="pre" name="install rpm key" command="rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release" when="modified" status="check"/> + </Group> + </Bundle> diff --git a/doc/development/caching.txt b/doc/development/caching.txt new file mode 100644 index 000000000..47d627278 --- /dev/null +++ b/doc/development/caching.txt @@ -0,0 +1,73 @@ +.. -*- mode: rst -*- + +.. _development-cache: + +============================ + Server-side Caching System +============================ + +.. versionadded:: 1.4.0 + +Bcfg2 caches two kinds of data: + +* The contents of all files that it reads in, including (often) an + optimized representation. E.g., XML files are cached both in their + raw (text) format, and also as :class:`lxml.etree._Element` objects. +* Arbitrary data, in the server-side caching system documented on this + page. + +The caching system keeps a single unified cache with all cache data in +it. Each individual datum stored in the cache is associated with any +number of "tags" -- simple terms that uniquely identify the datum. +This lets you very easily expire related data from multiple caches at +once; for isntance, for expiring all data related to a host: + +.. code-block:: python + + Bcfg2.Server.Cache.expire("foo.example.com") + +This would expire *all* data related to ``foo.example.com``, +regardless of which plugin cached it, and so on. + +This permits a high level of interoperation between different plugins +and the cache, which is necessary due to the wide distribution of data +in Bcfg2 and the many different data sources that can be incorported. +More technical details about writing code that uses the caches is below. + +Currently known caches are: + +.. currentmodule:: Bcfg2.Server.Plugins.Packages.Collection + ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Tags | Key(s) | Values | Use | ++=============+=======================================+=================================================+======================================================+ +| Metadata | Hostname | :class:`ClientMetadata | The :ref:`Metadata cache <server-caching>` | +| | | <Bcfg2.Server.Plugins.Metadata.ClientMetadata>` | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Probes, | Hostname | ``list`` of group names | Groups set by :ref:`server-plugins-probes-index` | +| probegroups | | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Probes, | Hostname | ``dict`` of ``<probe name>``: | Other data set by :ref:`server-plugins-probes-index` | +| probedata | | :class:`ProbeData | | +| | | <Bcfg2.Server.Plugins.Probes.ProbeData>` | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | :class:`Collection` | Kept by :ref:`server-plugins-generators-packages` in | +| collections | <Collection.cachekey>` | | order to expire repository metadata cached on disk | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | Hostname | :attr:`Packages Collection cache key | Used by the Packages plugin to return Collection | +| clients | | <Collection.cachekey>` | objects for clients. This is cross-referenced with | +| | | | the ``Packages, collections`` cache | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | ``set`` of package names | Cached results from looking up | +| pkg_groups | <Collection.cachekey>`, | | ``<Package group="..."/>`` entries | +| | hash of the selected package groups | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ +| Packages, | :attr:`Packages Collection cache key | ``set`` of package names | Cached results from resolving complete package sets | +| pkg_sets | <Collection.cachekey>`, | | for clients | +| | hash of the initial package selection | | | ++-------------+---------------------------------------+-------------------------------------------------+------------------------------------------------------+ + +These are enumerated so that they can be expired as needed by other +plugins or other code points. + +.. automodule:: Bcfg2.Server.Cache diff --git a/doc/development/cfg.txt b/doc/development/cfg.txt index a4360559f..f93bb42c7 100644 --- a/doc/development/cfg.txt +++ b/doc/development/cfg.txt @@ -55,11 +55,6 @@ exceptions: .. autoexception:: Bcfg2.Server.Plugin.exceptions.PluginInitError :noindex: -Global Variables -================ - -.. autodata:: Bcfg2.Server.Plugins.Cfg.CFG - Existing Cfg Handlers ===================== @@ -99,3 +94,4 @@ included for completeness. .. autoclass:: Bcfg2.Server.Plugins.Cfg.CfgEntrySet .. autoclass:: Bcfg2.Server.Plugins.Cfg.Cfg +.. automethod:: Bcfg2.Server.Plugins.Cfg.get_cfg diff --git a/doc/development/core.txt b/doc/development/core.txt index ecbcbebd3..f5cc7de67 100644 --- a/doc/development/core.txt +++ b/doc/development/core.txt @@ -86,4 +86,4 @@ Multiprocessing Core CherryPy Core ------------- -.. automodule:: Bcfg2.Server.CherryPyCore +.. automodule:: Bcfg2.Server.CherrypyCore diff --git a/doc/man/bcfg2-admin.txt b/doc/man/bcfg2-admin.txt index 6169ec537..7b19bd366 100644 --- a/doc/man/bcfg2-admin.txt +++ b/doc/man/bcfg2-admin.txt @@ -38,9 +38,6 @@ Modes backup Create an archive of the entire Bcfg2 repository. -bundle *action* - Display details about the available bundles (See BUNDLE OPTIONS - below). client *action* *client* [attribute=value] Add, edit, or remove clients entries in metadata (See CLIENT OPTIONS below). @@ -48,8 +45,12 @@ compare *old* *new* Compare two client configurations. Can be used to verify consistent behavior between releases. Determine differences between files or directories (See COMPARE OPTIONS below). +dbshell + Call the Django 'dbshell' command on the configured database. init Initialize a new repository (interactive). +initreports + Initialize the Reporting database. minestruct *client* [-f xml-file] [-g groups] Build structure entries based on client statistics extra entries (See MINESTRUCT OPTIONS below). @@ -58,34 +59,31 @@ perf pull *client* *entry-type* *entry-name* Install configuration information into repo based on client bad entries (See PULL OPTIONS below). -reports [init|load_stats|purge|scrub|update] - Interact with the dynamic reporting system (See REPORTS OPTIONS - below). -snapshots [init|dump|query|reports] - Interact with the Snapshots database (See SNAPSHOTS OPTIONS below). +purgereports + Purge historic and expired data from the Reporting database +reportssqlall + Call the Django 'shell' command on the Reporting database. +reportsstats + Print Reporting database statistics. +scrubreports + Scrub the Reporting database for duplicate reasons and orphaned + entries. +shell + Call the Django 'shell' command on the configured database. syncdb Sync the Django ORM with the configured database. tidy Remove unused files from repository. +updatereports + Apply database schema updates to the Reporting database. +validatedb + Call the Django 'validate' command on the configured database. viz [-H] [-b] [-k] [-o png-file] Create a graphviz diagram of client, group and bundle information (See VIZ OPTIONS below). xcmd Provides a XML-RPC Command Interface to the bcfg2-server. -BUNDLE OPTIONS -++++++++++++++ - -mode - One of the following. - - *list-xml* - List all available xml bundles - *list-genshi* - List all available genshi bundles - *show* - Interactive dialog to get details about the available bundles - CLIENT OPTIONS ++++++++++++++ @@ -110,11 +108,20 @@ attribute=value COMPARE OPTIONS +++++++++++++++ +-d *N*, --diff-lines *N* + Show only N lines of a diff + +-c, --color + Show colors even if not ryn from a TTY + +-q, --quiet + Only show that entries differ, not how they differ + old - Specify the location of the old configuration file. + Specify the location of the old configuration(s). new - Specify the location of the new configuration file. + Specify the location of the new configuration(s). MINESTRUCT OPTIONS ++++++++++++++++++ @@ -140,51 +147,24 @@ entry type entry name Specify the name of the entry to pull. -REPORTS OPTIONS -+++++++++++++++ - -load_stats [-s] [-c] [-03] - Load statistics data. - -purge [--client [n]] [--days [n]] [--expired] - Purge historic and expired data. - -scrub - Scrub the database for duplicate reasons and orphaned entries. - -update - Apply any updates to the reporting database. - -SNAPSHOTS OPTIONS -+++++++++++++++++ - -init - Initialize the snapshots database. - -query - Query the snapshots database. - -dump - Dump some of the contents of the snapshots database. - -reports [-a] [-b] [-e] [--date=MM-DD-YYYY] - Generate reports for clients in the snapshots database. - VIZ OPTIONS +++++++++++ --H +-H, --includehosts Include hosts in diagram. --b +-b, --includebundles Include bundles in diagram. --o <outfile> +-o *outfile*, --outfile *outfile* Write to outfile file instead of stdout. --k +-k, --includekey Add a shape/color key. +-c *hostname*, --only-client *hostname* + Only show groups and bundles for the named client + See Also -------- diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index b0ef905d1..f5612e08f 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -76,23 +76,22 @@ plugins A comma-delimited list of enabled server plugins. Currently available plugins are:: - Account + ACL Bundler Bzr Cfg Cvs Darcs - DBStats Decisions + Defaults Deps - Editor FileProbes Fossil Git + GroupLogic GroupPatterns Guppy Hg - Hostbase Ldap Metadata NagiosGen @@ -107,14 +106,9 @@ plugins Rules SEModules ServiceCompat - Snapshots SSHbase - SSLCA - Statistics Svn - TCheetah TemplateHelper - TGenshi Trigger Descriptions of each plugin can be found in their respective @@ -157,16 +151,10 @@ Server Plugins This section has a listing of all the plugins currently provided with Bcfg2. -Account Plugin -++++++++++++++ - -The account plugin manages authentication data, including the following. +ACL Plugin +++++++++++ -* ``/etc/passwd`` -* ``/etc/group`` -* ``/etc/security/limits.conf`` -* ``/etc/sudoers`` -* ``/root/.ssh/authorized_keys`` +The ACL plugin controls which hosts can make which XML-RPC calls. Bundler Plugin ++++++++++++++ @@ -193,25 +181,20 @@ contents for clients. In its simplest form, the Cfg repository is just a directory tree modeled off of the directory tree on your client machines. -Cvs Plugin (experimental) -+++++++++++++++++++++++++ +Cvs Plugin +++++++++++ The Cvs plugin allows you to track changes to your Bcfg2 repository using a Concurrent version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -Darcs Plugin (experimental) -+++++++++++++++++++++++++++ +Darcs Plugin +++++++++++++ The Darcs plugin allows you to track changes to your Bcfg2 repository using a Darcs version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -DBStats Plugin -++++++++++++++ - -Direct to database statistics plugin. - Decisions Plugin ++++++++++++++++ @@ -235,13 +218,6 @@ Deps Plugin The Deps plugin allows you to make a series of assertions like "Package X requires Package Y (and optionally also Package Z etc.)" -Editor Plugin -+++++++++++++ - -The Editor plugin attempts to allow you to partially manage -configuration for a file. Its use is not recommended and not well -documented. - FileProbes Plugin +++++++++++++++++ @@ -264,6 +240,12 @@ The Git plugin allows you to track changes to your Bcfg2 repository using a Git version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. +GroupLogic Plugin ++++++++++++++++++ + +The GroupLogic plugin lets you flexibly assign group membership with a +Genshi template. + GroupPatterns Plugin ++++++++++++++++++++ @@ -276,18 +258,13 @@ Guppy Plugin The Guppy plugin is used to trace memory leaks within the bcfg2-server process using Guppy. -Hg Plugin (experimental) -++++++++++++++++++++++++ +Hg Plugin ++++++++++ The Hg plugin allows you to track changes to your Bcfg2 repository using a Mercurial version control backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -Hostbase Plugin -+++++++++++++++ - -The Hostbase plugin is an IP management system built on top of Bcfg2. - Ldap Plugin +++++++++++ @@ -306,8 +283,8 @@ NagiosGen Plugin The NagiosGen plugin dynamically generates Nagios configuration files based on Bcfg2 data. -Ohai Plugin (experimental) -++++++++++++++++++++++++++ +Ohai Plugin ++++++++++++ The Ohai plugin is used to detect information about the client operating system. The data is reported back to the server using JSON. @@ -379,12 +356,6 @@ ServiceCompat Plugin The ServiceCompat plugin converts service entries for older clients. -Snapshots Plugin -++++++++++++++++ - -The Snapshots plugin stores various aspects of a client’s state when the -client checks in to the server. - SSHbase Plugin ++++++++++++++ @@ -392,17 +363,6 @@ The SSHbase generator plugin manages ssh host keys (both v1 and v2) for hosts. It also manages the ssh_known_hosts file. It can integrate host keys from other management domains and similarly export its keys. -SSLCA Plugin -++++++++++++ - -The SSLCA plugin is designed to handle creation of SSL privatekeys and -certificates on request. - -Statistics -++++++++++ - -The Statistics plugin is deprecated (see Reporting). - Svn Plugin ++++++++++ @@ -410,20 +370,6 @@ The Svn plugin allows you to track changes to your Bcfg2 repository using a Subversion backend. Currently, it enables you to get revision information out of your repository for reporting purposes. -TCheetah Plugin -+++++++++++++++ - -The TCheetah plugin allows you to use the cheetah templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. - -TGenshi Plugin -++++++++++++++ - -The TGenshi plugin allows you to use the Genshi templating system to -create files. It also allows you to include the results of probes -executed on the client in the created files. - Trigger Plugin ++++++++++++++ @@ -657,25 +603,12 @@ the configuration file. running in paranoid mode. Only the most recent versions of these copies will be kept. -Snapshots options ------------------ - -Specified in the **[snapshots]** section. These options control the -server snapshots functionality. - - driver - sqlite - - database - The name of the database to use for statistics data. - - e.g.: ``$REPOSITORY_DIR/etc/bcfg2.sqlite`` - -SSLCA options -------------- +SSL CA options +-------------- -These options are necessary to configure the SSLCA plugin and can be -found in the **[sslca_default]** section of the configuration file. +These options are necessary to configure the SSL CA feature of the Cfg +plugin and can be found in the **[sslca_default]** section of the +configuration file. config Specifies the location of the openssl configuration file for diff --git a/doc/server/acl.txt b/doc/server/acl.txt new file mode 100644 index 000000000..6ea276a53 --- /dev/null +++ b/doc/server/acl.txt @@ -0,0 +1,41 @@ +.. -*- mode: rst -*- + +.. _server-access-control: + +================ + Access Control +================ + +.. versionadded:: 1.4.0 + +Bcfg2 exposes various functions via XML-RPC calls. Some of these are +relatively benign (e.g., the calls necessary to generate a client +configuration) while others can be used to inspect potentially private +data on the server or very easily mount a denial of service attack. +As a result, access control lists to limit exposure of these calls is +built in. There are two possible ACL methods: built-in, and the +:ref:`server-plugins-misc-acls` plugin. + +The built-in approach simply applies a restrictive default ACL that +lets ``localhost`` perform all XML-RPC calls, and restricts all other +machines to only the calls necessary to run the Bcfg2 client. +Specifically: + +* If the remote client is ``127.0.0.1``, the call is allowed. Note + that, depending on where your Bcfg2 server listens and how it + communicates with itself, it likely will not identify to itself as + ``localhost``. +* If the remote client is not ``127.0.0.1`` and the call is any of the + ``set_debug`` or ``toggle_debug`` methods (including + ``[toggle|set]_core_debug``), it is rejected. +* If the remote client is not ``127.0.0.1`` and the call is + ``get_statistics`` (used by ``bcfg2-admin perf``), it is rejected. +* If the remote client is not ``127.0.0.1`` and the call includes a + ``.`` -- i.e., it is dispatched to any plugin, such as + ``Packages.Refresh`` -- then it is rejected. +* Otherwise, the call is allowed. + +The built-in ACL is *only* intended to ensure that Bcfg2 is secure by +default; it will not be sufficient in many (or even most) cases. In +these cases, it's recommended that you use the +:ref:`server-plugins-misc-acls` plugin. diff --git a/doc/server/caching.txt b/doc/server/caching.txt index 51245bd08..32be684db 100644 --- a/doc/server/caching.txt +++ b/doc/server/caching.txt @@ -13,7 +13,7 @@ Metadata Caching Caching (or, rather, cache expiration) is always a difficult problem, but it's particularly vexing in Bcfg2 due to the number of different -data sources incorporated. In 1.3.0, we introduce some limited +data sources incorporated. In 1.3.0, we introduced some limited caching of client metadata objects. Since a client metadata object can be generated anywhere from 7 to dozens of times per client run (depending on your templates), and since client metadata generation diff --git a/doc/server/info.txt b/doc/server/info.txt index 2c50f0031..8342e1cee 100644 --- a/doc/server/info.txt +++ b/doc/server/info.txt @@ -7,8 +7,7 @@ info.xml ======== Various file properties for entries served by most generator plugins, -including :ref:`server-plugins-generators-cfg`, -:ref:`server-plugins-generators-sslca`, and +including :ref:`server-plugins-generators-cfg` and :ref:`server-plugins-generators-sshbase`, are controlled through the use of ``info.xml`` files. diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index 4d35a5970..7a404c824 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -413,7 +413,7 @@ See :ref:`server-encryption` for more details on encryption in Bcfg2 in general. ``pubkey.xml`` -~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~ ``pubkey.xml`` only ever contains a single line: @@ -539,7 +539,8 @@ Example </Group> <Allow from="/root/.ssh/id_rsa.pub" host="foo.example.com"/> <Allow from="/home/foo_user/.ssh/id_rsa.pub"> - <Params command="/home/foo_user/.ssh/ssh_command_filter"/> + <Option name="command" value="/home/foo_user/.ssh/ssh_command_filter"/> + <Option name="no-X11-forwarding"/> </Allow> <Allow> ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDw/rgKQeARRAHK5bQQhAAe1b+gzdtqBXWrZIQ6cIaLgxqj76TwZ3DY4A6aW9RgC4zzd0p4a9MfsScUIB4+UeZsx9GopUj4U6H8Vz7S3pXxrr4E9logVLuSfOLFbI/wMWNRuOANqquLYQ+JYWKeP4kagkVp0aAWp7mH5IOI0rp0A6qE2you4ep9N/nKvHDrtypwhYBWprsgTUXXMHnAWGmyuHGYWxNYBV9AARPdAvZfb8ggtuwibcOULlyK4DdVNbDTAN1/BDBE1ve6WZDcrc386KhqUGj/yoRyPjNZ46uZiOjRr3cdY6yUZoCwzzxvm5vle6mEbLjHgjGEMQMArzM9 vendor@example.com @@ -560,29 +561,162 @@ Example Hopefully, the performance concerns can be resolved in a future release and these features can be added. +.. _server-plugins-generators-cfg-ssl-certificates: + +SSL Keys and Certificates +========================= + +Cfg can also create SSL keys and certs on the fly, and store the +generated data in the repo so that subsequent requests do not result +in repeated key/cert recreation. In the event that a new key or cert +is needed, the old file can simply be removed from the +repository, and the next time that host checks in, a new file will be +created. If that file happens to be the key, any dependent +certificates will also be regenerated. + +See also :ref:`appendix-guides-sslca_howto` for a detailed example +that uses the SSL key management feature to automate Bcfg2 certificate +authentication. + +Getting started +--------------- + +In order to use the SSL certificate generation feature, you must first +have at least one CA configured on your system. For details on +setting up your own OpenSSL based CA, please see +http://www.openssl.org/docs/apps/ca.html for details of the suggested +directory layout and configuration directives. + +For SSL cert generation to work, the openssl.cnf (or other +configuration file) for that CA must contain full (not relative) +paths. + +#. Add a section to your ``/etc/bcfg2.conf`` called ``sslca_foo``, + replacing foo with the name you wish to give your CA so you can + reference it in certificate definitions. (If you only have one CA, + you can name it ``sslca_default``, and it will be the default CA + for all other operations.) + +#. Under that section, add a ``config`` option that gives the location + of the ``openssl.cnf`` file for your CA. + +#. If necessary, add a ``passphrase`` option containing the passphrase + for the CA's private key. If no passphrase is entry exists, it is + assumed that the private key is stored unencrypted. + +#. Optionally, add a ``chaincert`` option that points to the location + of your ssl chaining certificate. This is used when preexisting + certificate hostfiles are found, so that they can be validated and + only regenerated if they no longer meet the specification. If + you're using a self signing CA this would be the CA cert that you + generated. If the chain cert is a root CA cert (e.g., if it is a + self-signing CA), also add an entry ``root_ca = true``. If + ``chaincert`` is omitted, certificate verification will not be + performed. + +#. Once all this is done, you should have a section in your + ``/etc/bcfg2.conf`` that looks similar to the following:: + + [sslca_default] + config = /etc/pki/CA/openssl.cnf + passphrase = youReallyThinkIdShareThis? + chaincert = /etc/pki/CA/chaincert.crt + root_ca = true + +#. You are now ready to create key and certificate definitions. For + this example we'll assume you've added Path entries for the key, + ``/etc/pki/tls/private/localhost.key``, and the certificate, + ``/etc/pki/tls/certs/localhost.crt`` to a bundle. + +#. Within the ``Cfg/etc/pki/tls/private/localhost.key`` directory, + create a `sslkey.xml`_ file containing the following: + + .. code-block:: xml + + <KeyInfo/> + +#. This will cause the generation of an SSL key when a client requests + that Path. (By default, it will be a 2048-bit RSA key; see + `sslkey.xml`_ for details on how to change the key type and size.) + +#. Similarly, create `sslcert.xml`_ in + ``Cfg/etc/pki/tls/certs/localhost.cfg/``, containing the following: + + .. code-block:: xml + + <CertInfo> + <Cert key="/etc/pki/tls/private/localhost.key" ca="foo"/> + </CertInfo> + +#. When a client requests the cert path, a certificate will be + generated using the key hostfile at the specified key location, + using the CA matching the ``ca`` attribute. ie. ``ca="foo"`` will + match ``[sslca_default]`` in your ``/etc/bcfg2.conf`` + +The :ref:`Bcfg2 bundle example +<server-plugins-structures-bundler-bcfg2-server>` contains entries to +automate the process of setting up a CA. + Configuration ------------- -In addition to ``privkey.xml`` and ``authorized_keys.xml``, described -above, the behavior of the SSH key generation feature can be -influenced by several options in the ``[sshkeys]`` section of -``bcfg2.conf``: +``bcfg2.conf`` +~~~~~~~~~~~~~~ -+----------------+---------------------------------------------------------+-----------------------+------------+ -| Option | Description | Values | Default | -+================+=========================================================+=======================+============+ -| ``passphrase`` | Use the named passphrase to encrypt private keys on the | String | None | -| | filesystem. The passphrase must be defined in the | | | -| | ``[encryption]`` section. See :ref:`server-encryption` | | | -| | for more details on encryption in Bcfg2 in general. | | | -+----------------+---------------------------------------------------------+-----------------------+------------+ -| ``category`` | Generate keys specific to groups in the given category. | String | None | -| | It is best to pick a category that all clients have a | | | -| | group from. | | | -+----------------+---------------------------------------------------------+-----------------------+------------+ +In ``bcfg2.conf``, you must declare your CA(s) in ``[sslca_<name>]`` +sections. At least one is required. Valid options are detailed +below, in `Cfg Configuration`_. + +Only the ``config`` option is required; i.e., the simplest possible CA +section is:: + + [sslca_default] + config = /etc/pki/CA/openssl.cnf + +``sslcert.xml`` +~~~~~~~~~~~~~~~ + +.. xml:schema:: sslca-cert.xsd + :linktotype: + :inlinetypes: CertType + +Example +^^^^^^^ + +.. code-block:: xml + + <CertInfo> + <subjectAltName>test.example.com</subjectAltName> + <Group name="apache"> + <Cert key="/etc/pki/tls/private/foo.key" days="730"/> + </Group> + <Group name="nginx"> + <Cert key="/etc/pki/tls/private/foo.key" days="730" + append_chain="true"/> + </Group> + </CertInfo> + +``sslkey.xml`` +~~~~~~~~~~~~~~ + +.. xml:schema:: sslca-key.xsd + :linktotype: + :inlinetypes: KeyType + +Example +^^^^^^^ + +.. code-block:: xml + + <KeyInfo> + <Group name="fast"> + <Key type="rsa" bits="1024"/> + </Group> + <Group name="secure"> + <Key type="rsa" bits="4096"/> + </Group> + </KeyInfo> -See :ref:`server-encryption` for more details on encryption in Bcfg2 -in general. .. _server-plugins-generators-cfg-validation: @@ -637,3 +771,56 @@ File permissions File permissions for entries handled by Cfg are controlled via the use of :ref:`server-info` files. Note that you **cannot** use both a Permissions entry and a Path entry to handle the same file. + +.. _server-plugins-generators-cfg-configuration: + +Cfg Configuration +================= + +The behavior of many bits of the Cfg plugin can be configured in +``bcfg2.conf`` with the following options. + +In addition to ``privkey.xml`` and ``authorized_keys.xml``, described +above, the behavior of the SSH key generation feature can be +influenced by several options in the ``[sshkeys]`` section of +``bcfg2.conf``: + ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| Section | Option | Description | Values | Default | ++=============+================+=========================================================+=======================+============+ +| ``cfg`` | ``passphrase`` | Use the named passphrase to encrypt created data on the | String | None | +| | | filesystem. (E.g., SSH and SSL keys.) The passphrase | | | +| | | must be defined in the ``[encryption]`` section. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``cfg`` | ``category`` | Generate data (e.g., SSH keys, SSL keys and certs) | String | None | +| | | specific to groups in the given category. It is best to | | | +| | | pick a category that all clients have a group from. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``cfg`` | ``validation`` | Whether or not to perform `Content Validation`_ | Boolean | True | +| | | specific to groups in the given category. It is best to | | | +| | | pick a category that all clients have a group from. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sshkeys`` | ``passphrase`` | Override the global Cfg passphrase with a specific | String | None | +| | | passphrase for encrypting created SSH private keys. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sshkeys`` | ``category`` | Override the global Cfg category with a specific | String | None | +| | | category for created SSH keys. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca`` | ``passphrase`` | Override the global Cfg passphrase with a specific | String | None | +| | | passphrase for encrypting created SSL keys. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca`` | ``category`` | Override the global Cfg category with a specific | String | None | +| | | category for created SSL keys and certs. | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``config`` | Path to the openssl config for the CA | String | None | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``passphrase`` | Passphrase for the CA private key | String | None | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``chaincert`` | Path to the SSL chaining certificate for verification | String | None | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ +| ``sslca_*`` | ``root_ca`` | Whether or not ``<chaincert>`` is a root CA (as | Boolean | False | +| | | opposed to an intermediate cert) | | | ++-------------+----------------+---------------------------------------------------------+-----------------------+------------+ + +See :ref:`server-encryption` for more details on encryption in Bcfg2 +in general. diff --git a/doc/server/plugins/generators/nagiosgen.txt b/doc/server/plugins/generators/nagiosgen.txt index d2643647b..1ccdd66c1 100644 --- a/doc/server/plugins/generators/nagiosgen.txt +++ b/doc/server/plugins/generators/nagiosgen.txt @@ -126,19 +126,19 @@ Create a nagios Bcfg2 bundle ``/var/lib/bcfg2/Bundler/nagios.xml`` <Bundle> <Path name='/etc/nagiosgen.status'/> - <Group name='rh'> + <Group name='redhat'> <Group name='nagios-server'> - <Path name='/etc/nagios/nagiosgen.cfg'/> + <Path name='/etc/nagios/conf.d/bcfg2.cfg'/> <Package name='libtool-libs'/> <Package name='nagios'/> <Package name='nagios-www'/> <Service name='nagios'/> </Group> </Group> - <Group name='debian-lenny'> + <Group name='debian-wheezy'> <Group name='nagios-server'> - <Path name='/etc/nagios3/nagiosgen.cfg' - altsrc='/etc/nagios/nagiosgen.cfg'/> + <Path name='/etc/nagios3/conf.d/bcfg2.cfg' + altsrc='/etc/nagios/conf.d/bcfg2.cfg'/> <Package name='nagios3'/> <Package name='nagios3-common'/> <Package name='nagios3-doc'/> @@ -161,10 +161,6 @@ Assign clients to nagios groups in <Bundle name='nagios'/> </Group> -Update nagios configuration file to use ``nagiosgen.cfg``:: - - cfg_file=/etc/nagios/nagiosgen.cfg - Note that some of these files are built on demand, each time a client in group "nagios-server" checks in with the Bcfg2 server. Local nagios instances can be configured to use the NagiosGen directory in the Bcfg2 diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index a7cdfad2d..8b317552f 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -428,17 +428,18 @@ Benefits to this include: * Much lower memory usage by the ``bcfg2-server`` process. * Much faster ``Packages.Refresh`` behavior. * More accurate dependency resolution. +* Better use of multiple processors/cores. Drawbacks include: -* More disk I/O. In some cases, you may have to raise the open file +* Resolution of package dependencies is slower and more + resource-intensive. At times it can be much slower, particularly + after running ``Packages.Refresh``. +* More disk I/O. This can be alleviated by putting + ``/var/lib/bcfg2/Packages/cache`` on tmpfs, but that offsets the + lower memory usage. In some cases, you may have to raise the open file limit for the user who runs your Bcfg2 server process, particularly if you have a lot of repositories. -* Resolution of package dependencies is slower in some cases, - particularly after running ``Packages.Refresh``. -* If you have a very large number of clients using a very small number - of repositories, using native yum libraries may actually increase - memory usage. Configuring the Yum Helper -------------------------- diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index a95d4a2a4..64dbc8597 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -479,8 +479,8 @@ If you wish, you can configure the Rules plugin to support regular expressions. This entails a small performance and memory usage penalty. To do so, add the following setting to ``bcfg2.conf``:: - [rules] - regex = yes + [rules] + regex = yes With regular expressions enabled, you can use a regex in the ``name`` attribute to match multiple abstract configuration entries. diff --git a/doc/server/plugins/generators/sslca.txt b/doc/server/plugins/generators/sslca.txt deleted file mode 100644 index 2a7e3ecad..000000000 --- a/doc/server/plugins/generators/sslca.txt +++ /dev/null @@ -1,361 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-plugins-generators-sslca: - -===== -SSLCA -===== - -SSLCA is a generator plugin designed to handle creation of SSL private -keys and certificates on request. - -Borrowing ideas from :ref:`server-plugins-generators-cfg-genshi` and -the :ref:`server-plugins-generators-sshbase` plugin, SSLCA automates -the generation of SSL certificates by allowing you to specify key and -certificate definitions. Then, when a client requests a Path that -contains such a definition within the SSLCA repository, the matching -key/cert is generated, and stored in a hostfile in the repo so that -subsequent requests do not result in repeated key/cert recreation. In -the event that a new key or cert is needed, the offending hostfile can -simply be removed from the repository, and the next time that host -checks in, a new file will be created. If that file happens to be the -key, any dependent certificates will also be regenerated. - -.. _getting-started: - -Getting started -=============== - -In order to use SSLCA, you must first have at least one CA configured -on your system. For details on setting up your own OpenSSL based CA, -please see http://www.openssl.org/docs/apps/ca.html for details of the -suggested directory layout and configuration directives. - -For SSLCA to work, the openssl.cnf (or other configuration file) for -that CA must contain full (not relative) paths. - -#. Add SSLCA to the **plugins** line in ``/etc/bcfg2.conf`` and - restart the server -- This enabled the SSLCA plugin on the Bcfg2 - server. - -#. Add a section to your ``/etc/bcfg2.conf`` called ``sslca_foo``, - replacing foo with the name you wish to give your CA so you can - reference it in certificate definitions. - -#. Under that section, add an entry for ``config`` that gives the - location of the openssl configuration file for your CA. - -#. If necessary, add an entry for ``passphrase`` containing the - passphrase for the CA's private key. We store this in - ``/etc/bcfg2.conf`` as the permissions on that file should have it - only readable by the bcfg2 user. If no passphrase is entry exists, - it is assumed that the private key is stored unencrypted. - -#. Optionally, Add an entry ``chaincert`` that points to the location - of your ssl chaining certificate. This is used when preexisting - certifcate hostfiles are found, so that they can be validated and - only regenerated if they no longer meet the specification. If - you're using a self signing CA this would be the CA cert that you - generated. If the chain cert is a root CA cert (e.g., if it is a - self-signing CA), also add an entry ``root_ca = true``. If - ``chaincert`` is omitted, certificate verification will not be - performed. - -#. Once all this is done, you should have a section in your - ``/etc/bcfg2.conf`` that looks similar to the following:: - - [sslca_default] - config = /etc/pki/CA/openssl.cnf - passphrase = youReallyThinkIdShareThis? - chaincert = /etc/pki/CA/chaincert.crt - root_ca = true - -#. You are now ready to create key and certificate definitions. For - this example we'll assume you've added Path entries for the key, - ``/etc/pki/tls/private/localhost.key``, and the certificate, - ``/etc/pki/tls/certs/localhost.crt`` to a bundle or base. - -#. Defining a key or certificate is similar to defining a Cfg file. - Under your Bcfg2's ``SSLCA/`` directory, create the directory - structure to match the path to your key. In this case this would be - something like - ``/var/lib/bcfg2/SSLCA/etc/pki/tls/private/localhost.key``. - -#. Within that directory, create a `key.xml`_ file containing the - following: - - .. code-block:: xml - - <KeyInfo> - <Key type="rsa" bits="2048" /> - </KeyInfo> - -#. This will cause the generation of an 2048 bit RSA key when a client - requests that Path. Alternatively you can specify ``dsa`` as the - keytype, or a different number of bits. - -#. Similarly, create the matching directory structure for the - certificate path, and a `cert.xml`_ containing the following: - - .. code-block:: xml - - <CertInfo> - <Cert format="pem" key="/etc/pki/tls/private/localhost.key" - ca="default" days="365" c="US" l="New York" st="New York" - o="Your Company Name" /> - </CertInfo> - -#. When a client requests the cert path, a certificate will be - generated using the key hostfile at the specified key location, - using the CA matching the ca attribute. ie. ca="default" will match - [sslca_default] in your ``/etc/bcfg2.conf`` - -.. _sslca-configuration: - -Configuration -============= - -bcfg2.conf ----------- - -``bcfg2.conf`` contains miscellaneous configuration options for the -SSLCA plugin. These are described in some detail above in -`getting-started`, but are also enumerated here as a reference. Any -booleans in the config file accept the values "1", "yes", "true", and -"on" for True, and "0", "no", "false", and "off" for False. - -Each directive below should appear at most once in each -``[sslca_<name>]`` section. The following directives are understood: - -+--------------+------------------------------------------+---------+---------+ -| Name | Description | Values | Default | -+==============+==========================================+=========+=========+ -| config | Path to the openssl config for the CA | String | None | -+--------------+------------------------------------------+---------+---------+ -| passphrase | Passphrase for the CA private key | String | None | -+--------------+------------------------------------------+---------+---------+ -| chaincert | Path to the SSL chaining certificate for | String | None | -| | verification | | | -+--------------+------------------------------------------+---------+---------+ -| root_ca | Whether or not ``<chaincert>`` is a root | Boolean | false | -| | CA (as opposed to an intermediate cert) | | | -+--------------+------------------------------------------+---------+---------+ - -Only ``config`` is required. - -cert.xml --------- - -.. xml:schema:: sslca-cert.xsd - :linktotype: - :inlinetypes: CertType - -Example -^^^^^^^ - -.. code-block:: xml - - <CertInfo> - <subjectAltName>test.example.com</subjectAltName> - <Group name="apache"> - <Cert key="/etc/pki/tls/private/foo.key" days="730"/> - </Group> - <Group name="nginx"> - <Cert key="/etc/pki/tls/private/foo.key" days="730" - append_chain="true"/> - </Group> - </CertInfo> - -key.xml -------- - -.. xml:schema:: sslca-key.xsd - :linktotype: - :inlinetypes: KeyType - -Example -^^^^^^^ - -.. code-block:: xml - - <KeyInfo> - <Group name="fast"> - <Key type="rsa" bits="1024"/> - </Group> - <Group name="secure"> - <Key type="rsa" bits="4096"/> - </Group> - </KeyInfo> - -Automated Bcfg2 SSL Authentication -================================== - -This section describes one possible scenario for automating ssl -certificate generation and distribution for bcfg2 client/server -communication using SSLCA. The process involves configuring a -certificate authority (CA), generating the CA cert and key pair, -configuring the bcfg2 SSLCA plugin and a Bundle to use the SSLCA -generated certs to authenticate the bcfg2 client and server. - -OpenSSL CA ----------- - -If you already have a SSL CA available you can skip this section, -otherwise you can easily build one on the server using openssl. The -paths should be adjusted to suite your preferences. - -#. Prepare the directories and files:: - - mkdir -p /etc/pki/CA/newcerts - mkdir /etc/pki/CA/crl - echo '01' > /etc/pki/CA/serial - touch /etc/pki/CA/index.txt - touch /etc/pki/CA/crlnumber - -#. Edit the ``openssl.cnf`` config file, and in the **[ CA_default ]** - section adjust the following parameters:: - - dir = /etc/pki # Where everything is kept - certs = /etc/pki/CA/certs # Where the issued certs are kept - database = /etc/pki/CA/index.txt # database index file. - new_certs_dir = /etc/pki/CA/newcerts # default place for new certs. - certificate = /etc/pki/CA/certs/bcfg2ca.crt # The CA certificate - serial = /etc/pki/CA/serial # The current serial number - crl_dir = /etc/pki/CA/crl # Where the issued crl are kept - crlnumber = /etc/pki/CA/crlnumber # the current crl number - crl = /etc/pki/CA/crl.pem # The current CRL - private_key = /etc/pki/CA/private/bcfg2ca.key # The private key - -#. Create the CA root certificate and key pair. You'll be asked to - supply a passphrase, and some organizational info. The most - important bit is **Common Name** which you should set to be the - hostname of your bcfg2 server that your clients will see when doing - a reverse DNS query on it's ip address.:: - - openssl req -new -x509 -extensions v3_ca -keyout bcfg2ca.key \ - -out bcfg2ca.crt -days 3650 - -#. Move the generated cert and key to the locations specified in - ``openssl.cnf``:: - - mv bcfg2ca.key /etc/pki/CA/private/ - mv bcfg2ca.crt /etc/pki/CA/certs/ - -Your self-signing CA is now ready to use. - -Bcfg2 ------ - -SSLCA -^^^^^ - -The SSLCA plugin was not designed specifically to manage bcfg2 -client/server communication though it is certainly able to provide -certificate generation and management services for that -purpose. You'll need to configure the **SSLCA** plugin to serve the -key, and certificate paths that we will define later in our client's -``bcfg2.conf`` file. - -The rest of these instructions will assume that you've configured the -**SSLCA** plugin as described above and that the files -``SSLCA/etc/pki/tls/certs/bcfg2client.crt/cert.xml`` and -``SSLCA/etc/pki/tls/private/bcfg2client.key/key.xml`` represent the -cert and key paths you want generated for SSL auth. - -Client Bundle -^^^^^^^^^^^^^ - -To automate the process of generating and distributing certs to the -clients we need define at least the Cert and Key paths served by the -SSLCA plugin, as well as the ca certificate path in a Bundle. For -example: - -.. code-block:: xml - - <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> - <Path name='/etc/pki/tls/bcfg2client.crt'/> - <Path name='/etc/pki/tls/private/bcfg2client.key'/> - -Here's a more complete example bcfg2-client bundle: - -.. code-block:: xml - - <Bundle> - <Path name='/etc/bcfg2.conf'/> - <Path name='/etc/cron.d/bcfg2-client'/> - <Package name='bcfg2'/> - <Service name='bcfg2'/> - <Group name='rpm'> - <Path name='/etc/sysconfig/bcfg2'/> - <Path name='/etc/pki/tls/certs/bcfg2ca.crt'/> - <Path name='/etc/pki/tls/certs/bcfg2client.crt'/> - <Path name='/etc/pki/tls/private/bcfg2client.key'/> - </Group> - <Group name='deb'> - <Path name='/etc/default/bcfg2' altsrc='/etc/sysconfig/bcfg2'/> - <Path name='/etc/ssl/certs/bcfg2ca.crt' altsrc='/etc/pki/tls/certs/bcfg2ca.crt'/> - <Path name='/etc/ssl/certs/bcfg2client.crt' altsrc='/etc/pki/tls/certs/bcfg2client.crt'/> - <Path name='/etc/ssl/private/bcfg2client.key' altsrc='/etc/pki/tls/private/bcfg2client.key'/> - </Group> - </Bundle> - -In the above example we told Bcfg2 that it also needs to serve -``/etc/bcfg2.conf``. This is optional but convenient. - -The ``bcfg2.conf`` client config needs at least 5 parameters set for -SSL auth. - -#. ``key`` : This is the host specific key that SSLCA will generate. -#. ``certificate`` : This is the host specific cert that SSLCA will - generate. -#. ``ca`` : This is a copy of your CA certificate. Not generated by - SSLCA. -#. ``user`` : Usually set to fqdn of client. This *shouldn't* be - required but is as of 1.3.0. See: - http://trac.mcs.anl.gov/projects/bcfg2/ticket/1019 -#. ``password`` : Set to arbitrary string when using certificate - auth. This also *shouldn't* be required. See: - http://trac.mcs.anl.gov/projects/bcfg2/ticket/1019 - -Here's what a functional **[communication]** section in a -``bcfg2.conf`` genshi template for clients might look like.:: - - [communication] - protocol = xmlrpc/ssl - {% if metadata.uuid != None %}\ - user = ${metadata.uuid} - {% end %}\ - password = DUMMYPASSWORDFORCERTAUTH - {% choose %}\ - {% when 'rpm' in metadata.groups %}\ - certificate = /etc/pki/tls/certs/bcfg2client.crt - key = /etc/pki/tls/private/bcfg2client.key - ca = /etc/pki/tls/certs/bcfg2ca.crt - {% end %}\ - {% when 'deb' in metadata.groups %}\ - certificate = /etc/ssl/certs/bcfg2client.crt - key = /etc/ssl/private/bcfg2client.key - ca = /etc/ssl/certs/bcfg2ca.crt - {% end %}\ - {% end %}\ - -As a client will not be able to authenticate with certificates it does -not yet posses we need to overcome the chicken and egg scenario the -first time we try to connect such a client to the server. We can do so -using password based auth to boot strap the client manually specifying -all the relevant auth parameters like so:: - - bcfg2 -qv -S https://fqdn.of.bcfg2-server:6789 -u fqdn.of.client \ - -x SUPER_SECRET_PASSWORD - -If all goes well the client should recieve a freshly generated key and -cert and you should be able to run ``bcfg2`` again without specifying -the connection parameters. - -If you do run into problems you may want to review -:ref:`appendix-guides-authentication`. - -TODO -==== - -#. Add generation of pkcs12 format certs diff --git a/doc/server/plugins/misc/acl.txt b/doc/server/plugins/misc/acl.txt index 73f99bf85..226b56a44 100644 --- a/doc/server/plugins/misc/acl.txt +++ b/doc/server/plugins/misc/acl.txt @@ -189,7 +189,7 @@ The ACL descriptions allow you to use '*' as a wildcard for any number of characters *other than* ``.``. That is: * ``*`` would match ``DeclareVersion`` and ``GetProbes``, but would - *not* match ``Git.Update`. + *not* match ``Git.Update``. * ``*.*`` would match ``Git.Update``, but not ``DeclareVersion`` or ``GetProbes``. @@ -200,3 +200,36 @@ could also do something like ``*.toggle_debug`` to allow a host to enable or disable debugging for all plugins. No other bash globbing is supported. + +Examples +======== + +The :ref:`default ACL list <server-access-control>` can be described +in ``ip.xml`` fairly simply: + +.. code-block:: xml + + <ACL> + <Allow address="127.0.0.1" method="*.*"/> + <Allow address="127.0.0.1" method="*"/> + <Deny method="*.*"/> + <Deny method="*_debug"/> + <Deny method="get_statistics"/> + <Allow method="*"/> + </ACL> + +A basic configuration that is still very secure but perhaps more +functional could be given in ``metadata.xml``: + +.. code-block:: xml + + <ACL> + <Group name="bcfg2-server"> + <Allow method="*.*"/> + <Allow method="*"/> + </Group> + <Deny method="*.*"/> + <Deny method="*_debug"/> + <Deny method="get_statistics"/> + <Allow method="*"/> + </ACL> diff --git a/doc/server/plugins/structures/bundler/bcfg2.txt b/doc/server/plugins/structures/bundler/bcfg2.txt index 7465f15cb..0fd0a3fdf 100644 --- a/doc/server/plugins/structures/bundler/bcfg2.txt +++ b/doc/server/plugins/structures/bundler/bcfg2.txt @@ -52,7 +52,7 @@ entries between Bundler and Rules. <BoundPOSIXUser name='bcfg2' shell='/sbin/nologin' gecos='Bcfg2 User'/> <Path name="/home/bcfg2/.ssh/id_rsa"/> - <!-- SSLCA setup --> + <!-- SSL CA setup --> <BoundPath name="/etc/pki/CA" type="directory" important="true" owner="bcfg2" group="bcfg2" mode="755"/> <BoundPath name="/etc/pki/CA/crl" type="directory" owner="bcfg2" @@ -85,4 +85,3 @@ entries between Bundler and Rules. name="create-CA-crlnumber" timing="post" when="always" status="check" command="[ -e /etc/pki/CA/crlnumber ] || touch /etc/pki/CA/crlnumber"/> </Bundle> - diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt index 25134cb89..0b6b8eb50 100644 --- a/doc/server/plugins/structures/bundler/index.txt +++ b/doc/server/plugins/structures/bundler/index.txt @@ -293,6 +293,7 @@ more complex example Bundles. .. toctree:: :maxdepth: 1 + bcfg2 kernel moab nagios diff --git a/doc/server/xml-common.txt b/doc/server/xml-common.txt index 073e409b2..3aacfd468 100644 --- a/doc/server/xml-common.txt +++ b/doc/server/xml-common.txt @@ -299,7 +299,7 @@ such an included file to conform to the schema, although in general the included files should be structure exactly like the parent file. Wildcard XInclude -~~~~~~~~~~~~~~~~~ +----------------- .. versionadded:: 1.3.1 @@ -324,60 +324,60 @@ tag, described above, if a glob may potentially find no files. Feature Matrix ============== -+-------------------------------------------------+--------------+--------+------------+------------+ -| File | Group/Client | Genshi | Encryption | XInclude | -+=================================================+==============+========+============+============+ -| :ref:`ACL ip.xml <server-plugins-misc-acl>` | No | No | No | Yes | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`ACL metadata.xml | Yes | Yes | Yes | Yes | -| <server-plugins-misc-acl>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Bundler | Yes | Yes | Yes | Yes | -| <server-plugins-structures-bundler-index>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`info.xml <server-info>` | Yes [#f1]_ | Yes | Yes | Yes | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`privkey.xml and pubkey.xml | Yes | Yes | Yes | Yes [#f2]_ | -| <server-plugins-generators-cfg-sshkeys>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`authorizedkeys.xml | Yes | Yes | Yes | Yes | -| <server-plugins-generators-cfg-sshkeys>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Decisions | Yes | Yes | Yes | Yes | -| <server-plugins-generators-decisions>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Defaults | Yes | Yes | Yes | Yes | -| <server-plugins-structures-defaults>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`FileProbes | Yes | Yes | Yes | Yes | -| <server-plugins-probes-fileprobes>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`GroupPatterns | No | No | No | Yes | -| <server-plugins-grouping-grouppatterns>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Metadata clients.xml | No | No | No | Yes | -| <server-plugins-grouping-metadata-clients-xml>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Metadata groups.xml | Yes [#f3]_ | No | No | Yes | -| <server-plugins-grouping-metadata-groups-xml>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`NagiosGen | Yes | Yes | Yes | Yes | -| <server-plugins-generators-nagiosgen>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Packages | Yes | Yes | Yes | Yes | -| <server-plugins-generators-packages>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Pkgmgr | Yes | No | No | No | -| <server-plugins-generators-pkgmgr>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Properties | Yes [#f4]_ | Yes | Yes | Yes | -| <server-plugins-connectors-properties>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`Rules <server-plugins-generators-rules>` | Yes | Yes | Yes | Yes | -+-------------------------------------------------+--------------+--------+------------+------------+ -| :ref:`SSLCA cert.xml and key.xml | Yes | Yes | Yes | Yes | -| <server-plugins-generators-sslca>` | | | | | -+-------------------------------------------------+--------------+--------+------------+------------+ ++---------------------------------------------------+--------------+--------+------------+------------+ +| File | Group/Client | Genshi | Encryption | XInclude | ++===================================================+==============+========+============+============+ +| :ref:`ACL ip.xml <server-plugins-misc-acl>` | No | No | No | Yes | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`ACL metadata.xml | Yes | Yes | Yes | Yes | +| <server-plugins-misc-acl>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Bundler | Yes | Yes | Yes | Yes | +| <server-plugins-structures-bundler-index>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`info.xml <server-info>` | Yes [#f1]_ | Yes | Yes | Yes | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`privkey.xml and pubkey.xml | Yes | Yes | Yes | Yes [#f2]_ | +| <server-plugins-generators-cfg-sshkeys>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`authorizedkeys.xml | Yes | Yes | Yes | Yes | +| <server-plugins-generators-cfg-sshkeys>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`sslcert.xml and sslkey.xml | Yes | Yes | Yes | Yes | +| <server-plugins-generators-cfg-ssl-certificates>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Decisions | Yes | Yes | Yes | Yes | +| <server-plugins-generators-decisions>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Defaults | Yes | Yes | Yes | Yes | +| <server-plugins-structures-defaults>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`FileProbes | Yes | Yes | Yes | Yes | +| <server-plugins-probes-fileprobes>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`GroupPatterns | No | No | No | Yes | +| <server-plugins-grouping-grouppatterns>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Metadata clients.xml | No | No | No | Yes | +| <server-plugins-grouping-metadata-clients-xml>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Metadata groups.xml | Yes [#f3]_ | No | No | Yes | +| <server-plugins-grouping-metadata-groups-xml>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`NagiosGen | Yes | Yes | Yes | Yes | +| <server-plugins-generators-nagiosgen>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Packages | Yes | Yes | Yes | Yes | +| <server-plugins-generators-packages>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Pkgmgr | Yes | No | No | No | +| <server-plugins-generators-pkgmgr>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Properties | Yes [#f4]_ | Yes | Yes | Yes | +| <server-plugins-connectors-properties>` | | | | | ++---------------------------------------------------+--------------+--------+------------+------------+ +| :ref:`Rules <server-plugins-generators-rules>` | Yes | Yes | Yes | Yes | ++---------------------------------------------------+--------------+--------+------------+------------+ .. rubric:: Footnotes diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 5e64caae9..91ebc0020 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -153,7 +153,6 @@ SEModules ServiceCompat Snapshots SSHbase -SSLCA Statistics Svn TCheetah diff --git a/schemas/authorizedkeys.xsd b/schemas/authorizedkeys.xsd index 20e568a07..c3cd50181 100644 --- a/schemas/authorizedkeys.xsd +++ b/schemas/authorizedkeys.xsd @@ -49,6 +49,42 @@ <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> + <xsd:complexType name="OptionContainerType"> + <xsd:annotation> + <xsd:documentation> + An **OptionContainerType** is a tag used to provide logic. + Child entries of an OptionContainerType tag only apply to + machines that match the condition specified -- either + membership in a group, or a matching client name. + :xml:attribute:`OptionContainerType:negate` can be set to + negate the sense of the match. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="0" maxOccurs="unbounded"> + <xsd:element name="Group" type="OptionContainerType"/> + <xsd:element name="Client" type="OptionContainerType"/> + <xsd:element name="Option" type="AuthorizedKeysOptionType"/> + </xsd:choice> + <xsd:attribute name='name' type='xsd:string'> + <xsd:annotation> + <xsd:documentation> + The name of the client or group to match on. Child entries + will only apply to this client or group (unless + :xml:attribute:`OptionContainerType:negate` is set). + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name='negate' type='xsd:boolean'> + <xsd:annotation> + <xsd:documentation> + Negate the sense of the match, so that child entries only + apply to a client if it is not a member of the given group + or does not have the given name. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + <xsd:complexType name="AllowType" mixed="true"> <xsd:annotation> <xsd:documentation> @@ -58,7 +94,9 @@ </xsd:annotation> <xsd:choice minOccurs="0" maxOccurs="unbounded"> <xsd:group ref="py:genshiElements"/> - <xsd:element name="Params" type="AuthorizedKeysParamsType"/> + <xsd:element name="Group" type="OptionContainerType"/> + <xsd:element name="Client" type="OptionContainerType"/> + <xsd:element name="Option" type="AuthorizedKeysOptionType"/> </xsd:choice> <xsd:attribute name="from" type="xsd:string"> <xsd:annotation> @@ -86,16 +124,28 @@ <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> - <xsd:complexType name="AuthorizedKeysParamsType"> + <xsd:complexType name="AuthorizedKeysOptionType"> <xsd:annotation> <xsd:documentation> - Specify parameters for public key authentication and - connection. See :manpage:`sshd(8)` for details on allowable - parameters. + Specify options for public key authentication and connection. + See :manpage:`sshd(8)` for details on allowable options. </xsd:documentation> </xsd:annotation> - <xsd:attributeGroup ref="py:genshiAttrs"/> - <xsd:anyAttribute processContents="lax"/> + <xsd:attribute name="name" type="xsd:string" use="required"> + <xsd:annotation> + <xsd:documentation> + The name of the sshd option. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="value" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + The value of the sshd option. This can be omitted for + options that take no value. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> <xsd:complexType name="AuthorizedKeysType"> diff --git a/schemas/sslca-cert.xsd b/schemas/sslca-cert.xsd index a3f6db94d..7330ca0ff 100644 --- a/schemas/sslca-cert.xsd +++ b/schemas/sslca-cert.xsd @@ -2,7 +2,8 @@ xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> - Schema for :ref:`server-plugins-generators-sslca` ``cert.xml`` + Schema for :ref:`server-plugins-generators-cfg-ssl-certificates` + ``sslcert.xml`` </xsd:documentation> </xsd:annotation> @@ -76,7 +77,7 @@ <xsd:documentation> The full path to the key entry to use for this certificate. This is the *client* path; e.g., for a key defined at - ``/var/lib/bcfg2/SSLCA/etc/pki/tls/private/foo.key/key.xml``, + ``/var/lib/bcfg2/SSLCA/etc/pki/tls/private/foo.key/sslkey.xml``, **key** should be ``/etc/pki/tls/private/foo.key``. </xsd:documentation> </xsd:annotation> @@ -92,8 +93,8 @@ <xsd:annotation> <xsd:documentation> The name of the CA (from :ref:`bcfg2.conf - <sslca-configuration>`) to use to generate this - certificate. + <server-plugins-generators-cfg-configuration>`) to use + to generate this certificate. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/schemas/sslca-key.xsd b/schemas/sslca-key.xsd index 261b71e1a..496da859f 100644 --- a/schemas/sslca-key.xsd +++ b/schemas/sslca-key.xsd @@ -2,7 +2,8 @@ xmlns:py="http://genshi.edgewall.org/" xml:lang="en"> <xsd:annotation> <xsd:documentation> - Schema for :ref:`server-plugins-generators-sslca` ``key.xml`` + Schema for :ref:`server-plugins-generators-cfg-ssl-certificates` + ``sslkey.xml`` </xsd:documentation> </xsd:annotation> @@ -91,11 +92,26 @@ <xsd:element name="Client" type="SSLCAKeyGroupType"/> <xsd:element name="KeyInfo" type="KeyInfoType"/> </xsd:choice> - <xsd:attribute name="lax_decryption" type="xsd:boolean"> + <xsd:attribute name="perhost" type="xsd:boolean"> <xsd:annotation> <xsd:documentation> - Override the global lax_decryption setting in - ``bcfg2.conf``. + Create keys on a per-host basis (rather than on a per-group + basis). + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="category" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + Create keys specific to the given category, instead of + specific to the category given in ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="priority" type="xsd:positiveInteger" default="50"> + <xsd:annotation> + <xsd:documentation> + Create group-specific keys with the given priority. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/schemas/types.xsd b/schemas/types.xsd index 5dec03cdb..5abc35144 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -115,7 +115,10 @@ <xsd:attribute type='ActionTimingEnum' name='timing'> <xsd:annotation> <xsd:documentation> - When the action is run. + When the action is run. Actions with "pre" timing are run + after important entries have been installed and before + bundle entries are installed. Actions with "post" timing + are run after bundle entries are installed. </xsd:documentation> </xsd:annotation> </xsd:attribute> @@ -123,9 +126,7 @@ <xsd:annotation> <xsd:documentation> If the action is always run, or is only run when a bundle - has been modified. Actions that run before bundle - installation ("pre" and "both") ignore the setting of - ``when`` and are always run regardless. + has been modified. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 482320e0d..d7a70e202 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -54,6 +54,10 @@ class POSIXFile(POSIXTool): def verify(self, entry, modlist): ondisk = self._exists(entry) tempdata, is_binary = self._get_data(entry) + if isinstance(tempdata, str) and str != unicode: + tempdatasize = len(tempdata) + else: + tempdatasize = len(tempdata.encode(Bcfg2.Options.setup.encoding)) different = False content = None @@ -62,7 +66,7 @@ class POSIXFile(POSIXTool): # they're clearly different different = True content = "" - elif len(tempdata) != ondisk[stat.ST_SIZE]: + elif tempdatasize != ondisk[stat.ST_SIZE]: # next, see if the size of the target file is different # from the size of the desired content different = True @@ -73,6 +77,9 @@ class POSIXFile(POSIXTool): # for everything else try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), + encoding=Bcfg2.Options.setup.encoding).read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) @@ -90,7 +97,7 @@ class POSIXFile(POSIXTool): def _write_tmpfile(self, entry): """ Write the file data to a temp file """ - filedata, _ = self._get_data(entry) + filedata = self._get_data(entry)[0] # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions # protections on that directory, and also to avoid issues with @@ -106,7 +113,11 @@ class POSIXFile(POSIXTool): (os.path.dirname(entry.get('name')), err)) return False try: - os.fdopen(newfd, 'w').write(filedata) + if isinstance(filedata, str) and str != unicode: + os.fdopen(newfd, 'w').write(filedata) + else: + os.fdopen(newfd, 'wb').write( + filedata.encode(Bcfg2.Options.setup.encoding)) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index c9164cb88..bd2f8f87e 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -525,7 +525,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if entry.get("secontext") == "__default__": try: wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + selinux.matchpathcon( + path, ondisk[stat.ST_MODE])[1].split(":")[2] except OSError: errors.append("%s has no default SELinux context" % entry.get("name")) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index aab2459f2..4a808aa60 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -590,14 +590,15 @@ class SvcTool(Tool): if not self.handlesEntry(entry): continue + estatus = entry.get('status') restart = entry.get("restart", "true").lower() - if (restart == "false" or + if (restart == "false" or estatus == 'ignore' or (restart == "interactive" and not Bcfg2.Options.setup.interactive)): continue success = False - if entry.get('status') == 'on': + if estatus == 'on': if Bcfg2.Options.setup.service_mode == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 3cba93fff..433fb570a 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -12,9 +12,9 @@ import argparse import tempfile import Bcfg2.Logger import Bcfg2.Options -import XML # pylint: disable=W0403 -import Proxy # pylint: disable=W0403 -import Tools # pylint: disable=W0403 +from Bcfg2.Client import XML +from Bcfg2.Client import Proxy +from Bcfg2.Client import Tools from Bcfg2.Utils import locked, Executor, safe_input from Bcfg2.version import __version__ # pylint: disable=W0622 @@ -88,7 +88,6 @@ class Client(object): options = Proxy.ComponentProxy.options + [ Bcfg2.Options.Common.syslog, - Bcfg2.Options.Common.location, Bcfg2.Options.Common.interactive, Bcfg2.Options.BooleanOption( "-q", "--quick", help="Disable some checksum verification"), @@ -194,7 +193,11 @@ class Client(object): ret = XML.Element("probe-data", name=name, source=probe.get('source')) try: scripthandle, scriptname = tempfile.mkstemp() - script = os.fdopen(scripthandle, 'w') + if sys.hexversion >= 0x03000000: + script = os.fdopen(scripthandle, 'w', + encoding=Bcfg2.Options.setup.encoding) + else: + script = os.fdopen(scripthandle, 'w') try: script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) @@ -666,13 +669,15 @@ class Client(object): # first process prereq actions for bundle in bundles[:]: if bundle.tag == 'Bundle': - bmodified = any(item in self.whitelist for item in bundle) + bmodified = any((item in self.whitelist or + item in self.modified) for item in bundle) else: bmodified = False actions = [a for a in bundle.findall('./Action') if (a.get('timing') in ['pre', 'both'] and (bmodified or a.get('when') == 'always'))] - # now we process all "always actions" + # now we process all "pre" and "both" actions that are either + # always or the bundle has been modified if Bcfg2.Options.setup.interactive: self.promptFilter(iprompt, actions) self.DispatchInstallCalls(actions) diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index f9fd42d33..0f7995e0f 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -236,7 +236,6 @@ def setup_logging(): logging.root.setLevel(logging.DEBUG) logging.root.debug("Configured logging: %s" % "; ".join(params)) - print("Configured logging: %s" % "; ".join(params)) logging.already_setup = True diff --git a/src/lib/Bcfg2/Options/Actions.py b/src/lib/Bcfg2/Options/Actions.py index 637a09577..8b97f1da8 100644 --- a/src/lib/Bcfg2/Options/Actions.py +++ b/src/lib/Bcfg2/Options/Actions.py @@ -2,7 +2,7 @@ import sys import argparse -from Parser import get_parser # pylint: disable=W0403 +from Bcfg2.Options.Parser import get_parser __all__ = ["ConfigFileAction", "ComponentAction", "PluginsAction"] diff --git a/src/lib/Bcfg2/Options/Common.py b/src/lib/Bcfg2/Options/Common.py index eb4af5bb6..9ba08eb87 100644 --- a/src/lib/Bcfg2/Options/Common.py +++ b/src/lib/Bcfg2/Options/Common.py @@ -1,12 +1,10 @@ """ Common options used in multiple different contexts. """ from Bcfg2.Utils import classproperty -# pylint: disable=W0403 -import Types -from Actions import PluginsAction, ComponentAction -from Parser import repository as _repository_option -from Options import Option, PathOption, BooleanOption -# pylint: enable=W0403 +from Bcfg2.Options import Types +from Bcfg2.Options.Actions import PluginsAction, ComponentAction +from Bcfg2.Options.Parser import repository as _repository_option +from Bcfg2.Options import Option, PathOption, BooleanOption __all__ = ["Common"] diff --git a/src/lib/Bcfg2/Options/OptionGroups.py b/src/lib/Bcfg2/Options/OptionGroups.py index 70cb5d0dd..465358fab 100644 --- a/src/lib/Bcfg2/Options/OptionGroups.py +++ b/src/lib/Bcfg2/Options/OptionGroups.py @@ -3,7 +3,7 @@ import re import copy import fnmatch -from Options import Option # pylint: disable=W0403 +from Bcfg2.Options import Option from itertools import chain __all__ = ["OptionGroup", "ExclusiveOptionGroup", "Subparser", diff --git a/src/lib/Bcfg2/Options/Options.py b/src/lib/Bcfg2/Options/Options.py index d60c536cf..be7e7c646 100644 --- a/src/lib/Bcfg2/Options/Options.py +++ b/src/lib/Bcfg2/Options/Options.py @@ -4,9 +4,9 @@ need to be associated with an option parser; it exists on its own.""" import os import copy -import Types # pylint: disable=W0403 import fnmatch import argparse +from Bcfg2.Options import Types from Bcfg2.Compat import ConfigParser diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index dd7874d35..bede85a1f 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -5,7 +5,7 @@ import sys import argparse from Bcfg2.version import __version__ from Bcfg2.Compat import ConfigParser -from Options import Option, PathOption, BooleanOption # pylint: disable=W0403 +from Bcfg2.Options import Option, PathOption, BooleanOption __all__ = ["setup", "OptionParserException", "Parser", "get_parser"] @@ -201,8 +201,7 @@ class Parser(argparse.ArgumentParser): # check whether the specified bcfg2.conf exists if not os.path.exists(bootstrap.config): - print("Could not read %s" % bootstrap.config) - return 1 + self.error("Could not read %s" % bootstrap.config) self.add_config_file(self.configfile.dest, bootstrap.config) # phase 2: re-parse command line, loading additional @@ -212,7 +211,7 @@ class Parser(argparse.ArgumentParser): while not self.parsed: self.parsed = True self._set_defaults() - self.parse_known_args(namespace=self.namespace) + self.parse_known_args(args=self.argv, namespace=self.namespace) self._parse_config_options() self._finalize() self._parse_config_options() diff --git a/src/lib/Bcfg2/Options/Subcommands.py b/src/lib/Bcfg2/Options/Subcommands.py index 7d7a3f928..660bd5077 100644 --- a/src/lib/Bcfg2/Options/Subcommands.py +++ b/src/lib/Bcfg2/Options/Subcommands.py @@ -8,11 +8,9 @@ import copy import shlex import logging from Bcfg2.Compat import StringIO -# pylint: disable=W0403 -from OptionGroups import Subparser -from Options import PositionalArgument -from Parser import Parser, setup as master_setup -# pylint: enable=W0403 +from Bcfg2.Options import PositionalArgument +from Bcfg2.Options.OptionGroups import Subparser +from Bcfg2.Options.Parser import Parser, setup as master_setup __all__ = ["Subcommand", "HelpCommand", "CommandRegistry", "register_commands"] diff --git a/src/lib/Bcfg2/Options/__init__.py b/src/lib/Bcfg2/Options/__init__.py index 546068f1f..96465ec56 100644 --- a/src/lib/Bcfg2/Options/__init__.py +++ b/src/lib/Bcfg2/Options/__init__.py @@ -1,10 +1,10 @@ """ Bcfg2 options parsing. """ -# pylint: disable=W0611,W0401,W0403 -import Types -from Common import * -from Parser import * -from Actions import * -from Options import * -from Subcommands import * -from OptionGroups import * +# pylint: disable=W0611,W0401 +from Bcfg2.Options import Types +from Bcfg2.Options.Options import * +from Bcfg2.Options.Common import * +from Bcfg2.Options.Parser import * +from Bcfg2.Options.Actions import * +from Bcfg2.Options.Subcommands import * +from Bcfg2.Options.OptionGroups import * diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index a1e6025e3..a93f1b0ae 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -44,15 +44,11 @@ class ReportingCollector(object): else: level = logging.WARNING - Bcfg2.Logger.setup_logging('bcfg2-report-collector', - to_console=logging.INFO, - to_syslog=Bcfg2.Options.setup.syslog, - to_file=Bcfg2.Options.setup.logging, - level=level) + Bcfg2.Logger.setup_logging() self.logger = logging.getLogger('bcfg2-report-collector') try: - self.transport = Bcfg2.Options.setup.transport() + self.transport = Bcfg2.Options.setup.reporting_transport() self.storage = Bcfg2.Options.setup.reporting_storage() except TransportError: self.logger.error("Failed to load transport: %s" % @@ -82,7 +78,7 @@ class ReportingCollector(object): """Startup the processing and go!""" self.terminate = threading.Event() atexit.register(self.shutdown) - self.context = daemon.DaemonContext() + self.context = daemon.DaemonContext(detach_process=True) if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 9505682a7..69da9c571 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -378,7 +378,7 @@ class DjangoORM(StorageBase): def validate(self): """Validate backend storage. Should be called once when loaded""" - settings.read_config(repo=Bcfg2.Options.setup.repository) + settings.read_config() # verify our database schema try: diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index d901ded56..189967cb0 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -10,7 +10,6 @@ import select import time import traceback import Bcfg2.Options -import Bcfg2.CommonOptions import Bcfg2.Server.FileMonitor from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError from Bcfg2.Reporting.Transport.base import TransportBase, TransportError diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py index 9fbf8c9d5..9a0a4262f 100644 --- a/src/lib/Bcfg2/Reporting/Transport/base.py +++ b/src/lib/Bcfg2/Reporting/Transport/base.py @@ -4,6 +4,7 @@ The base for all server -> collector Transports import os import sys +import Bcfg2.Options from Bcfg2.Logger import Debuggable diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 24cb46bd9..59059c240 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -59,7 +59,7 @@ class ccolors: # pylint: disable=C0103,W0232 def gen_password(length): """Generates a random alphanumeric password with length characters.""" - chars = string.letters + string.digits + chars = string.ascii_letters + string.digits return "".join(random.choice(chars) for i in range(length)) @@ -1091,7 +1091,7 @@ class Viz(_ServerAdminCmd): help="Show a key for different digraph shapes"), Bcfg2.Options.Option( "-c", "--only-client", metavar="<hostname>", - help="Show only the groups, bundles for the named client"), + help="Only show groups and bundles for the named client"), Bcfg2.Options.PathOption( "-o", "--outfile", help="Write viz output to an output file")] diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 179a6aa9f..0023e9313 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -113,7 +113,6 @@ class BuiltinCore(NetworkCore): keyfile=Bcfg2.Options.setup.key, certfile=Bcfg2.Options.setup.cert, register=False, - timeout=1, ca=Bcfg2.Options.setup.ca) except: # pylint: disable=W0702 err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Server/Cache.py b/src/lib/Bcfg2/Server/Cache.py index 842098eda..d05eb0bf6 100644 --- a/src/lib/Bcfg2/Server/Cache.py +++ b/src/lib/Bcfg2/Server/Cache.py @@ -1,14 +1,180 @@ -""" An implementation of a simple memory-backed cache. Right now this -doesn't provide many features, but more (time-based expiration, etc.) -can be added as necessary. """ +""" ``Bcfg2.Server.Cache`` is an implementation of a simple +memory-backed cache. Right now this doesn't provide many features, but +more (time-based expiration, etc.) can be added as necessary. +The normal workflow is to get a Cache object, which is simply a dict +interface to the unified cache that automatically uses a certain tag +set. For instance: -class Cache(dict): - """ an implementation of a simple memory-backed cache """ +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache['foo.example.com'] = ['group1', 'group2'] + +This would create a Cache object that automatically tags its entries +with ``frozenset(["Probes", "probegroups"])``, and store the list +``['group1', 'group1']`` with the *additional* tag +``foo.example.com``. So the unified backend cache would then contain +a single entry: + +.. code-block:: python + + {frozenset(["Probes", "probegroups", "foo.example.com"]): + ['group1', 'group2']} + +In addition to the dict interface, Cache objects (returned from +:func:`Bcfg2.Server.Cache.Cache`) have one additional method, +``expire()``, which is mostly identical to +:func:`Bcfg2.Server.Cache.expire`, except that it is specific to the +tag set of the cache object. E.g., to expire all ``foo.example.com`` +records for a given cache, you could do: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache.expire("foo.example.com") + +This is mostly functionally identical to: + +.. code-block:: python + + Bcfg2.Server.Cache.expire("Probes", "probegroups", "foo.example.com") + +It's not completely identical, though; the first example will expire, +at most, exactly one item from the cache. The second example will +expire all items that are tagged with a superset of the given tags. +To illustrate the difference, consider the following two examples: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes") + groupcache.expire("probegroups") + + Bcfg2.Server.Cache.expire("Probes", "probegroups") + +The former will not expire any data, because there is no single datum +tagged with ``"Probes", "probegroups"``. The latter will expire *all* +items tagged with ``"Probes", "probegroups"`` -- i.e., the entire +cache. In this case, the latter call is equivalent to: + +.. code-block:: python + + groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + groupcache.expire() + +""" + +from Bcfg2.Compat import MutableMapping + + +class _Cache(MutableMapping): + """ The object returned by :func:`Bcfg2.Server.Cache.Cache` that + presents a dict-like interface to the portion of the unified cache + that uses the specified tags. """ + def __init__(self, registry, tags): + self._registry = registry + self._tags = tags + + def __getitem__(self, key): + return self._registry[self._tags | set([key])] + + def __setitem__(self, key, value): + self._registry[self._tags | set([key])] = value + + def __delitem__(self, key): + del self._registry[self._tags | set([key])] + + def __iter__(self): + for item in self._registry.iterate(*self._tags): + yield list(item.difference(self._tags))[0] + + def keys(self): + """ List cache keys """ + return list(iter(self)) + + def __len__(self): + return len(list(iter(self))) def expire(self, key=None): """ expire all items, or a specific item, from the cache """ if key is None: - self.clear() - elif key in self: - del self[key] + expire(*self._tags) + else: + tags = self._tags | set([key]) + # py 2.5 doesn't support mixing *args and explicit keyword + # args + kwargs = dict(exact=True) + expire(*tags, **kwargs) + + def __repr__(self): + return repr(dict(self)) + + def __str__(self): + return str(dict(self)) + + +class _CacheRegistry(dict): + """ The grand unified cache backend which contains all cache + items. """ + + def iterate(self, *tags): + """ Iterate over all items that match the given tags *and* + have exactly one additional tag. This is used to get items + for :class:`Bcfg2.Server.Cache._Cache` objects that have been + instantiated via :func:`Bcfg2.Server.Cache.Cache`. """ + tags = frozenset(tags) + for key in self.keys(): + if key.issuperset(tags) and len(key.difference(tags)) == 1: + yield key + + def iter_all(self, *tags): + """ Iterate over all items that match the given tags, + regardless of how many additional tags they have (or don't + have). This is used to expire all cache data that matches a + set of tags. """ + tags = frozenset(tags) + for key in list(self.keys()): + if key.issuperset(tags): + yield key + + +_cache = _CacheRegistry() # pylint: disable=C0103 +_hooks = [] # pylint: disable=C0103 + + +def Cache(*tags): # pylint: disable=C0103 + """ A dict interface to the cache data tagged with the given + tags. """ + return _Cache(_cache, frozenset(tags)) + + +def expire(*tags, **kwargs): + """ Expire all items, a set of items, or one specific item from + the cache. If ``exact`` is set to True, then if the given tag set + doesn't match exactly one item in the cache, nothing will be + expired. """ + exact = kwargs.pop("exact", False) + count = 0 + if not tags: + count = len(_cache) + _cache.clear() + elif exact: + if frozenset(tags) in _cache: + count = 1 + del _cache[frozenset(tags)] + else: + for match in _cache.iter_all(*tags): + count += 1 + del _cache[match] + + for hook in _hooks: + hook(tags, exact, count) + + +def add_expire_hook(func): + """ Add a hook that will be called when an item is expired from + the cache. The callable passed in must take three options: the + first will be the tag set that was expired; the second will be the + state of the ``exact`` flag (True or False); and the third will be + the number of items that were expired from the cache. """ + _hooks.append(func) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 501a78bc0..69d61580f 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -23,6 +23,7 @@ from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics +from Bcfg2.Server.Plugins.Metadata import MetadataGroup try: import psyco @@ -78,9 +79,23 @@ class NoExposedMethod (Exception): method exposed with the given name. """ -# pylint: disable=W0702 +class DefaultACL(Plugin, ClientACLs): + """ Default ACL 'plugin' that provides security by default. This + is only loaded if no other ClientACLs plugin is enabled. """ + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.ClientACLs.__init__(self) + + def check_acl_ip(self, address, rmi): + return (("." not in rmi and + not rmi.endswith("_debug") and + rmi != 'get_statistics') or + address[0] == "127.0.0.1") + + # in core we frequently want to catch all exceptions, regardless of # type, so disable the pylint rule that catches that. +# pylint: disable=W0702 class Core(object): """ The server core is the container for all Bcfg2 server logic @@ -186,6 +201,10 @@ class Core(object): # load plugins Bcfg2.settings.read_config() + # mapping of group name => plugin name to record where groups + # that are created by Connector plugins came from + self._dynamic_groups = dict() + #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -207,7 +226,7 @@ class Core(object): #: A :class:`Bcfg2.Server.Cache.Cache` object for caching client #: metadata - self.metadata_cache = Cache() + self.metadata_cache = Cache("Metadata") #: Whether or not it's possible to use the Django database #: backend for plugins that have that capability @@ -227,20 +246,6 @@ class Core(object): self.logger.error("Updating database %s failed: %s" % (Bcfg2.Options.setup.db_name, err)) - def expire_caches_by_type(self, base_cls, key=None): - """ Expire caches for all - :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that - are instances of ``base_cls``. - - :param base_cls: The base plugin interface class to match (see - :mod:`Bcfg2.Server.Plugin.interfaces`) - :type base_cls: type - :param key: The cache key to expire - """ - for plugin in self.plugins_by_type(base_cls): - if isinstance(plugin, Bcfg2.Server.Plugin.Caching): - plugin.expire_cache(key) - def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -296,7 +301,7 @@ class Core(object): continue self.logger.info("File monitor thread terminated") - @Bcfg2.Server.Statistics.track_statistics() + @track_statistics() def _update_vcs_revision(self): """ Update the revision of the current configuration on-disk from the VCS plugin """ @@ -358,14 +363,16 @@ class Core(object): "failed to instantiate Core") raise CoreInitError("No Metadata Plugin") + # ensure that an ACL plugin is loaded + if not self.plugins_by_type(Bcfg2.Server.Plugin.ClientACLs): + self.init_plugin(DefaultACL) + def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. - :param plugin: The name of the plugin. This is just the name - of the plugin, in the appropriate case. I.e., - ``Cfg``, not ``Bcfg2.Server.Plugins.Cfg``. - :type plugin: string + :param plugin: The plugin class to load. + :type plugin: type :returns: None """ self.logger.debug("Loading plugin %s" % plugin.name) @@ -683,7 +690,7 @@ class Core(object): if event.code2str() == 'deleted': return Bcfg2.Options.get_parser().reparse() - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.metadata_cache.expire() def block_for_fam_events(self, handle_events=False): """ Block until all fam events have been handleed, optionally @@ -848,8 +855,35 @@ class Core(object): (client_name, sys.exc_info()[1])) connectors = self.plugins_by_type(Connector) for conn in connectors: - grps = conn.get_additional_groups(imd) - self.metadata.merge_additional_groups(imd, grps) + groups = conn.get_additional_groups(imd) + groupnames = [] + for group in groups: + if isinstance(group, MetadataGroup): + groupname = group.name + if groupname in self._dynamic_groups: + if self._dynamic_groups[groupname] == conn.name: + self.metadata.groups[groupname] = group + else: + self.logger.warning( + "Refusing to clobber dynamic group %s " + "defined by %s" % + (self._dynamic_groups[groupname], + groupname)) + elif groupname in self.metadata.groups: + # not recorded as a dynamic group, but + # present in metadata.groups -- i.e., a + # static group + self.logger.warning( + "Refusing to clobber predefined group %s" % + groupname) + else: + self.metadata.groups[groupname] = group + self._dynamic_groups[groupname] = conn.name + groupnames.append(groupname) + else: + groupnames.append(group) + + self.metadata.merge_additional_groups(imd, groupnames) for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) @@ -1070,7 +1104,7 @@ class Core(object): # that's created for RecvProbeData doesn't get cached. # I.e., the next metadata object that's built, after probe # data is processed, is cached. - self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + self.metadata_cache.expire(client) try: xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index e64a6627f..dd3f46b07 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -215,7 +215,7 @@ class CryptoTool(object): """ get the passphrase for the current file """ if not Bcfg2.Options.setup.passphrases: raise PassphraseError("No passphrases available in %s" % - Bcfg2.Options.setup.configfile) + Bcfg2.Options.setup.config) pname = None if Bcfg2.Options.setup.passphrase: @@ -229,7 +229,7 @@ class CryptoTool(object): return (pname, passphrase) except KeyError: raise PassphraseError("Could not find passphrase %s in %s" % - (pname, Bcfg2.Options.setup.configfile)) + (pname, Bcfg2.Options.setup.config)) else: if len(Bcfg2.Options.setup.passphrases) == 1: pname, passphrase = Bcfg2.Options.setup.passphrases.items()[0] @@ -285,7 +285,7 @@ class CfgEncryptor(Encryptor): if self.passphrase is None: raise PassphraseError("Multiple passphrases found in %s, " "specify one on the command line with -p" % - Bcfg2.Options.setup.configfile) + Bcfg2.Options.setup.config) def encrypt(self): return ssl_encrypt(self.data, self.passphrase) @@ -367,19 +367,19 @@ class PropertiesCryptoMixin(object): """ Get the passphrase to use to encrypt or decrypt a given element """ pname = element.get("encrypted") - if pname in self.passphrases: - passphrase = self.passphrases[pname] + if pname in Bcfg2.Options.setup.passphrases: + passphrase = Bcfg2.Options.setup.passphrases[pname] elif self.passphrase: if pname: self.logger.warning("Passphrase %s not found in %s, " "using passphrase given on command line" % - (pname, Bcfg2.Options.setup.configfile)) + (pname, Bcfg2.Options.setup.config)) passphrase = self.passphrase pname = self.pname else: raise PassphraseError("Multiple passphrases found in %s, " "specify one on the command line with -p" % - Bcfg2.Options.setup.configfile) + Bcfg2.Options.setup.config) return (pname, passphrase) def _write(self, filename, data): @@ -579,7 +579,7 @@ class CLI(object): if data is None: data = getattr(tool, mode)() - if not data: + if data is None: self.logger.error("Failed to %s %s, skipping" % (mode, fname)) continue if Bcfg2.Options.setup.stdout: diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py index 64621aa53..3a1ed7433 100644 --- a/src/lib/Bcfg2/Server/Info.py +++ b/src/lib/Bcfg2/Server/Info.py @@ -92,7 +92,7 @@ class InfoCmd(Bcfg2.Options.Subcommand): # pylint: disable=W0223 """ Given a list of globs, select the items from candidates that match the globs """ # special cases to speed things up: - if globs is None or '*' in globs: + if not globs or '*' in globs: return candidates has_wildcards = False for glob in globs: diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 2f245561b..de7ae038a 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -39,8 +39,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Cfg/**/pubkey.xml": "pubkey.xsd", "Cfg/**/authorizedkeys.xml": "authorizedkeys.xsd", "Cfg/**/authorized_keys.xml": "authorizedkeys.xsd", + "Cfg/**/sslcert.xml": "sslca-cert.xsd", + "Cfg/**/sslkey.xml": "sslca-key.xsd", "SSHbase/**/info.xml": "info.xsd", - "SSLCA/**/info.xml": "info.xsd", "TGenshi/**/info.xml": "info.xsd", "TCheetah/**/info.xml": "info.xsd", "Bundler/*.xml": "bundle.xsd", @@ -55,8 +56,6 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "GroupPatterns/config.xml": "grouppatterns.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", - "SSLCA/**/cert.xml": "sslca-cert.xsd", - "SSLCA/**/key.xml": "sslca-key.xsd", "GroupLogic/groups.xml": "grouplogic.xsd" } diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 517140178..c42009bdd 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -16,32 +16,15 @@ import threading import lxml.etree import multiprocessing import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin from itertools import cycle -from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import Queue, Empty, wraps from Bcfg2.Server.Core import Core, exposed from Bcfg2.Server.BuiltinCore import BuiltinCore from multiprocessing.connection import Listener, Client -class DispatchingCache(Cache, Bcfg2.Server.Plugin.Debuggable): - """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates - cache expiration events to child nodes. """ - - #: The method to send over the pipe to expire the cache - method = "expire_metadata_cache" - - def __init__(self, *args, **kwargs): - self.rpc_q = kwargs.pop("queue") - Bcfg2.Server.Plugin.Debuggable.__init__(self) - Cache.__init__(self, *args, **kwargs) - - def expire(self, key=None): - self.rpc_q.publish(self.method, args=[key]) - Cache.expire(self, key=key) - - class RPCQueue(Bcfg2.Server.Plugin.Debuggable): """ An implementation of a :class:`multiprocessing.Queue` designed for several additional use patterns: @@ -304,16 +287,9 @@ class ChildCore(Core): return rmi @exposed - def expire_metadata_cache(self, client=None): - """ Expire the metadata cache for a client """ - self.metadata_cache.expire(client) - - @exposed - def RecvProbeData(self, address, _): - """ Expire the probe cache for a client """ - self.expire_caches_by_type(Bcfg2.Server.Plugin.Probing, - key=self.resolve_client(address, - metadata=False)[0]) + def expire_cache(self, *tags, **kwargs): + """ Expire cached data """ + Bcfg2.Server.Cache.expire(*tags, exact=kwargs.pop("exact", False)) @exposed def GetConfig(self, client): @@ -368,8 +344,6 @@ class MultiprocessingCore(BuiltinCore): #: used to send or publish commands to children. self.rpc_q = RPCQueue() - self.metadata_cache = DispatchingCache(queue=self.rpc_q) - #: A list of children that will be cycled through self._all_children = [] @@ -392,6 +366,7 @@ class MultiprocessingCore(BuiltinCore): self.logger.debug("Started %s children: %s" % (len(self._all_children), self._all_children)) self.children = cycle(self._all_children) + Bcfg2.Server.Cache.add_expire_hook(self.cache_dispatch) return BuiltinCore._run(self) def shutdown(self): @@ -464,16 +439,11 @@ class MultiprocessingCore(BuiltinCore): def set_debug(self, address, debug): self.rpc_q.set_debug(debug) self.rpc_q.publish("set_debug", args=[address, debug]) - self.metadata_cache.set_debug(debug) return BuiltinCore.set_debug(self, address, debug) - @exposed - def RecvProbeData(self, address, probedata): - rv = BuiltinCore.RecvProbeData(self, address, probedata) - # we don't want the children to actually process probe data, - # so we don't send the data, just the fact that we got some. - self.rpc_q.publish("RecvProbeData", args=[address, None]) - return rv + def cache_dispatch(self, tags, exact, _): + """ Publish cache expiration events to child nodes. """ + self.rpc_q.publish("expire_cache", args=tags, kwargs=dict(exact=exact)) @exposed def GetConfig(self, address): diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 2d157eba9..7a3d887fe 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -597,17 +597,14 @@ class XMLFileBacked(FileBacked): Index.__doc__ = FileBacked.Index.__doc__ def add_monitor(self, fpath): - """ Add a FAM monitor to a file that has been XIncluded. This - is only done if the constructor got both a ``fam`` object and - ``should_monitor`` set to True. + """ Add a FAM monitor to a file that has been XIncluded. :param fpath: The full path to the file to monitor :type fpath: string :returns: None """ self.extra_monitors.append(fpath) - if self.should_monitor: - self.fam.AddMonitor(fpath, self) + self.fam.AddMonitor(fpath, self) def __iter__(self): return iter(self.entries) @@ -631,6 +628,9 @@ class StructFile(XMLFileBacked): #: the file being cached __identifier__ = None + #: Whether or not to enable encryption + encryption = True + #: Callbacks used to determine if children of items with the given #: tags should be included in the return value of #: :func:`Bcfg2.Server.Plugin.helpers.StructFile.Match` and @@ -674,7 +674,7 @@ class StructFile(XMLFileBacked): self.logger.error('Genshi parse error in %s: %s' % (self.name, err)) - if HAS_CRYPTO: + if HAS_CRYPTO and self.encryption: lax_decrypt = self.xdata.get( "lax_decryption", str(Bcfg2.Options.setup.lax_decryption)).lower() == "true" @@ -925,6 +925,7 @@ class PriorityStructFile(StructFile): __init__.__doc__ = StructFile.__init__.__doc__ def Index(self): + StructFile.Index(self) try: self.priority = int(self.xdata.get('priority')) except (ValueError, TypeError): @@ -955,13 +956,13 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): def HandleEvent(self, event): XMLDirectoryBacked.HandleEvent(self, event) self.Entries = {} - for src in list(self.entries.values()): - for itype, children in list(src.items.items()): - for child in children: - try: - self.Entries[itype][child] = self.BindEntry - except KeyError: - self.Entries[itype] = {child: self.BindEntry} + for src in self.entries.values(): + for child in src.xdata.iterchildren(): + if child.tag in ['Group', 'Client']: + continue + if child.tag not in self.Entries: + self.Entries[child.tag] = dict() + self.Entries[child.tag][child.get("name")] = self.BindEntry HandleEvent.__doc__ = XMLDirectoryBacked.HandleEvent.__doc__ def _matches(self, entry, metadata, candidate): # pylint: disable=W0613 diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 30275f6ad..522c6a220 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -221,10 +221,32 @@ class Connector(object): def get_additional_groups(self, metadata): # pylint: disable=W0613 """ Return a list of additional groups for the given client. + Each group can be either the name of a group (a string), or a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that defines other data besides just the name. Note that you + cannot return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that clobbers a group defined by another plugin; the original + group will be used instead. For instance, assume the + following in ``Metadata/groups.xml``: + + .. code-block:: xml + + <Groups> + ... + <Group name="foo" public="false"/> + </Groups> + + You could not subsequently return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + with ``public=True``; a warning would be issued, and the + original (non-public) ``foo`` group would be used. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of strings + :return: list of strings or + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` + objects. """ return list() @@ -632,22 +654,3 @@ class ClientACLs(object): :returns: bool """ return True - - -class Caching(object): - """ A plugin that caches more than just the data received from the - FAM. This presents a unified interface to clear the cache. """ - - def expire_cache(self, key=None): - """ Expire the cache associated with the given key. - - :param key: The key to expire the cache for. Because cache - implementations vary tremendously between plugins, - this could be any number of things, but generally - a hostname. It also may or may not be possible to - expire the cache for a single host; this interface - does not require any guarantee about that. - :type key: varies - :returns: None - """ - raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index f91bac634..b3824fb57 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -52,15 +52,12 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data) #: Bundles by bundle name, rather than filename self.bundles = dict() - __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def HandleEvent(self, event): Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) self.bundles = dict([(b.bundle_name, b) for b in self.entries.values()]) - HandleEvent.__doc__ = \ - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent.__doc__ def BuildStructures(self, metadata): bundleset = [] @@ -121,5 +118,3 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, data.remove(child) bundleset.append(data) return bundleset - BuildStructures.__doc__ = \ - Bcfg2.Server.Plugin.Structure.BuildStructures.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index c08d3ec44..895752c9c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -5,7 +5,7 @@ access. """ import lxml.etree import Bcfg2.Options from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgGenerator, CFG +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, get_cfg from Bcfg2.Server.Plugins.Metadata import ClientMetadata @@ -25,7 +25,7 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): CfgGenerator.__init__(self, fname, None) StructFile.__init__(self, fname) self.cache = dict() - self.core = CFG.core + self.core = get_cfg().core __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): @@ -38,10 +38,13 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): spec = self.XMLMatch(metadata) rv = [] for allow in spec.findall("Allow"): - params = '' - if allow.find("Params") is not None: - params = ",".join("=".join(p) - for p in allow.find("Params").attrib.items()) + options = [] + for opt in allow.findall("Option"): + if opt.get("value"): + options.append("%s=%s" % (opt.get("name"), + opt.get("value"))) + else: + options.append(opt.get("name")) pubkey_name = allow.get("from") if pubkey_name: @@ -85,6 +88,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): (metadata.hostname, lxml.etree.tostring(allow))) continue - rv.append(" ".join([params, pubkey]).strip()) + rv.append(" ".join([",".join(options), pubkey]).strip()) return "\n".join(rv) get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 7bb5d3cf5..e5611d50b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -5,17 +5,11 @@ import shutil import tempfile import Bcfg2.Options from Bcfg2.Utils import Executor -from Bcfg2.Server.Plugin import StructFile -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError +from Bcfg2.Server.Plugins.Cfg import XMLCfgCreator, CfgCreationError from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator -try: - import Bcfg2.Server.Encryption - HAS_CRYPTO = True -except ImportError: - HAS_CRYPTO = False -class CfgPrivateKeyCreator(CfgCreator, StructFile): +class CfgPrivateKeyCreator(XMLCfgCreator): """The CfgPrivateKeyCreator creates SSH keys on the fly. """ #: Different configurations for different clients/groups can be @@ -25,6 +19,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): #: Handle XML specifications of private keys __basenames__ = ['privkey.xml'] + cfg_section = "sshkeys" options = [ Bcfg2.Options.Option( cf=("sshkeys", "category"), dest="sshkeys_category", @@ -34,27 +29,12 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): help="Passphrase used to encrypt generated SSH private keys")] def __init__(self, fname): - CfgCreator.__init__(self, fname) - StructFile.__init__(self, fname) - + XMLCfgCreator.__init__(self, fname) pubkey_path = os.path.dirname(self.name) + ".pub" pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path)) self.pubkey_creator = CfgPublicKeyCreator(pubkey_name) self.cmd = Executor() - __init__.__doc__ = CfgCreator.__init__.__doc__ - - @property - def passphrase(self): - """ The passphrase used to encrypt private keys """ - if HAS_CRYPTO and Bcfg2.Options.setup.sshkeys_passphrase: - return Bcfg2.Options.setup.passphrases[ - Bcfg2.Options.setup.sshkeys_passphrase] - return None - - def handle_event(self, event): - CfgCreator.handle_event(self, event) - StructFile.HandleEvent(self, event) - handle_event.__doc__ = CfgCreator.handle_event.__doc__ + __init__.__doc__ = XMLCfgCreator.__init__.__doc__ def _gen_keypair(self, metadata, spec=None): """ Generate a keypair according to the given client medata @@ -117,45 +97,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): shutil.rmtree(tempdir) raise - def get_specificity(self, metadata, spec=None): - """ Get config settings for key generation specificity - (per-host or per-group). - - :param metadata: The client metadata to create data for - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param spec: The key specification to follow when creating the - keys. This should be an XML document that only - contains key specification data that applies to - the given client metadata, and may be obtained by - doing ``self.XMLMatch(metadata)`` - :type spec: lxml.etree._Element - :returns: dict - A dict of specificity arguments suitable for - passing to - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` - or - :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` - """ - if spec is None: - spec = self.XMLMatch(metadata) - category = spec.get("category", Bcfg2.Options.setup.sshkeys_category) - if category is None: - per_host_default = "true" - else: - per_host_default = "false" - per_host = spec.get("perhost", per_host_default).lower() == "true" - - specificity = dict(host=metadata.hostname) - if category and not per_host: - group = metadata.group_in_category(category) - if group: - specificity = dict(group=group, - prio=int(spec.get("priority", 50))) - else: - self.logger.info("Cfg: %s has no group in category %s, " - "creating host-specific key" % - (metadata.hostname, category)) - return specificity - # pylint: disable=W0221 def create_data(self, entry, metadata, return_pair=False): """ Create data for the given entry on the given client @@ -176,7 +117,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): ``return_pair`` is set to True """ spec = self.XMLMatch(metadata) - specificity = self.get_specificity(metadata, spec) + specificity = self.get_specificity(metadata) filename = self._gen_keypair(metadata, spec) try: @@ -190,12 +131,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): # encrypt the private key, write to the proper place, and # return it privkey = open(filename).read() - if HAS_CRYPTO and self.passphrase: - self.debug_log("Cfg: Encrypting key data at %s" % filename) - privkey = Bcfg2.Server.Encryption.ssl_encrypt(privkey, - self.passphrase) - specificity['ext'] = '.crypt' - self.write_data(privkey, **specificity) if return_pair: diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py index 4c61e338e..de1848159 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py @@ -4,7 +4,7 @@ to create SSH keys on the fly. """ import lxml.etree from Bcfg2.Server.Plugin import StructFile, PluginExecutionError -from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, CFG +from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, get_cfg class CfgPublicKeyCreator(CfgCreator, StructFile): @@ -17,7 +17,7 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): creation of a keypair when a public key is created. """ #: Different configurations for different clients/groups can be - #: handled with Client and Group tags within privkey.xml + #: handled with Client and Group tags within pubkey.xml __specific__ = False #: Handle XML specifications of private keys @@ -29,7 +29,7 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): def __init__(self, fname): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) - self.cfg = CFG + self.cfg = get_cfg() __init__.__doc__ = CfgCreator.__init__.__doc__ def create_data(self, entry, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py new file mode 100644 index 000000000..92fcc4cd8 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py @@ -0,0 +1,255 @@ +""" Cfg creator that creates SSL certs """ + +import os +import sys +import tempfile +import lxml.etree +import Bcfg2.Options +from Bcfg2.Utils import Executor +from Bcfg2.Compat import ConfigParser +from Bcfg2.Server.FileMonitor import get_fam +from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator, \ + CfgCreator, CfgVerifier, CfgVerificationError, get_cfg + + +class CfgSSLCACertCreator(XMLCfgCreator, CfgVerifier): + """ This class acts as both a Cfg creator that creates SSL certs, + and as a Cfg verifier that verifies SSL certs. """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within pubkey.xml + __specific__ = False + + #: Handle XML specifications of private keys + __basenames__ = ['sslcert.xml'] + + cfg_section = "sslca" + options = [ + Bcfg2.Options.Option( + cf=("sslca", "category"), dest="sslca_category", + help="Metadata category that generated SSL keys are specific to"), + Bcfg2.Options.Option( + cf=("sslca", "passphrase"), dest="sslca_passphrase", + help="Passphrase used to encrypt generated SSL keys"), + Bcfg2.Options.WildcardSectionGroup( + Bcfg2.Options.PathOption( + cf=("sslca_*", "config"), + help="Path to the openssl config for the CA"), + Bcfg2.Options.Option( + cf=("sslca_*", "passphrase"), + help="Passphrase for the CA private key"), + Bcfg2.Options.PathOption( + cf=("sslca_*", "chaincert"), + help="Path to the SSL chaining certificate for verification"), + Bcfg2.Options.BooleanOption( + cf=("sslca_*", "root_ca"), + help="Whether or not <chaincert> is a root CA (as opposed to " + "an intermediate cert"), + prefix="")] + + def __init__(self, fname): + XMLCfgCreator.__init__(self, fname) + CfgVerifier.__init__(self, fname, None) + self.cmd = Executor() + self.cfg = get_cfg() + + def build_req_config(self, metadata): + """ Generates a temporary openssl configuration file that is + used to generate the required certificate request. """ + fd, fname = tempfile.mkstemp() + cfp = ConfigParser.ConfigParser({}) + cfp.optionxform = str + defaults = dict( + req=dict( + default_md='sha1', + distinguished_name='req_distinguished_name', + req_extensions='v3_req', + x509_extensions='v3_req', + prompt='no'), + req_distinguished_name=dict(), + v3_req=dict(subjectAltName='@alt_names'), + alt_names=dict()) + for section in list(defaults.keys()): + cfp.add_section(section) + for key in defaults[section]: + cfp.set(section, key, defaults[section][key]) + spec = self.XMLMatch(metadata) + cert = spec.find("Cert") + altnamenum = 1 + altnames = spec.findall('subjectAltName') + altnames.extend(list(metadata.aliases)) + altnames.append(metadata.hostname) + for altname in altnames: + cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) + altnamenum += 1 + for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: + if cert.get(item): + cfp.set('req_distinguished_name', item, cert.get(item)) + cfp.set('req_distinguished_name', 'CN', metadata.hostname) + self.debug_log("Cfg: Writing temporary CSR config to %s" % fname) + try: + cfp.write(os.fdopen(fd, 'w')) + except IOError: + raise CfgCreationError("Cfg: Failed to write temporary CSR config " + "file: %s" % sys.exc_info()[1]) + return fname + + def build_request(self, keyfile, metadata): + """ Create the certificate request """ + req_config = self.build_req_config(metadata) + try: + fd, req = tempfile.mkstemp() + os.close(fd) + cert = self.XMLMatch(metadata).find("Cert") + days = cert.get("days", "365") + cmd = ["openssl", "req", "-new", "-config", req_config, + "-days", days, "-key", keyfile, "-text", "-out", req] + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate CSR: %s" % + result.error) + return req + finally: + try: + os.unlink(req_config) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR " + "config: %s" % sys.exc_info()[1]) + + def get_ca(self, name): + """ get a dict describing a CA from the config file """ + rv = dict() + prefix = "sslca_%s_" % name + for attr in dir(Bcfg2.Options.setup): + if attr.startswith(prefix): + rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) + return rv + + def create_data(self, entry, metadata): + """ generate a new cert """ + self.logger.info("Cfg: Generating new SSL cert for %s" % self.name) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + req = self.build_request(self._get_keyfile(cert, metadata), metadata) + try: + days = cert.get('days', '365') + cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, + "-days", days, "-batch"] + passphrase = ca.get('passphrase') + if passphrase: + cmd.extend(["-passin", "pass:%s" % passphrase]) + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate cert: %s" % + result.error) + except KeyError: + raise CfgCreationError("Cfg: [sslca_%s] section has no 'config' " + "option" % cert.get('ca', 'default')) + finally: + try: + os.unlink(req) + except OSError: + self.logger.error("Cfg: Failed to unlink temporary CSR: %s " % + sys.exc_info()[1]) + data = result.stdout + if cert.get('append_chain') and 'chaincert' in ca: + data += open(ca['chaincert']).read() + + self.write_data(data, **self.get_specificity(metadata)) + return data + + def verify_entry(self, entry, metadata, data): + fd, fname = tempfile.mkstemp() + self.debug_log("Cfg: Writing SSL cert %s to temporary file %s for " + "verification" % (entry.get("name"), fname)) + os.fdopen(fd, 'w').write(data) + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get('ca', 'default')) + try: + if ca.get('chaincert'): + self.verify_cert_against_ca(fname, entry, metadata) + self.verify_cert_against_key(fname, + self._get_keyfile(cert, metadata)) + finally: + os.unlink(fname) + + def _get_keyfile(self, cert, metadata): + """ Given a <Cert/> element and client metadata, return the + full path to the file on the filesystem that the key lives in.""" + keypath = cert.get("key") + eset = self.cfg.entries[keypath] + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + # SSL key needs to be created + try: + creator = eset.best_matching(metadata, + eset.get_handlers(metadata, + CfgCreator)) + except PluginExecutionError: + raise CfgCreationError("Cfg: No SSL key or key creator " + "defined for %s" % keypath) + + keyentry = lxml.etree.Element("Path", name=keypath) + creator.create_data(keyentry, metadata) + + tries = 0 + while True: + if tries >= 10: + raise CfgCreationError("Cfg: Timed out waiting for event " + "on SSL key at %s" % keypath) + get_fam().handle_events_in_interval(1) + try: + return eset.best_matching(metadata).name + except PluginExecutionError: + tries += 1 + continue + + def verify_cert_against_ca(self, filename, entry, metadata): + """ + check that a certificate validates against the ca cert, + and that it has not expired. + """ + cert = self.XMLMatch(metadata).find("Cert") + ca = self.get_ca(cert.get("ca", "default")) + chaincert = ca.get('chaincert') + cmd = ["openssl", "verify"] + is_root = ca.get('root_ca', "false").lower() == 'true' + if is_root: + cmd.append("-CAfile") + else: + # verifying based on an intermediate cert + cmd.extend(["-purpose", "sslserver", "-untrusted"]) + cmd.extend([chaincert, filename]) + self.debug_log("Cfg: Verifying %s against CA" % entry.get("name")) + result = self.cmd.run(cmd) + if result.stdout == cert + ": OK\n": + self.debug_log("Cfg: %s verified successfully against CA" % + entry.get("name")) + else: + raise CfgVerificationError("%s failed verification against CA: %s" + % (entry.get("name"), result.error)) + + def _get_modulus(self, fname, ftype="x509"): + """ get the modulus from the given file """ + cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] + self.debug_log("Cfg: Getting modulus of %s for verification: %s" % + (fname, " ".join(cmd))) + result = self.cmd.run(cmd) + if not result.success: + raise CfgVerificationError("Failed to get modulus of %s: %s" % + (fname, result.error)) + return result.stdout.strip() + + def verify_cert_against_key(self, filename, keyfile): + """ check that a certificate validates against its private + key. """ + cert = self._get_modulus(filename) + key = self._get_modulus(keyfile, ftype="rsa") + if cert == key: + self.debug_log("Cfg: %s verified successfully against key %s" % + (filename, keyfile)) + else: + raise CfgVerificationError("%s failed verification against key %s" + % (filename, keyfile)) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py new file mode 100644 index 000000000..a158302be --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCAKeyCreator.py @@ -0,0 +1,36 @@ +""" Cfg creator that creates SSL keys """ + +from Bcfg2.Utils import Executor +from Bcfg2.Server.Plugins.Cfg import CfgCreationError, XMLCfgCreator + + +class CfgSSLCAKeyCreator(XMLCfgCreator): + """ Cfg creator that creates SSL keys """ + + #: Different configurations for different clients/groups can be + #: handled with Client and Group tags within sslkey.xml + __specific__ = False + + __basenames__ = ["sslkey.xml"] + + cfg_section = "sslca" + + def create_data(self, entry, metadata): + self.logger.info("Cfg: Generating new SSL key for %s" % self.name) + spec = self.XMLMatch(metadata) + key = spec.find("Key") + if not key: + key = dict() + ktype = key.get('type', 'rsa') + bits = key.get('bits', '2048') + if ktype == 'rsa': + cmd = ["openssl", "genrsa", bits] + elif ktype == 'dsa': + cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] + result = Executor().run(cmd) + if not result.success: + raise CfgCreationError("Failed to generate key %s for %s: %s" % + (self.name, metadata.hostname, + result.error)) + self.write_data(result.stdout, **self.get_specificity(metadata)) + return result.stdout diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 99afac7eb..eea0a3456 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -10,16 +10,29 @@ import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, any, oct_mode +from Bcfg2.Compat import u_str, unicode, b64encode, any, walk_packages # pylint: enable=W0622 -#: CFG is a reference to the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` -#: plugin object created by the Bcfg2 core. This is provided so that -#: the handler objects can access it as necessary, since the existing -#: :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and -#: :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no -#: facility for passing it otherwise. -CFG = None +try: + import Bcfg2.Server.Encryption + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +_handlers = [m[1] # pylint: disable=C0103 + for m in walk_packages(path=__path__)] + +_CFG = None + + +def get_cfg(): + """ Get the :class:`Bcfg2.Server.Plugins.Cfg.Cfg` plugin object + created by the Bcfg2 core. This is provided so that the handler + objects can access it as necessary, since the existing + :class:`Bcfg2.Server.Plugin.helpers.GroupSpool` and + :class:`Bcfg2.Server.Plugin.helpers.EntrySet` classes have no + facility for passing it otherwise.""" + return _CFG class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): @@ -288,7 +301,7 @@ class CfgCreator(CfgBaseFileMatcher): :type name: string .. ----- - .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgCreator.__specific__ + .. autoattribute:: Bcfg2.Server.Plugins.Cfg.CfgInfo.__specific__ """ CfgBaseFileMatcher.__init__(self, fname, None) @@ -310,7 +323,9 @@ class CfgCreator(CfgBaseFileMatcher): ``host`` is given, it will be host-specific. It will be group-specific if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, the filename will be - non-specific. + non-specific. In general, this will be called as:: + + self.get_filename(**self.get_specificity(metadata)) :param host: The file applies to the given host :type host: bool @@ -341,6 +356,9 @@ class CfgCreator(CfgBaseFileMatcher): written as a host-specific file, or as a group-specific file if ``group`` and ``prio`` are given. If neither ``host`` nor ``group`` is given, it will be written as a non-specific file. + In general, this will be called as:: + + self.write_data(data, **self.get_specificity(metadata)) :param data: The data to write :type data: string @@ -360,7 +378,7 @@ class CfgCreator(CfgBaseFileMatcher): :raises: :exc:`Bcfg2.Server.Plugins.Cfg.CfgCreationError` """ fileloc = self.get_filename(host=host, group=group, prio=prio, ext=ext) - self.debug_log("%s: Writing new file %s" % (self.name, fileloc)) + self.debug_log("Cfg: Writing new file %s" % fileloc) try: os.makedirs(os.path.dirname(fileloc)) except OSError: @@ -376,6 +394,95 @@ class CfgCreator(CfgBaseFileMatcher): raise CfgCreationError("Could not write %s: %s" % (fileloc, err)) +class XMLCfgCreator(CfgCreator, # pylint: disable=W0223 + Bcfg2.Server.Plugin.StructFile): + """ A CfgCreator that uses XML to describe how data should be + generated. """ + + #: Whether or not the created data from this class can be + #: encrypted + encryptable = True + + #: Encryption and creation settings can be stored in bcfg2.conf, + #: either under the [cfg] section, or under the named section. + cfg_section = None + + def __init__(self, name): + CfgCreator.__init__(self, name) + Bcfg2.Server.Plugin.StructFile.__init__(self, name) + + def handle_event(self, event): + CfgCreator.handle_event(self, event) + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event) + + @property + def passphrase(self): + """ The passphrase used to encrypt created data """ + if self.cfg_section: + localopt = "%s_passphrase" % self.cfg_section + passphrase = getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_passphrase) + else: + passphrase = Bcfg2.Options.setup.cfg_passphrase + if passphrase is None: + return None + try: + return Bcfg2.Options.setup.passphrases[passphrase] + except KeyError: + raise CfgCreationError("%s: No such passphrase: %s" % + (self.__class__.__name__, passphrase)) + + @property + def category(self): + """ The category to which created data is specific """ + if self.cfg_section: + localopt = "%s_category" % self.cfg_section + return getattr(Bcfg2.Options.setup, localopt, + Bcfg2.Options.setup.cfg_category) + else: + return Bcfg2.Options.setup.cfg_category + + def write_data(self, data, host=None, group=None, prio=0, ext=''): + if HAS_CRYPTO and self.encryptable and self.passphrase: + self.debug_log("Cfg: Encrypting created data") + data = Bcfg2.Server.Encryption.ssl_encrypt(data, self.passphrase) + ext = '.crypt' + CfgCreator.write_data(self, data, host=host, group=group, prio=prio, + ext=ext) + + def get_specificity(self, metadata): + """ Get config settings for key generation specificity + (per-host or per-group). + + :param metadata: The client metadata to create data for + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + :returns: dict - A dict of specificity arguments suitable for + passing to + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.write_data` + or + :func:`Bcfg2.Server.Plugins.Cfg.CfgCreator.get_filename` + """ + category = self.xdata.get("category", self.category) + if category is None: + per_host_default = "true" + else: + per_host_default = "false" + per_host = self.xdata.get("perhost", + per_host_default).lower() == "true" + + specificity = dict(host=metadata.hostname) + if category and not per_host: + group = metadata.group_in_category(category) + if group: + specificity = dict(group=group, + prio=int(self.xdata.get("priority", 50))) + else: + self.logger.info("Cfg: %s has no group in category %s, " + "creating host-specific data" % + (metadata.hostname, category)) + return specificity + + class CfgVerificationError(Exception): """ Raised by :func:`Bcfg2.Server.Plugins.Cfg.CfgVerifier.verify_entry` when an @@ -411,7 +518,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): def __init__(self, basename, path, entry_type): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type) self.specific = None - self._handlers = None __init__.__doc__ = Bcfg2.Server.Plugin.EntrySet.__doc__ def set_debug(self, debug): @@ -420,14 +526,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): entry.set_debug(debug) return rv - @property - def handlers(self): - """ A list of Cfg handler classes. """ - if self._handlers is None: - self._handlers = Bcfg2.Options.setup.cfg_handlers - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -444,7 +542,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): # process a bogus changed event like a created return - for hdlr in self.handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -783,32 +881,27 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, '--cfg-validation', cf=('cfg', 'validation'), default=True, help='Run validation on Cfg files'), Bcfg2.Options.Option( + cf=('cfg', 'category'), dest="cfg_category", + help='The default name of the metadata category that created data ' + 'is specific to'), + Bcfg2.Options.Option( + cf=('cfg', 'passphrase'), dest="cfg_passphrase", + help='The default passphrase name used to encrypt created data'), + Bcfg2.Options.Option( cf=("cfg", "handlers"), dest="cfg_handlers", help="Cfg handlers to load", type=Bcfg2.Options.Types.comma_list, action=CfgHandlerAction, - default=['CfgAuthorizedKeysGenerator', 'CfgEncryptedGenerator', - 'CfgCheetahGenerator', 'CfgEncryptedCheetahGenerator', - 'CfgGenshiGenerator', 'CfgEncryptedGenshiGenerator', - 'CfgExternalCommandVerifier', 'CfgInfoXML', - 'CfgPlaintextGenerator', - 'CfgPrivateKeyCreator', 'CfgPublicKeyCreator'])] + default=_handlers)] def __init__(self, core, datastore): - global CFG # pylint: disable=W0603 + global _CFG # pylint: disable=W0603 Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullTarget.__init__(self) - self._handlers = None - CFG = self + Bcfg2.Options.setup.cfg_handlers.sort( + key=operator.attrgetter("__priority__")) + _CFG = self __init__.__doc__ = Bcfg2.Server.Plugin.GroupSpool.__init__.__doc__ - @property - def handlers(self): - """ A list of Cfg handler classes. """ - if self._handlers is None: - self._handlers = Bcfg2.Options.setup.cfg_handlers - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def has_generator(self, entry, metadata): """ Return True if the given entry can be generated for the given metadata; False otherwise diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index aa336ff23..1da7c8fec 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -4,6 +4,7 @@ template to dynamically set additional groups for clients. """ import os import lxml.etree import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Metadata import MetadataGroup class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): @@ -11,10 +12,17 @@ class GroupLogicConfig(Bcfg2.Server.Plugin.StructFile): create = lxml.etree.Element("GroupLogic", nsmap=dict(py="http://genshi.edgewall.org/")) - def _match(self, item, metadata): + def _match(self, item, metadata, *args): if item.tag == 'Group' and not len(item.getchildren()): return [item] - return Bcfg2.Server.Plugin.StructFile._match(self, item, metadata) + return Bcfg2.Server.Plugin.StructFile._match(self, item, metadata, + *args) + + def _xml_match(self, item, metadata, *args): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return Bcfg2.Server.Plugin.StructFile._xml_match(self, item, metadata, + *args) class GroupLogic(Bcfg2.Server.Plugin.Plugin, @@ -30,5 +38,11 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, should_monitor=True) def get_additional_groups(self, metadata): - return [el.get("name") - for el in self.config.XMLMatch(metadata).findall("Group")] + rv = [] + for el in self.config.XMLMatch(metadata).findall("Group"): + if el.get("category"): + rv.append(MetadataGroup(el.get("name"), + category=el.get("category"))) + else: + rv.append(el.get("name")) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 12ece1f19..db104b27e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -16,10 +16,10 @@ import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked +from Bcfg2.Server.Cache import Cache from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 from Bcfg2.version import Bcfg2VersionInfo - # pylint: disable=C0103 ClientVersions = None MetadataClientModel = None @@ -89,7 +89,7 @@ def load_django_models(): def keys(self): """ Get keys for the mapping """ - return [c.hostname for c in MetadataClientModel.objects.all()] + return list(iter(self)) def __contains__(self, key): try: @@ -102,17 +102,12 @@ def load_django_models(): class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): """Handles xml config files and all XInclude statements""" - def __init__(self, metadata, watch_clients, basefile): - # we tell XMLFileBacked _not_ to add a monitor for this file, - # because the main Metadata plugin has already added one. - # then we immediately set should_monitor to the proper value, - # so that XInclude'd files get properly watched + def __init__(self, metadata, basefile): fpath = os.path.join(metadata.data, basefile) toptag = os.path.splitext(basefile)[0].title() Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, should_monitor=False, create=toptag) - self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile self.data = None @@ -257,8 +252,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): def add_monitor(self, fpath): self.extras.append(fpath) - if self.should_monitor: - self.fam.AddMonitor(fpath, self.metadata) + self.fam.AddMonitor(fpath, self.metadata) def HandleEvent(self, event=None): """Handle fam events""" @@ -500,7 +494,6 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" @@ -518,12 +511,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, help='Default client authentication method')] options_parsed_hook = staticmethod(load_django_models) - def __init__(self, core, datastore, watch_clients=True): + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) - self.watch_clients = watch_clients self.states = dict() self.extra = dict() self.handlers = dict() @@ -554,22 +545,26 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} - # mappings of predicate -> MetadataGroup object + # mappings of groupname -> [predicates] self.group_membership = dict() self.negated_groups = dict() + # list of group names in document order + self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions(core, # pylint: disable=E1102 datastore) else: self.versions = dict() + self.uuid = {} self.session_cache = {} + self.cache = Cache("Metadata") self.default = None self.pdirty = False self.password = Bcfg2.Options.setup.password self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients), + self.list_clients, self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -595,17 +590,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _handle_file(self, fname): """ set up the necessary magic for handling a metadata file (clients.xml or groups.xml, e.g.) """ - if self.watch_clients: - try: - Bcfg2.Server.FileMonitor.get_fam().AddMonitor( - os.path.join(self.data, fname), self) - except: - err = sys.exc_info()[1] - msg = "Unable to add file monitor for %s: %s" % (fname, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - self.states[fname] = False - xmlcfg = XMLMetadataConfig(self, self.watch_clients, fname) + try: + Bcfg2.Server.FileMonitor.get_fam().AddMonitor( + os.path.join(self.data, fname), self) + except: + err = sys.exc_info()[1] + msg = "Unable to add file monitor for %s: %s" % (fname, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + self.states[fname] = False + xmlcfg = XMLMetadataConfig(self, fname) aname = re.sub(r'[^A-z0-9_]', '_', os.path.basename(fname)) self.handlers[xmlcfg.HandleEvent] = getattr(self, "_handle_%s_event" % aname) @@ -860,51 +854,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() + def _get_condition(self, element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ + negate = element.get('negate', 'false').lower() == 'true' + pname = element.get("name") + if element.tag == 'Group': + return lambda c, g, _: negate != (pname in g) + elif element.tag == 'Client': + return lambda c, g, _: negate != (pname == c) + + def _get_category_condition(self, grpname): + """ get a predicate that returns False if a client is already + a member of a group in the given group's category, True + otherwise""" + return lambda client, _, categories: \ + bool(self._check_category(client, grpname, categories)) + + def _aggregate_conditions(self, conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ + return lambda client, groups, cats: \ + all(cond(client, groups, cats) for cond in conditions) + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ self.groups = {} - # these three functions must be separate functions in order to - # ensure that the scope is right for the closures they return - def get_condition(element): - """ Return a predicate that returns True if a client meets - the condition specified in the given Group or Client - element """ - negate = element.get('negate', 'false').lower() == 'true' - pname = element.get("name") - if element.tag == 'Group': - return lambda c, g, _: negate != (pname in g) - elif element.tag == 'Client': - return lambda c, g, _: negate != (pname == c) - - def get_category_condition(category, gname): - """ get a predicate that returns False if a client is - already a member of a group in the given category, True - otherwise """ - def in_cat(client, groups, categories): # pylint: disable=W0613 - """ return True if the client is already a member of a - group in the category given in the enclosing function, - False otherwise """ - if category in categories: - if (gname not in self.groups or - client not in self.groups[gname].warned): - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, gname, category, - client, categories[category])) - if gname in self.groups: - self.groups[gname].warned.append(client) - return False - return True - return in_cat - - def aggregate_conditions(conditions): - """ aggregate all conditions on a given group declaration - into a single predicate """ - return lambda client, groups, cats: \ - all(cond(client, groups, cats) for cond in conditions) - # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old @@ -930,6 +907,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() + self.ordered_groups = [] # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is @@ -940,40 +918,44 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): - if ((el.tag != 'Group' and el.tag != 'Client') or - el.getchildren()): + if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren(): continue conditions = [] for parent in el.iterancestors(): - cond = get_condition(parent) + cond = self._get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[aggregate_conditions(conditions)] = \ - self.groups[gname] + self.negated_groups.setdefault(gname, []) + self.negated_groups[gname].append( + self._aggregate_conditions(conditions)) else: if self.groups[gname].category: - conditions.append( - get_category_condition(self.groups[gname].category, - gname)) + conditions.append(self._get_category_condition(gname)) - self.group_membership[aggregate_conditions(conditions)] = \ - self.groups[gname] + if gname not in self.ordered_groups: + self.ordered_groups.append(gname) + self.group_membership.setdefault(gname, []) + self.group_membership[gname].append( + self._aggregate_conditions(conditions)) self.states['groups.xml'] = True - def expire_cache(self, key=None): - self.core.metadata_cache.expire(key) - def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.expire_cache() + self.cache.expire() + + # clear out the list of category suppressions that + # have been warned about, since this may change when + # clients.xml or groups.xml changes. + for group in self.groups.values(): + group.warned = [] event_handler(event) if False not in list(self.states.values()) and self.debug_flag: @@ -1112,30 +1094,85 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() while numgroups != len(groups): numgroups = len(groups) - for predicate, group in self.group_membership.items(): - if group.name in groups: + newgroups = set() + removegroups = set() + for grpname in self.ordered_groups: + if grpname in groups: continue - if predicate(client, groups, categories): - groups.add(group.name) - if group.category: - categories[group.category] = group.name - for predicate, group in self.negated_groups.items(): - if group.name not in groups: + if any(p(client, groups, categories) + for p in self.group_membership[grpname]): + newgroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + categories[self.groups[grpname].category] = grpname + groups.update(newgroups) + for grpname, predicates in self.negated_groups.items(): + if grpname not in groups: continue - if predicate(client, groups, categories): - groups.remove(group.name) - if group.category: - del categories[group.category] + if any(p(client, groups, categories) for p in predicates): + removegroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + del categories[self.groups[grpname].category] + groups.difference_update(removegroups) return (groups, categories) + def _check_category(self, client, grpname, categories): + """ Determine if the given client is already a member of a + group in the same category as the named group. + + The return value is one of three possibilities: + + * If the client is already a member of a group in the same + category, then False is returned (i.e., the category check + failed); + * If the group is not in any categories, then True is returned; + * If the group is not a member of a group in the category, + then the name of the category is returned. This makes it + easy to add the category to the ClientMetadata object (or + other category list). + + If a pure boolean value is required, you can do + ``bool(self._check_category(...))``. + """ + if grpname not in self.groups: + return True + category = self.groups[grpname].category + if not category: + return True + if category in categories: + if client not in self.groups[grpname].warned: + self.logger.warning("%s: Group %s suppressed by category %s; " + "%s already a member of %s" % + (self.name, grpname, category, + client, categories[category])) + self.groups[grpname].warned.append(client) + return False + return category + + def _check_and_add_category(self, client, grpname, categories): + """ If the client is not a member of a group in the same + category as the named group, then the category is added to + ``categories``. + :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used + to determine if the category can be added. + + If the category check failed, returns False; otherwise, + returns True. """ + rv = self._check_category(client, grpname, categories) + if rv and rv is not True: + categories[rv] = grpname + return True + return rv + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") client = client.lower() - if client in self.core.metadata_cache: - return self.core.metadata_cache[client] + if client in self.cache: + return self.cache[client] if client in self.aliases: client = self.aliases[client] @@ -1149,30 +1186,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Handles setting categories and category suppression. Returns the new profile for the client (which might be unchanged). """ - groups.add(grpname) if grpname in self.groups: - group = self.groups[grpname] - category = group.category - if category: - if category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, grpname, category, - client, categories[category])) - return - categories[category] = grpname - if not profile and group.is_profile: + if not self._check_and_add_category(client, grpname, + categories): + return profile + groups.add(grpname) + if not profile and self.groups[grpname].is_profile: return grpname else: return profile + else: + groups.add(grpname) + return profile if client not in self.clients: pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] + self.debug_log("%s: Adding new client with profile %s" % + (self.name, pgroup)) elif self.default: pgroup = self.default + self.debug_log("%s: Adding new client with default profile %s" + % (self.name, pgroup)) if pgroup: self.set_profile(client, pgroup, (None, None), @@ -1189,6 +1225,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[cgroup] = MetadataGroup(cgroup) profile = _add_group(cgroup) + # we do this before setting the default because there may be + # groups set in <Client> tags in groups.xml that we want to + # set groups, categories = self._merge_groups(client, groups, categories=categories) @@ -1230,15 +1269,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, addresses, categories, uuid, password, version, self.query) if self.core.metadata_cache_mode == 'initial': - self.core.metadata_cache[client] = rv + self.cache[client] = rv return rv def get_all_group_names(self): """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) - all_groups.update([g.name for g in self.group_membership.values()]) - all_groups.update([g.name for g in self.negated_groups.values()]) + all_groups.update(self.group_membership.keys()) + all_groups.update(self.negated_groups.keys()) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups @@ -1251,7 +1290,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.clients): + for client in self.list_clients(): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1259,34 +1298,33 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.groups.issuperset(groups): + rv.append(client) + return rv def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.bundles.issuperset(bundles): + rv.append(client) + return rv def merge_additional_groups(self, imd, groups): for group in groups: if group in imd.groups: continue - if group in self.groups and self.groups[group].category: - category = self.groups[group].category - if self.groups[group].category in imd.categories: - self.logger.warning("%s: Group %s suppressed by category " - "%s; %s already a member of %s" % - (self.name, group, category, - imd.hostname, - imd.categories[category])) - continue - imd.categories[category] = group + if not self._check_and_add_category(imd.hostname, group, + imd.categories): + continue imd.groups.add(group) - self._merge_groups(imd.hostname, imd.groups, - categories=imd.categories) - + self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) @@ -1451,7 +1489,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] - for client in list(self.clients): + for client in list(self.list_clients()): if not include_client(client): continue if client in self.clientgroups: diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index dcd495d77..a27664215 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -21,9 +21,9 @@ class NagiosGen(Plugin, Generator): self.config = \ StructFile(os.path.join(self.data, 'config.xml'), should_monitor=True, create=self.name) - self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + self.Entries = { + 'Path': {'/etc/nagiosgen.status': self.createhostconfig, + '/etc/nagios/conf.d/bcfg2.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 5af9c1591..56285705a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -8,6 +8,7 @@ import glob import shutil import lxml.etree import Bcfg2.Options +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin from Bcfg2.Compat import urlopen, HTTPError, URLError, MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ @@ -81,7 +82,6 @@ class OnDemandDict(MutableMapping): class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -136,12 +136,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ - [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] - def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -185,7 +181,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`, #: a unique key identifying the collection by its *config*, #: which could be shared among multiple clients. - self.collections = dict() + self.collections = Bcfg2.Server.Cache.Cache("Packages", "collections") #: clients is a cache mapping of hostname -> #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey` @@ -193,21 +189,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. - self.clients = dict() - - #: groupcache caches group lookups. It maps Collections (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of package groups, and thence to the packages - #: indicated by those groups. - self.groupcache = dict() - - #: pkgcache caches complete package sets. It maps Collections - #: (via - #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) - #: to sets of initial packages, and thence to the final - #: (complete) package selections resolved from the initial - #: packages - self.pkgcache = dict() + self.clients = Bcfg2.Server.Cache.Cache("Packages", "cache") + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ @@ -400,11 +383,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups.sort() # check for this set of groups in the group cache + gcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_groups", + collection.cachekey) gkey = hash(tuple(groups)) - if gkey not in self.groupcache[collection.cachekey]: - self.groupcache[collection.cachekey][gkey] = \ - collection.get_groups(groups) - for pkgs in self.groupcache[collection.cachekey][gkey].values(): + if gkey not in gcache: + gcache[gkey] = collection.get_groups(groups) + for pkgs in gcache[gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution @@ -412,10 +396,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # check for this set of packages in the package cache pkey = hash(tuple(base)) - if pkey not in self.pkgcache[collection.cachekey]: - self.pkgcache[collection.cachekey][pkey] = \ - collection.complete(base) - packages, unknown = self.pkgcache[collection.cachekey][pkey] + pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", + collection.cachekey) + if pkey not in pcache: + pcache[pkey] = collection.complete(base) + packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -441,7 +426,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True - def expire_cache(self, _=None): + def child_reload(self, _=None): + """ Reload the Packages configuration on a child process. """ self.Reload() def _load_config(self, force_update=False): @@ -472,10 +458,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, collection.setup_data(force_update) # clear Collection and package caches - self.clients = dict() - self.collections = dict() - self.groupcache = dict() - self.pkgcache = dict() + Bcfg2.Server.Cache.expire("Packages") for source in self.sources.entries: cachefiles.add(source.cachefile) @@ -551,11 +534,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - collection = Collection(metadata, [], self.cachepath, self.data) - ckey = collection.cachekey - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) - return collection + return Collection(metadata, [], self.cachepath, self.data) if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -592,8 +571,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection - self.groupcache.setdefault(ckey, dict()) - self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): @@ -642,8 +619,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata """ - if metadata.hostname in self.clients: - del self.clients[metadata.hostname] + self.clients.expire(metadata.hostname) def end_statistics(self, metadata): """ Hook to clear the cache for this client in :attr:`clients` diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index f75d88d8f..560546c70 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -8,8 +8,11 @@ import copy import operator import lxml.etree import Bcfg2.Server +import Bcfg2.Server.Cache import Bcfg2.Server.Plugin +from Bcfg2.Compat import unicode # pylint: disable=W0622 import Bcfg2.Server.FileMonitor +from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics HAS_DJANGO = False @@ -63,6 +66,215 @@ except ImportError: HAS_YAML = False +class ProbeStore(Debuggable): + """ Caching abstraction layer between persistent probe data + storage and the Probes plugin.""" + + def __init__(self, core, datastore): # pylint: disable=W0613 + Debuggable.__init__(self) + self._groupcache = Bcfg2.Server.Cache.Cache("Probes", "probegroups") + self._datacache = Bcfg2.Server.Cache.Cache("Probes", "probedata") + + def get_groups(self, hostname): + """ Get the list of groups for the given host """ + if hostname not in self._groupcache: + self._load_groups(hostname) + return self._groupcache.get(hostname, []) + + def set_groups(self, hostname, groups): + """ Set the list of groups for the given host """ + raise NotImplementedError + + def get_data(self, hostname): + """ Get a dict of probe data for the given host """ + if hostname not in self._datacache: + self._load_data(hostname) + return self._datacache.get(hostname, dict()) + + def set_data(self, hostname, data): + """ Set probe data for the given host """ + raise NotImplementedError + + def _load_groups(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def _load_data(self, hostname): + """ When probe groups are not found in the cache, this + function is called to load them from the backend (XML or + database). """ + raise NotImplementedError + + def commit(self): + """ Commit the current data in the cache to the persistent + backend store. This is not used with the + :class:`Bcfg2.Server.Plugins.Probes.DBProbeStore`, because it + commits on every change. """ + pass + + +class DBProbeStore(ProbeStore, Bcfg2.Server.Plugin.DatabaseBacked): + """ Caching abstraction layer between the database and the Probes + plugin. """ + create = False + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) + ProbeStore.__init__(self, core, datastore) + + def _load_groups(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + groupdata = ProbesGroupsModel.objects.filter(hostname=hostname) + self._groupcache[hostname] = list(set(r.group for r in groupdata)) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + for group in groups: + try: + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + except ProbesGroupsModel.MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=hostname, + group=group) + ProbesGroupsModel.objects.filter( + hostname=hostname).exclude(group__in=groups).delete() + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def _load_data(self, hostname): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + ts_set = False + for pdata in ProbesDataModel.objects.filter(hostname=hostname): + if not ts_set: + self._datacache[hostname].timestamp = \ + time.mktime(pdata.timestamp.timetuple()) + ts_set = True + self._datacache[hostname][pdata.probe] = ProbeData(pdata.data) + Bcfg2.Server.Cache.expire("Metadata", hostname) + + @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + self._datacache[hostname][probe] = pdata + record, created = ProbesDataModel.objects.get_or_create( + hostname=hostname, + probe=probe) + expire_metadata |= created + if record.data != pdata: + record.data = pdata + record.save() + expire_metadata = True + qset = ProbesDataModel.objects.filter( + hostname=hostname).exclude(probe__in=data.keys()) + if len(qset): + qset.delete() + expire_metadata = True + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + +class XMLProbeStore(ProbeStore): + """ Caching abstraction layer between ``probed.xml`` and the + Probes plugin.""" + def __init__(self, core, datastore): + ProbeStore.__init__(self, core, datastore) + self._fname = os.path.join(datastore, 'probed.xml') + self._load_data() + + def _load_data(self, _=None): + """ Load probe data from probed.xml """ + Bcfg2.Server.Cache.expire("Probes", "probegroups") + Bcfg2.Server.Cache.expire("Probes", "probedata") + if not os.path.exists(self._fname): + self.commit() + try: + data = lxml.etree.parse(self._fname, + parser=Bcfg2.Server.XMLParser).getroot() + except (IOError, lxml.etree.XMLSyntaxError): + err = sys.exc_info()[1] + self.logger.error("Failed to read file probed.xml: %s" % err) + return + for client in data.getchildren(): + self._datacache[client.get('name')] = \ + ClientProbeDataSet(timestamp=client.get("timestamp")) + self._groupcache[client.get('name')] = [] + for pdata in client: + if pdata.tag == 'Probe': + self._datacache[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get("value")) + elif pdata.tag == 'Group': + self._groupcache[client.get('name')].append( + pdata.get('name')) + + Bcfg2.Server.Cache.expire("Metadata") + + def _load_groups(self, hostname): + self._load_data(hostname) + + def commit(self): + """ Write received probe data to probed.xml """ + top = lxml.etree.Element("Probed") + for client, probed in sorted(self._datacache.items()): + # make a copy of probe data for this client in case it + # submits probe data while we're trying to write + # probed.xml + probedata = copy.copy(probed) + ctag = \ + lxml.etree.SubElement(top, 'Client', name=client, + timestamp=str(int(probedata.timestamp))) + for probe in sorted(probedata): + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe].decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=self._datacache[client][probe]) + for group in sorted(self._groupcache[client]): + lxml.etree.SubElement(ctag, "Group", name=group) + try: + top.getroottree().write(self._fname, + xml_declaration=False, + pretty_print='true') + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to write probed.xml: %s" % err) + + def set_groups(self, hostname, groups): + Bcfg2.Server.Cache.expire("Probes", "probegroups", hostname) + olddata = self._groupcache.get(hostname, []) + self._groupcache[hostname] = groups + if olddata != groups: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + def set_data(self, hostname, data): + Bcfg2.Server.Cache.expire("Probes", "probedata", hostname) + self._datacache[hostname] = ClientProbeDataSet() + expire_metadata = False + for probe, pdata in data.items(): + olddata = self._datacache[hostname].get(probe, ProbeData('')) + self._datacache[hostname][probe] = pdata + expire_metadata |= olddata != data + if expire_metadata: + Bcfg2.Server.Cache.expire("Metadata", hostname) + + class ClientProbeDataSet(dict): """ dict of probe => [probe data] that records a timestamp for each host """ @@ -79,7 +291,10 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + if isinstance(data, unicode): + return str.__new__(cls, data.encode('utf-8')) + else: + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -195,12 +410,13 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' + groupline_re = re.compile(r'^group:\s*(?P<groupname>\S+)\s*') + options = [ Bcfg2.Options.BooleanOption( cf=('probes', 'use_database'), dest="probes_db", @@ -209,7 +425,6 @@ class Probes(Bcfg2.Server.Plugin.Probing, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Probing.__init__(self) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) @@ -219,191 +434,48 @@ class Probes(Bcfg2.Server.Plugin.Probing, err = sys.exc_info()[1] raise Bcfg2.Server.Plugin.PluginInitError(err) - self.probedata = dict() - self.cgroups = dict() - self.load_data() - __init__.__doc__ = Bcfg2.Server.Plugin.DatabaseBacked.__init__.__doc__ - - @track_statistics() - def write_data(self, client): - """ Write probe data out for use with bcfg2-info """ - if self._use_db: - return self._write_data_db(client) - else: - return self._write_data_xml(client) - - def _write_data_xml(self, _): - """ Write received probe data to probed.xml """ - top = lxml.etree.Element("Probed") - for client, probed in sorted(self.probedata.items()): - # make a copy of probe data for this client in case it - # submits probe data while we're trying to write - # probed.xml - probedata = copy.copy(probed) - ctag = \ - lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probedata.timestamp))) - for probe in sorted(probedata): - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=self.probedata[client][probe]) - for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(ctag, "Group", name=group) - try: - top.getroottree().write(os.path.join(self.data, 'probed.xml'), - xml_declaration=False, - pretty_print='true') - except IOError: - err = sys.exc_info()[1] - self.logger.error("Failed to write probed.xml: %s" % err) - - @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock - def _write_data_db(self, client): - """ Write received probe data to the database """ - for probe, data in self.probedata[client.hostname].items(): - pdata = \ - ProbesDataModel.objects.get_or_create(hostname=client.hostname, - probe=probe)[0] - if pdata.data != data: - pdata.data = data - pdata.save() - - ProbesDataModel.objects.filter( - hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() - - for group in self.cgroups[client.hostname]: - try: - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - except ProbesGroupsModel.MultipleObjectsReturned: - ProbesGroupsModel.objects.filter(hostname=client.hostname, - group=group).delete() - ProbesGroupsModel.objects.get_or_create( - hostname=client.hostname, - group=group) - ProbesGroupsModel.objects.filter( - hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() - - def expire_cache(self, key=None): - self.load_data(client=key) - - def load_data(self, client=None): - """ Load probe data from the appropriate backend (probed.xml - or the database) """ if self._use_db: - return self._load_data_db(client=client) - else: - # the XML backend doesn't support loading data for single - # clients, so it reloads all data - return self._load_data_xml() - - def _load_data_xml(self): - """ Load probe data from probed.xml """ - try: - data = lxml.etree.parse(os.path.join(self.data, 'probed.xml'), - parser=Bcfg2.Server.XMLParser).getroot() - except (IOError, lxml.etree.XMLSyntaxError): - err = sys.exc_info()[1] - self.logger.error("Failed to read file probed.xml: %s" % err) - return - self.probedata = {} - self.cgroups = {} - for client in data.getchildren(): - self.probedata[client.get('name')] = \ - ClientProbeDataSet(timestamp=client.get("timestamp")) - self.cgroups[client.get('name')] = [] - for pdata in client: - if pdata.tag == 'Probe': - self.probedata[client.get('name')][pdata.get('name')] = \ - ProbeData(pdata.get("value")) - elif pdata.tag == 'Group': - self.cgroups[client.get('name')].append(pdata.get('name')) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) - - def _load_data_db(self, client=None): - """ Load probe data from the database """ - if client is None: - self.probedata = {} - self.cgroups = {} - probedata = ProbesDataModel.objects.all() - groupdata = ProbesGroupsModel.objects.all() + self.probestore = DBProbeStore(core, datastore) else: - self.probedata.pop(client, None) - self.cgroups.pop(client, None) - probedata = ProbesDataModel.objects.filter(hostname=client) - groupdata = ProbesGroupsModel.objects.filter(hostname=client) - - for pdata in probedata: - if pdata.hostname not in self.probedata: - self.probedata[pdata.hostname] = ClientProbeDataSet( - timestamp=time.mktime(pdata.timestamp.timetuple())) - self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in groupdata: - if pgroup.hostname not in self.cgroups: - self.cgroups[pgroup.hostname] = [] - self.cgroups[pgroup.hostname].append(pgroup.group) - - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, - key=client) + self.probestore = XMLProbeStore(core, datastore) @track_statistics() - def GetProbes(self, meta): - return self.probes.get_probe_data(meta) - GetProbes.__doc__ = Bcfg2.Server.Plugin.Probing.GetProbes.__doc__ + def GetProbes(self, metadata): + return self.probes.get_probe_data(metadata) @track_statistics() def ReceiveData(self, client, datalist): - if self.core.metadata_cache_mode in ['cautious', 'aggressive']: - if client.hostname in self.cgroups: - olddata = copy.copy(self.cgroups[client.hostname]) - else: - olddata = [] - - cgroups = [] - cprobedata = ClientProbeDataSet() + cgroups = set() + cdata = dict() for data in datalist: - self.ReceiveDataItem(client, data, cgroups, cprobedata) - self.cgroups[client.hostname] = cgroups - self.probedata[client.hostname] = cprobedata - - if (self.core.metadata_cache_mode in ['cautious', 'aggressive'] and - olddata != self.cgroups[client.hostname]): - self.core.metadata_cache.expire(client.hostname) - self.write_data(client) - ReceiveData.__doc__ = Bcfg2.Server.Plugin.Probing.ReceiveData.__doc__ - - def ReceiveDataItem(self, client, data, cgroups, cprobedata): - """Receive probe results pertaining to client.""" + groups, cdata[data.get("name")] = \ + self.ReceiveDataItem(client, data) + cgroups.update(groups) + self.probestore.set_groups(client.hostname, list(cgroups)) + self.probestore.set_data(client.hostname, cdata) + self.probestore.commit() + + def ReceiveDataItem(self, client, data): + """ Receive probe results pertaining to client. Returns a + tuple of (<probe groups>, <probe data>). """ if data.text is None: self.logger.info("Got null response to probe %s from %s" % (data.get('name'), client.hostname)) - cprobedata[data.get('name')] = ProbeData('') - return + return [], '' dlines = data.text.split('\n') self.logger.debug("Processing probe from %s: %s:%s" % (client.hostname, data.get('name'), [line.strip() for line in dlines])) + groups = [] for line in dlines[:]: - if line.split(':')[0] == 'group': - newgroup = line.split(':')[1].strip() - if newgroup not in cgroups: - cgroups.append(newgroup) + match = self.groupline_re.match(line) + if match: + groups.append(match.group("groupname")) dlines.remove(line) - dobj = ProbeData("\n".join(dlines)) - cprobedata[data.get('name')] = dobj - - def get_additional_groups(self, meta): - return self.cgroups.get(meta.hostname, list()) - get_additional_groups.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_groups.__doc__ - - def get_additional_data(self, meta): - return self.probedata.get(meta.hostname, ClientProbeDataSet()) - get_additional_data.__doc__ = \ - Bcfg2.Server.Plugin.Connector.get_additional_data.__doc__ + return (groups, ProbeData("\n".join(dlines))) + + def get_additional_groups(self, metadata): + return self.probestore.get_groups(metadata.hostname) + + def get_additional_data(self, metadata): + return self.probestore.get_data(metadata.hostname) diff --git a/src/lib/Bcfg2/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index 541116db3..b5c60c875 100644 --- a/src/lib/Bcfg2/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -19,9 +19,10 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): self._regex_cache = dict() def HandlesEntry(self, entry, metadata): - if entry.tag in self.Entries: - return self._matches(entry, metadata, - self.Entries[entry.tag].keys()) + for src in self.entries.values(): + for candidate in src.XMLMatch(metadata).xpath("//%s" % entry.tag): + if self._matches(entry, metadata, candidate): + return True return False HandleEntry = Bcfg2.Server.Plugin.PrioDir.BindEntry diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 8ce4e8a54..f3f711b77 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -5,7 +5,6 @@ import os import sys import socket import shutil -import logging import tempfile import Bcfg2.Options import Bcfg2.Server.Plugin @@ -14,16 +13,10 @@ from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Compat import any, u_str, b64encode # pylint: disable=W0622 -LOGGER = logging.getLogger(__name__) - class KeyData(Bcfg2.Server.Plugin.SpecificData): """ class to handle key data for HostKeyEntrySet """ - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific) - self.encoding = encoding - def __lt__(self, other): return self.name < other.name @@ -40,19 +33,20 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): entry.text = b64encode(self.data) else: try: - entry.text = u_str(self.data, self.encoding) + entry.text = u_str(self.data, Bcfg2.Options.setup.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("Please verify you are using the proper encoding") + self.logger.error(msg) + self.logger.error("Please verify you are using the proper " + "encoding") raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: msg = "Error in specification for %s: %s" % (entry.get('name'), sys.exc_info()[1]) - LOGGER.error(msg) - LOGGER.error("You need to specify base64 encoding for %s" % - entry.get('name')) + self.logger.error(msg) + self.logger.error("You need to specify base64 encoding for %s" + % entry.get('name')) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if entry.text in ['', None]: entry.set('empty', 'true') @@ -61,16 +55,12 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): """ EntrySet to handle all kinds of host keys """ def __init__(self, basename, path): - if basename.startswith("ssh_host_key"): - self.encoding = "base64" - else: - self.encoding = None Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData) self.metadata = {'owner': 'root', 'group': 'root', 'type': 'file'} - if self.encoding is not None: - self.metadata['encoding'] = self.encoding + if basename.startswith("ssh_host_key"): + self.metadata['encoding'] = "base64" if basename.endswith('.pub'): self.metadata['mode'] = '0644' else: @@ -89,7 +79,6 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -123,7 +112,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -150,9 +138,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk self.cmd = Executor() - def expire_cache(self, key=None): - self.__skn = False - def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py deleted file mode 100644 index 74d8833f4..000000000 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ /dev/null @@ -1,387 +0,0 @@ -""" The SSLCA generator handles the creation and management of ssl -certificates and their keys. """ - -import os -import sys -import tempfile -import lxml.etree -import Bcfg2.Server.Plugin -from Bcfg2.Utils import Executor -from Bcfg2.Compat import ConfigParser -from Bcfg2.Server.Plugin import PluginExecutionError - - -class SSLCAXMLSpec(Bcfg2.Server.Plugin.StructFile): - """ Base class to handle key.xml and cert.xml """ - encryption = False - attrs = dict() - tag = None - - def get_spec(self, metadata): - """ Get a specification for the type of object described by - this SSLCA XML file for the given client metadata object """ - entries = [e for e in self.Match(metadata) if e.tag == self.tag] - if len(entries) == 0: - raise PluginExecutionError("No matching %s entry found for %s " - "in %s" % (self.tag, - metadata.hostname, - self.name)) - elif len(entries) > 1: - self.logger.warning( - "More than one matching %s entry found for %s in %s; " - "using first match" % (self.tag, metadata.hostname, self.name)) - rv = dict() - for attr, default in self.attrs.items(): - val = entries[0].get(attr.lower(), default) - if default in ['true', 'false']: - rv[attr] = val == 'true' - else: - rv[attr] = val - return rv - - -class SSLCAKeySpec(SSLCAXMLSpec): - """ Handle key.xml files """ - attrs = dict(bits='2048', type='rsa') - tag = 'Key' - - -class SSLCACertSpec(SSLCAXMLSpec): - """ Handle cert.xml files """ - attrs = dict(ca='default', - format='pem', - key=None, - days='365', - C=None, - L=None, - ST=None, - OU=None, - O=None, - emailAddress=None, - append_chain='false') - tag = 'Cert' - - def get_spec(self, metadata): - rv = SSLCAXMLSpec.get_spec(self, metadata) - rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "subjectAltName"] - return rv - - -class SSLCADataFile(Bcfg2.Server.Plugin.SpecificData): - """ Handle key and cert files """ - def bind_entry(self, entry, _): - """ Bind the data in the file to the given abstract entry """ - entry.text = self.data - entry.set("type", "file") - return entry - - -class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): - """ Entry set to handle SSLCA entries and XML files """ - def __init__(self, _, path, entry_type, parent=None): - Bcfg2.Server.Plugin.EntrySet.__init__(self, os.path.basename(path), - path, entry_type) - self.parent = parent - self.key = None - self.cert = None - self.cmd = Executor(timeout=120) - - def handle_event(self, event): - action = event.code2str() - fpath = os.path.join(self.path, event.filename) - - if event.filename == 'key.xml': - if action in ['exists', 'created', 'changed']: - self.key = SSLCAKeySpec(fpath) - self.key.HandleEvent(event) - elif event.filename == 'cert.xml': - if action in ['exists', 'created', 'changed']: - self.cert = SSLCACertSpec(fpath) - self.cert.HandleEvent(event) - else: - Bcfg2.Server.Plugin.EntrySet.handle_event(self, event) - - def build_key(self, entry, metadata): - """ - either grabs a prexisting key hostfile, or triggers the generation - of a new key if one doesn't exist. - """ - # TODO: verify key fits the specs - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new key %s" % filename) - key_spec = self.key.get_spec(metadata) - ktype = key_spec['type'] - bits = key_spec['bits'] - if ktype == 'rsa': - cmd = ["openssl", "genrsa", bits] - elif ktype == 'dsa': - cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] - self.debug_log("SSLCA: Generating new key: %s" % " ".join(cmd)) - result = self.cmd.run(cmd) - if not result.success: - raise PluginExecutionError("SSLCA: Failed to generate key %s for " - "%s: %s" % (entry.get("name"), - metadata.hostname, - result.error)) - open(os.path.join(self.path, filename), 'w').write(result.stdout) - return result.stdout - - def build_cert(self, entry, metadata, keyfile): - """ generate a new cert """ - filename = "%s.H_%s" % (os.path.basename(entry.get('name')), - metadata.hostname) - self.logger.info("SSLCA: Generating new cert %s" % filename) - cert_spec = self.cert.get_spec(metadata) - ca = self.parent.get_ca(cert_spec['ca']) - req_config = None - req = None - try: - req_config = self.build_req_config(metadata) - req = self.build_request(keyfile, req_config, metadata) - days = cert_spec['days'] - cmd = ["openssl", "ca", "-config", ca['config'], "-in", req, - "-days", days, "-batch"] - passphrase = ca.get('passphrase') - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - - def _scrub_pass(arg): - """ helper to scrub the passphrase from the - argument list """ - if arg.startswith("pass:"): - return "pass:******" - else: - return arg - else: - _scrub_pass = lambda a: a - - self.debug_log("SSLCA: Generating new certificate: %s" % - " ".join(_scrub_pass(a) for a in cmd)) - result = self.cmd.run(cmd) - if not result.success: - raise PluginExecutionError("SSLCA: Failed to generate cert: %s" - % result.error) - finally: - try: - if req_config and os.path.exists(req_config): - os.unlink(req_config) - if req and os.path.exists(req): - os.unlink(req) - except OSError: - self.logger.error("SSLCA: Failed to unlink temporary files: %s" - % sys.exc_info()[1]) - cert = result.stdout - if cert_spec['append_chain'] and 'chaincert' in ca: - cert += open(ca['chaincert']).read() - - open(os.path.join(self.path, filename), 'w').write(cert) - return cert - - def build_req_config(self, metadata): - """ - generates a temporary openssl configuration file that is - used to generate the required certificate request - """ - # create temp request config file - fd, fname = tempfile.mkstemp() - cfp = ConfigParser.ConfigParser({}) - cfp.optionxform = str - defaults = { - 'req': { - 'default_md': 'sha1', - 'distinguished_name': 'req_distinguished_name', - 'req_extensions': 'v3_req', - 'x509_extensions': 'v3_req', - 'prompt': 'no' - }, - 'req_distinguished_name': {}, - 'v3_req': { - 'subjectAltName': '@alt_names' - }, - 'alt_names': {} - } - for section in list(defaults.keys()): - cfp.add_section(section) - for key in defaults[section]: - cfp.set(section, key, defaults[section][key]) - cert_spec = self.cert.get_spec(metadata) - altnamenum = 1 - altnames = cert_spec['subjectaltname'] - altnames.extend(list(metadata.aliases)) - altnames.append(metadata.hostname) - for altname in altnames: - cfp.set('alt_names', 'DNS.' + str(altnamenum), altname) - altnamenum += 1 - for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: - if cert_spec[item]: - cfp.set('req_distinguished_name', item, cert_spec[item]) - cfp.set('req_distinguished_name', 'CN', metadata.hostname) - self.debug_log("SSLCA: Writing temporary request config to %s" % fname) - try: - cfp.write(os.fdopen(fd, 'w')) - except IOError: - raise PluginExecutionError("SSLCA: Failed to write temporary CSR " - "config file: %s" % sys.exc_info()[1]) - return fname - - def build_request(self, keyfile, req_config, metadata): - """ - creates the certificate request - """ - fd, req = tempfile.mkstemp() - os.close(fd) - days = self.cert.get_spec(metadata)['days'] - cmd = ["openssl", "req", "-new", "-config", req_config, - "-days", days, "-key", keyfile, "-text", "-out", req] - self.debug_log("SSLCA: Generating new CSR: %s" % " ".join(cmd)) - result = self.cmd.run(cmd) - if not result.success: - raise PluginExecutionError("SSLCA: Failed to generate CSR: %s" % - result.error) - return req - - def verify_cert(self, filename, keyfile, entry, metadata): - """ Perform certification verification against the CA and - against the key """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - do_verify = ca.get('chaincert') - if do_verify: - return (self.verify_cert_against_ca(filename, entry, metadata) and - self.verify_cert_against_key(filename, keyfile)) - return True - - def verify_cert_against_ca(self, filename, entry, metadata): - """ - check that a certificate validates against the ca cert, - and that it has not expired. - """ - ca = self.parent.get_ca(self.cert.get_spec(metadata)['ca']) - chaincert = ca.get('chaincert') - cert = os.path.join(self.path, filename) - cmd = ["openssl", "verify"] - is_root = ca.get('root_ca', "false").lower() == 'true' - if is_root: - cmd.append("-CAfile") - else: - # verifying based on an intermediate cert - cmd.extend(["-purpose", "sslserver", "-untrusted"]) - cmd.extend([chaincert, cert]) - self.debug_log("SSLCA: Verifying %s against CA: %s" % - (entry.get("name"), " ".join(cmd))) - result = self.cmd.run(cmd) - if result.stdout == cert + ": OK\n": - self.debug_log("SSLCA: %s verified successfully against CA" % - entry.get("name")) - return True - self.logger.warning("SSLCA: %s failed verification against CA: %s" % - (entry.get("name"), result.error)) - return False - - def _get_modulus(self, fname, ftype="x509"): - """ get the modulus from the given file """ - cmd = ["openssl", ftype, "-noout", "-modulus", "-in", fname] - self.debug_log("SSLCA: Getting modulus of %s for verification: %s" % - (fname, " ".join(cmd))) - result = self.cmd.run(cmd) - if not result.success: - self.logger.warning("SSLCA: Failed to get modulus of %s: %s" % - (fname, result.error)) - return result.stdout.strip() - - def verify_cert_against_key(self, filename, keyfile): - """ - check that a certificate validates against its private key. - """ - - certfile = os.path.join(self.path, filename) - cert = self._get_modulus(certfile) - key = self._get_modulus(keyfile, ftype="rsa") - if cert == key: - self.debug_log("SSLCA: %s verified successfully against key %s" % - (filename, keyfile)) - return True - self.logger.warning("SSLCA: %s failed verification against key %s" % - (filename, keyfile)) - return False - - def bind_entry(self, entry, metadata): - if self.key: - self.bind_info_to_entry(entry, metadata) - try: - return self.best_matching(metadata).bind_entry(entry, metadata) - except PluginExecutionError: - entry.text = self.build_key(entry, metadata) - entry.set("type", "file") - return entry - elif self.cert: - key = self.cert.get_spec(metadata)['key'] - cleanup_keyfile = False - try: - keyfile = self.parent.entries[key].best_matching(metadata).name - except PluginExecutionError: - cleanup_keyfile = True - # create a temp file with the key in it - fd, keyfile = tempfile.mkstemp() - os.chmod(keyfile, 384) # 0600 - el = lxml.etree.Element('Path', name=key) - self.parent.core.Bind(el, metadata) - os.fdopen(fd, 'w').write(el.text) - - try: - self.bind_info_to_entry(entry, metadata) - try: - best = self.best_matching(metadata) - if self.verify_cert(best.name, keyfile, entry, metadata): - return best.bind_entry(entry, metadata) - except PluginExecutionError: - pass - # if we get here, it's because either a) there was no best - # matching entry; or b) the existing cert did not verify - entry.text = self.build_cert(entry, metadata, keyfile) - entry.set("type", "file") - return entry - finally: - if cleanup_keyfile: - try: - os.unlink(keyfile) - except OSError: - err = sys.exc_info()[1] - self.logger.error("SSLCA: Failed to unlink temporary " - "key %s: %s" % (keyfile, err)) - - -class SSLCA(Bcfg2.Server.Plugin.GroupSpool): - """ The SSLCA generator handles the creation and management of ssl - certificates and their keys. """ - __author__ = 'g.hagger@gmail.com' - - options = Bcfg2.Server.Plugin.GroupSpool.options + [ - Bcfg2.Options.WildcardSectionGroup( - Bcfg2.Options.PathOption( - cf=("sslca_*", "config"), - help="Path to the openssl config for the CA"), - Bcfg2.Options.Option( - cf=("sslca_*", "passphrase"), - help="Passphrase for the CA private key"), - Bcfg2.Options.PathOption( - cf=("sslca_*", "chaincert"), - help="Path to the SSL chaining certificate for verification"), - Bcfg2.Options.BooleanOption( - cf=("sslca_*", "root_ca"), - help="Whether or not <chaincert> is a root CA (as opposed to " - "an intermediate cert"))] - - # python 2.5 doesn't support mixing *magic and keyword arguments - es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) - es_child_cls = SSLCADataFile - - def get_ca(self, name): - """ get a dict describing a CA from the config file """ - rv = dict() - prefix = "sslca_%s_" % name - for attr in dir(Bcfg2.Options.setup): - if attr.startswith(prefix): - rv[attr[len(prefix):]] = getattr(Bcfg2.Options.setup, attr) - return rv diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 646124fcc..6a3948f40 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -5,7 +5,6 @@ better. """ import os import sys import socket -import select import signal import logging import ssl @@ -237,22 +236,22 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): return False return True - ### need to override do_POST here def do_POST(self): try: max_chunk_size = 10 * 1024 * 1024 size_remaining = int(self.headers["content-length"]) L = [] while size_remaining: - try: - select.select([self.rfile.fileno()], [], [], 3) - except select.error: - self.logger.error("Got select timeout") - raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size).decode('utf-8')) + chunk = self.rfile.read(chunk_size).decode('utf-8') + if not chunk: + break + L.append(chunk) size_remaining -= len(L[-1]) data = ''.join(L) + if data is None: + return # response has been sent + response = self.server._marshaled_dispatch(self.client_address, data) if sys.hexversion >= 0x03000000: @@ -265,6 +264,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): (self.client_address, sys.exc_info()[1])) try: self.send_response(500) + self.send_header("Content-length", "0") self.end_headers() except: (etype, msg) = sys.exc_info()[:2] @@ -321,14 +321,11 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def finish(self): # shut down the connection - if not self.wfile.closed: - try: - self.wfile.flush() - self.wfile.close() - except socket.error: - err = sys.exc_info()[1] - self.logger.warning("Error closing connection: %s" % err) - self.rfile.close() + try: + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.finish(self) + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, @@ -446,8 +443,6 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, self.handle_request() except socket.timeout: pass - except select.error: - pass except: self.logger.error("Got unexpected error in handle_request", exc_info=1) diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 236f87d0a..10057b63e 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -2,12 +2,13 @@ used by both client and server. Stuff that doesn't fit anywhere else. """ +import fcntl +import logging import os import re -import sys -import fcntl import select -import logging +import shlex +import sys import subprocess import threading from Bcfg2.Compat import input, any # pylint: disable=W0622 @@ -216,12 +217,17 @@ class Executor(object): :type timeout: float :returns: :class:`Bcfg2.Utils.ExecutorResult` """ + shell = False + if 'shell' in kwargs: + shell = kwargs['shell'] if isinstance(command, str): cmdstr = command + if not shell: + command = shlex.split(cmdstr) else: cmdstr = " ".join(command) self.logger.debug("Running: %s" % cmdstr) - args = dict(shell=False, bufsize=16384, close_fds=True) + args = dict(shell=shell, bufsize=16384, close_fds=True) args.update(kwargs) args.update(stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 42d415232..7ddf58aed 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -165,7 +165,8 @@ class _OptionContainer(object): dest='db_schema'), Bcfg2.Options.Option( cf=('database', 'options'), help='Database options', - dest='db_opts', type=Bcfg2.Options.Types.comma_dict), + dest='db_opts', type=Bcfg2.Options.Types.comma_dict, + default=dict()), Bcfg2.Options.Option( cf=('reporting', 'timezone'), help='Django timezone'), Bcfg2.Options.BooleanOption( diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index bb7db5e14..5a752b2ac 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -734,12 +734,13 @@ class TestPOSIXTool(TestTool): gather_data_rv[idx] = val ptool._gather_data.return_value = tuple(gather_data_rv) + stat_mode = 17407 mtime = 1344430414 + stat_rv = (stat_mode, Mock(), Mock(), Mock(), Mock(), Mock(), Mock(), + Mock(), mtime, Mock()) + gather_data_rv[0] = stat_rv entry = reset() entry.set("mtime", str(mtime)) - stat_rv = MagicMock() - stat_rv.__getitem__.return_value = mtime - gather_data_rv[0] = stat_rv ptool._gather_data.return_value = tuple(gather_data_rv) self.assertTrue(ptool._verify_metadata(entry)) ptool._gather_data.assert_called_with(entry.get("name")) @@ -811,7 +812,7 @@ class TestPOSIXTool(TestTool): ptool._gather_data.assert_called_with(entry.get("name")) ptool._verify_acls.assert_called_with(entry, path=entry.get("name")) - mock_matchpathcon.assert_called_with(entry.get("name"), 0) + mock_matchpathcon.assert_called_with(entry.get("name"), stat_mode) self.assertEqual(entry.get("current_exists", 'true'), 'true') for attr, idx, val in expected: self.assertEqual(entry.get(attr), val) @@ -826,7 +827,7 @@ class TestPOSIXTool(TestTool): ptool._gather_data.assert_called_with(entry.get("name")) ptool._verify_acls.assert_called_with(entry, path=entry.get("name")) - mock_matchpathcon.assert_called_with(entry.get("name"), 0) + mock_matchpathcon.assert_called_with(entry.get("name"), stat_mode) self.assertEqual(entry.get("current_exists", 'true'), 'true') for attr, idx, val in expected: self.assertEqual(entry.get(attr), val) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestCache.py b/testsuite/Testsrc/Testlib/TestServer/TestCache.py new file mode 100644 index 000000000..7c26e52b8 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestCache.py @@ -0,0 +1,54 @@ +import os +import sys + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +from Bcfg2.Server.Cache import * + + +class TestCache(Bcfg2TestCase): + def test_cache(self): + md_cache = Cache("Metadata") + md_cache['foo.example.com'] = 'foo metadata' + md_cache['bar.example.com'] = 'bar metadata' + self.assertItemsEqual(list(iter(md_cache)), + ["foo.example.com", "bar.example.com"]) + + probe_cache = Cache("Probes", "data") + probe_cache['foo.example.com'] = 'foo probe data' + probe_cache['bar.example.com'] = 'bar probe data' + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + md_cache.expire("foo.example.com") + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + probe_cache.expire("bar.example.com") + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com"]) + + probe_cache['bar.example.com'] = 'bar probe data' + self.assertItemsEqual(list(iter(md_cache)), ["bar.example.com"]) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com", "bar.example.com"]) + + expire("bar.example.com") + self.assertEqual(len(md_cache), 0) + self.assertItemsEqual(list(iter(probe_cache)), + ["foo.example.com"]) + + probe_cache2 = Cache("Probes", "data") + self.assertItemsEqual(list(iter(probe_cache)), + list(iter(probe_cache2))) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py b/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py index 2267e9d6c..cfb0c023b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestEncryption.py @@ -22,129 +22,128 @@ except ImportError: HAS_CRYPTO = False -if can_skip or HAS_CRYPTO: - class TestEncryption(Bcfg2TestCase): - plaintext = """foo bar +class TestEncryption(Bcfg2TestCase): + plaintext = """foo bar baz ö \t\tquux """ + "a" * 16384 # 16K is completely arbitrary - iv = "0123456789ABCDEF" - salt = "01234567" - algo = "des_cbc" - - @skipUnless(HAS_CRYPTO, "Encryption libraries not found") - def setUp(self): - Bcfg2.Options.setup.algorithm = "aes_256_cbc" - - def test_str_crypt(self): - """ test str_encrypt/str_decrypt """ - key = "a simple key" - - # simple symmetrical test with no options - crypted = str_encrypt(self.plaintext, key) - self.assertEqual(self.plaintext, str_decrypt(crypted, key)) - - # symmetrical test with lots of options - crypted = str_encrypt(self.plaintext, key, - iv=self.iv, salt=self.salt, - algorithm=self.algo) - self.assertEqual(self.plaintext, - str_decrypt(crypted, key, iv=self.iv, - algorithm=self.algo)) - - # test that different algorithms are actually used - self.assertNotEqual(str_encrypt(self.plaintext, key), - str_encrypt(self.plaintext, key, - algorithm=self.algo)) - - # test that different keys are actually used - self.assertNotEqual(str_encrypt(self.plaintext, key), - str_encrypt(self.plaintext, "different key")) - - # test that different IVs are actually used - self.assertNotEqual(str_encrypt(self.plaintext, key, iv=self.iv), - str_encrypt(self.plaintext, key)) - - # test that errors are raised on bad decrypts - crypted = str_encrypt(self.plaintext, key, algorithm=self.algo) - self.assertRaises(EVPError, str_decrypt, - crypted, "bogus key", algorithm=self.algo) - self.assertRaises(EVPError, str_decrypt, - crypted, key) # bogus algorithm - - def test_ssl_crypt(self): - """ test ssl_encrypt/ssl_decrypt """ - passwd = "a simple passphrase" - - # simple symmetrical test - crypted = ssl_encrypt(self.plaintext, passwd) - self.assertEqual(self.plaintext, ssl_decrypt(crypted, passwd)) - - # more complex symmetrical test - crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo, - salt=self.salt) - self.assertEqual(self.plaintext, - ssl_decrypt(crypted, passwd, algorithm=self.algo)) - - # test that different algorithms are actually used - self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), - ssl_encrypt(self.plaintext, passwd, - algorithm=self.algo)) - - # test that different passwords are actually used - self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), - ssl_encrypt(self.plaintext, "different pass")) - - # there's no reasonable test we can do to see if the - # output is base64-encoded, unfortunately, but if it's - # obviously not we fail - crypted = ssl_encrypt(self.plaintext, passwd) - self.assertRegexpMatches(crypted, r'^[A-Za-z0-9+/]+[=]{0,2}$') - - # test that errors are raised on bad decrypts - crypted = ssl_encrypt(self.plaintext, passwd, - algorithm=self.algo) - self.assertRaises(EVPError, ssl_decrypt, - crypted, "bogus passwd", algorithm=self.algo) - self.assertRaises(EVPError, ssl_decrypt, - crypted, passwd) # bogus algorithm - - def test_bruteforce_decrypt(self): - passwd = "a simple passphrase" - crypted = ssl_encrypt(self.plaintext, passwd) - - # test with no passphrases given nor in config - Bcfg2.Options.setup.passphrases = dict() - self.assertRaises(EVPError, - bruteforce_decrypt, crypted) - - # test with good passphrase given in function call - self.assertEqual(self.plaintext, - bruteforce_decrypt(crypted, - passphrases=["bogus pass", - passwd, - "also bogus"])) - - # test with no good passphrase given nor in config - self.assertRaises(EVPError, - bruteforce_decrypt, - crypted, passphrases=["bogus", "also bogus"]) - - # test with good passphrase in config file - Bcfg2.Options.setup.passphrases = dict(bogus="bogus", - real=passwd, - bogus2="also bogus") - self.assertEqual(self.plaintext, - bruteforce_decrypt(crypted)) - - # test that passphrases given in function call take - # precedence over config - self.assertRaises(EVPError, - bruteforce_decrypt, crypted, - passphrases=["bogus", "also bogus"]) - - # test that different algorithms are used - crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo) - self.assertEqual(self.plaintext, - bruteforce_decrypt(crypted, algorithm=self.algo)) + iv = "0123456789ABCDEF" + salt = "01234567" + algo = "des_cbc" + + @skipUnless(HAS_CRYPTO, "Encryption libraries not found") + def setUp(self): + Bcfg2.Options.setup.algorithm = "aes_256_cbc" + + def test_str_crypt(self): + """ test str_encrypt/str_decrypt """ + key = "a simple key" + + # simple symmetrical test with no options + crypted = str_encrypt(self.plaintext, key) + self.assertEqual(self.plaintext, str_decrypt(crypted, key)) + + # symmetrical test with lots of options + crypted = str_encrypt(self.plaintext, key, + iv=self.iv, salt=self.salt, + algorithm=self.algo) + self.assertEqual(self.plaintext, + str_decrypt(crypted, key, iv=self.iv, + algorithm=self.algo)) + + # test that different algorithms are actually used + self.assertNotEqual(str_encrypt(self.plaintext, key), + str_encrypt(self.plaintext, key, + algorithm=self.algo)) + + # test that different keys are actually used + self.assertNotEqual(str_encrypt(self.plaintext, key), + str_encrypt(self.plaintext, "different key")) + + # test that different IVs are actually used + self.assertNotEqual(str_encrypt(self.plaintext, key, iv=self.iv), + str_encrypt(self.plaintext, key)) + + # test that errors are raised on bad decrypts + crypted = str_encrypt(self.plaintext, key, algorithm=self.algo) + self.assertRaises(EVPError, str_decrypt, + crypted, "bogus key", algorithm=self.algo) + self.assertRaises(EVPError, str_decrypt, + crypted, key) # bogus algorithm + + def test_ssl_crypt(self): + """ test ssl_encrypt/ssl_decrypt """ + passwd = "a simple passphrase" + + # simple symmetrical test + crypted = ssl_encrypt(self.plaintext, passwd) + self.assertEqual(self.plaintext, ssl_decrypt(crypted, passwd)) + + # more complex symmetrical test + crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo, + salt=self.salt) + self.assertEqual(self.plaintext, + ssl_decrypt(crypted, passwd, algorithm=self.algo)) + + # test that different algorithms are actually used + self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), + ssl_encrypt(self.plaintext, passwd, + algorithm=self.algo)) + + # test that different passwords are actually used + self.assertNotEqual(ssl_encrypt(self.plaintext, passwd), + ssl_encrypt(self.plaintext, "different pass")) + + # there's no reasonable test we can do to see if the + # output is base64-encoded, unfortunately, but if it's + # obviously not we fail + crypted = ssl_encrypt(self.plaintext, passwd) + self.assertRegexpMatches(crypted, r'^[A-Za-z0-9+/]+[=]{0,2}$') + + # test that errors are raised on bad decrypts + crypted = ssl_encrypt(self.plaintext, passwd, + algorithm=self.algo) + self.assertRaises(EVPError, ssl_decrypt, + crypted, "bogus passwd", algorithm=self.algo) + self.assertRaises(EVPError, ssl_decrypt, + crypted, passwd) # bogus algorithm + + def test_bruteforce_decrypt(self): + passwd = "a simple passphrase" + crypted = ssl_encrypt(self.plaintext, passwd) + + # test with no passphrases given nor in config + Bcfg2.Options.setup.passphrases = dict() + self.assertRaises(EVPError, + bruteforce_decrypt, crypted) + + # test with good passphrase given in function call + self.assertEqual(self.plaintext, + bruteforce_decrypt(crypted, + passphrases=["bogus pass", + passwd, + "also bogus"])) + + # test with no good passphrase given nor in config + self.assertRaises(EVPError, + bruteforce_decrypt, + crypted, passphrases=["bogus", "also bogus"]) + + # test with good passphrase in config file + Bcfg2.Options.setup.passphrases = dict(bogus="bogus", + real=passwd, + bogus2="also bogus") + self.assertEqual(self.plaintext, + bruteforce_decrypt(crypted)) + + # test that passphrases given in function call take + # precedence over config + self.assertRaises(EVPError, + bruteforce_decrypt, crypted, + passphrases=["bogus", "also bogus"]) + + # test that different algorithms are used + crypted = ssl_encrypt(self.plaintext, passwd, algorithm=self.algo) + self.assertEqual(self.plaintext, + bruteforce_decrypt(crypted, algorithm=self.algo)) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 7006e29e3..dbab60abc 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -24,9 +24,8 @@ from TestServer.TestPlugin.Testinterfaces import TestGenerator try: from Bcfg2.Server.Encryption import EVPError - HAS_CRYPTO = True except: - HAS_CRYPTO = False + pass def tostring(el): @@ -607,24 +606,20 @@ class TestXMLFileBacked(TestFileBacked): xfb.add_monitor("/test/test2.xml") self.assertIn("/test/test2.xml", xfb.extra_monitors) - if self.should_monitor is not True: - xfb = self.get_obj() - xfb.fam = Mock() - xfb.add_monitor("/test/test3.xml") - self.assertFalse(xfb.fam.AddMonitor.called) - self.assertIn("/test/test3.xml", xfb.extra_monitors) - - if self.should_monitor is not False: - xfb = self.get_obj(should_monitor=True) - xfb.fam = Mock() - xfb.add_monitor("/test/test4.xml") - xfb.fam.AddMonitor.assert_called_with("/test/test4.xml", xfb) - self.assertIn("/test/test4.xml", xfb.extra_monitors) + xfb = self.get_obj() + xfb.fam = Mock() + xfb.add_monitor("/test/test4.xml") + xfb.fam.AddMonitor.assert_called_with("/test/test4.xml", xfb) + self.assertIn("/test/test4.xml", xfb.extra_monitors) class TestStructFile(TestXMLFileBacked): test_obj = StructFile + def setUp(self): + TestXMLFileBacked.setUp(self) + set_setup_default("lax_decryption", False) + def _get_test_data(self): """ build a very complex set of test data """ # top-level group and client elements @@ -707,11 +702,10 @@ class TestStructFile(TestXMLFileBacked): @patch("genshi.template.TemplateLoader") def test_Index(self, mock_TemplateLoader): - has_crypto = Bcfg2.Server.Plugin.helpers.HAS_CRYPTO - Bcfg2.Server.Plugin.helpers.HAS_CRYPTO = False TestXMLFileBacked.test_Index(self) sf = self.get_obj() + sf.encryption = False sf.encoding = Mock() (xdata, groups, subgroups, children, subchildren, standalone) = \ self._get_test_data() @@ -736,10 +730,10 @@ class TestStructFile(TestXMLFileBacked): self.assertEqual(sf.template, loader.load.return_value) - Bcfg2.Server.Plugin.helpers.HAS_CRYPTO = has_crypto - @skipUnless(HAS_CRYPTO, "No crypto libraries found, skipping") def test_Index_crypto(self): + if not self.test_obj.encryption: + return Bcfg2.Options.setup.lax_decryption = False sf = self.get_obj() sf._decrypt = Mock() @@ -1197,13 +1191,20 @@ class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): Mock()) def inner(): pd = self.get_obj() - test1 = Mock() - test1.items = dict(Path=["/etc/foo.conf", "/etc/bar.conf"]) - test2 = Mock() - test2.items = dict(Path=["/etc/baz.conf"], - Package=["quux", "xyzzy"]) - pd.entries = {"/test1.xml": test1, - "/test2.xml": test2} + test1 = lxml.etree.Element("Test") + lxml.etree.SubElement(test1, "Path", name="/etc/foo.conf") + lxml.etree.SubElement(lxml.etree.SubElement(test1, + "Group", name="foo"), + "Path", name="/etc/bar.conf") + + test2 = lxml.etree.Element("Test") + lxml.etree.SubElement(test2, "Path", name="/etc/baz.conf") + lxml.etree.SubElement(test2, "Package", name="quux") + lxml.etree.SubElement(lxml.etree.SubElement(test2, + "Group", name="bar"), + "Package", name="xyzzy") + pd.entries = {"/test1.xml": Mock(xdata=test1), + "/test2.xml": Mock(xdata=test2)} pd.HandleEvent(Mock()) self.assertItemsEqual(pd.Entries, dict(Path={"/etc/foo.conf": pd.BindEntry, @@ -1375,10 +1376,10 @@ class TestSpecificData(TestDebuggable): sd = self.get_obj() sd.handle_event(event) self.assertFalse(mock_open.called) - if hasattr(sd, 'data'): - self.assertIsNone(sd.data) - else: + try: self.assertFalse(hasattr(sd, 'data')) + except AssertionError: + self.assertIsNone(sd.data) event = Mock() mock_open.return_value.read.return_value = "test" diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py index e5cef8fa2..f41ae8a46 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgAuthorizedKeysGenerator.py @@ -27,12 +27,12 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): TestCfgGenerator.setUp(self) TestStructFile.setUp(self) - def get_obj(self, name=None, core=None, fam=None): + @patch("Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.get_cfg") + def get_obj(self, mock_get_cfg, name=None, core=None, fam=None): if name is None: name = self.path - Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CFG = Mock() if core is not None: - Bcfg2.Server.Plugins.Cfg.CfgAuthorizedKeysGenerator.CFG.core = core + mock_get_cfg.return_value.core = core return self.test_obj(name) @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") @@ -111,17 +111,18 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): reset() host = "baz.example.com" spec = lxml.etree.Element("AuthorizedKeys") - lxml.etree.SubElement( - lxml.etree.SubElement(spec, - "Allow", - attrib={"from": pubkey, "host": host}), - "Params", foo="foo", bar="bar=bar") + allow = lxml.etree.SubElement(spec, "Allow", + attrib={"from": pubkey, "host": host}) + lxml.etree.SubElement(allow, "Option", name="foo", value="foo") + lxml.etree.SubElement(allow, "Option", name="bar") + lxml.etree.SubElement(allow, "Option", name="baz", value="baz=baz") akg.XMLMatch.return_value = spec params, actual_host, actual_pubkey = akg.get_data(entry, metadata).split() self.assertEqual(actual_host, host) self.assertEqual(actual_pubkey, pubkey) - self.assertItemsEqual(params.split(","), ["foo=foo", "bar=bar=bar"]) + self.assertItemsEqual(params.split(","), ["foo=foo", "bar", + "baz=baz=baz"]) akg.XMLMatch.assert_called_with(metadata) akg.core.build_metadata.assert_called_with(host) self.assertEqual(akg.core.Bind.call_args[0][0].get("name"), pubkey) @@ -131,10 +132,10 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): spec = lxml.etree.Element("AuthorizedKeys") text = lxml.etree.SubElement(spec, "Allow") text.text = "ssh-rsa publickey /foo/bar\n" - lxml.etree.SubElement(text, "Params", foo="foo") + lxml.etree.SubElement(text, "Option", name="foo") akg.XMLMatch.return_value = spec self.assertEqual(akg.get_data(entry, metadata), - "foo=foo %s" % text.text.strip()) + "foo %s" % text.text.strip()) akg.XMLMatch.assert_called_with(metadata) self.assertFalse(akg.core.build_metadata.called) self.assertFalse(akg.core.Bind.called) @@ -143,7 +144,7 @@ class TestCfgAuthorizedKeysGenerator(TestCfgGenerator, TestStructFile): lxml.etree.SubElement(spec, "Allow", attrib={"from": pubkey}) akg.XMLMatch.return_value = spec self.assertItemsEqual(akg.get_data(entry, metadata).splitlines(), - ["foo=foo %s" % text.text.strip(), + ["foo %s" % text.text.strip(), "profile %s" % pubkey]) akg.XMLMatch.assert_called_with(metadata) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py index e1ffa7272..2285fb458 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py @@ -17,31 +17,30 @@ from common import * from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator -if HAS_CHEETAH or can_skip: - class TestCfgCheetahGenerator(TestCfgGenerator): - test_obj = CfgCheetahGenerator +class TestCfgCheetahGenerator(TestCfgGenerator): + test_obj = CfgCheetahGenerator - @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") - def setUp(self): - TestCfgGenerator.setUp(self) - set_setup_default("repository", datastore) + @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") + def setUp(self): + TestCfgGenerator.setUp(self) + set_setup_default("repository", datastore) - @patch("Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.Template") - def test_get_data(self, mock_Template): - ccg = self.get_obj() - ccg.data = "data" - entry = lxml.etree.Element("Path", name="/test.txt") - metadata = Mock() + @patch("Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.Template") + def test_get_data(self, mock_Template): + ccg = self.get_obj() + ccg.data = "data" + entry = lxml.etree.Element("Path", name="/test.txt") + metadata = Mock() - self.assertEqual(ccg.get_data(entry, metadata), - mock_Template.return_value.respond.return_value) - mock_Template.assert_called_with( - "data".decode(Bcfg2.Options.setup.encoding), - compilerSettings=ccg.settings) - tmpl = mock_Template.return_value - tmpl.respond.assert_called_with() - self.assertEqual(tmpl.metadata, metadata) - self.assertEqual(tmpl.name, entry.get("name")) - self.assertEqual(tmpl.path, entry.get("name")) - self.assertEqual(tmpl.source_path, ccg.name) - self.assertEqual(tmpl.repo, datastore) + self.assertEqual(ccg.get_data(entry, metadata), + mock_Template.return_value.respond.return_value) + mock_Template.assert_called_with( + "data".decode(Bcfg2.Options.setup.encoding), + compilerSettings=ccg.settings) + tmpl = mock_Template.return_value + tmpl.respond.assert_called_with() + self.assertEqual(tmpl.metadata, metadata) + self.assertEqual(tmpl.name, entry.get("name")) + self.assertEqual(tmpl.path, entry.get("name")) + self.assertEqual(tmpl.source_path, ccg.name) + self.assertEqual(tmpl.repo, datastore) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py index 46062569d..4c987551b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedCheetahGenerator.py @@ -30,18 +30,17 @@ except ImportError: HAS_CRYPTO = False -if can_skip or (HAS_CRYPTO and HAS_CHEETAH): - class TestCfgEncryptedCheetahGenerator(TestCfgCheetahGenerator, - TestCfgEncryptedGenerator): - test_obj = CfgEncryptedCheetahGenerator +class TestCfgEncryptedCheetahGenerator(TestCfgCheetahGenerator, + TestCfgEncryptedGenerator): + test_obj = CfgEncryptedCheetahGenerator - @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") - @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") - def setUp(self): - pass + @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + @skipUnless(HAS_CHEETAH, "Cheetah libraries not found, skipping") + def setUp(self): + pass - def test_handle_event(self): - TestCfgEncryptedGenerator.test_handle_event(self) + def test_handle_event(self): + TestCfgEncryptedGenerator.test_handle_event(self) - def test_get_data(self): - TestCfgCheetahGenerator.test_get_data(self) + def test_get_data(self): + TestCfgCheetahGenerator.test_get_data(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py index 5409cf863..873ebd837 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenerator.py @@ -18,54 +18,53 @@ from common import * from TestServer.TestPlugins.TestCfg.Test_init import TestCfgGenerator -if can_skip or HAS_CRYPTO: - class TestCfgEncryptedGenerator(TestCfgGenerator): - test_obj = CfgEncryptedGenerator +class TestCfgEncryptedGenerator(TestCfgGenerator): + test_obj = CfgEncryptedGenerator - @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") - def setUp(self): - TestCfgGenerator.setUp(self) + @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + def setUp(self): + TestCfgGenerator.setUp(self) - @patchIf(HAS_CRYPTO, - "Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.bruteforce_decrypt") - def test_handle_event(self, mock_decrypt): - @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") - def inner(mock_handle_event): - def reset(): - mock_decrypt.reset_mock() - mock_handle_event.reset_mock() + @patchIf(HAS_CRYPTO, + "Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator.bruteforce_decrypt") + def test_handle_event(self, mock_decrypt): + @patch("Bcfg2.Server.Plugins.Cfg.CfgGenerator.handle_event") + def inner(mock_handle_event): + def reset(): + mock_decrypt.reset_mock() + mock_handle_event.reset_mock() - def get_event_data(obj, event): - obj.data = "encrypted" + def get_event_data(obj, event): + obj.data = "encrypted" - mock_handle_event.side_effect = get_event_data - mock_decrypt.side_effect = lambda d, **kw: "plaintext" - event = Mock() - ceg = self.get_obj() - ceg.handle_event(event) - mock_handle_event.assert_called_with(ceg, event) - mock_decrypt.assert_called_with("encrypted") - self.assertEqual(ceg.data, "plaintext") + mock_handle_event.side_effect = get_event_data + mock_decrypt.side_effect = lambda d, **kw: "plaintext" + event = Mock() + ceg = self.get_obj() + ceg.handle_event(event) + mock_handle_event.assert_called_with(ceg, event) + mock_decrypt.assert_called_with("encrypted") + self.assertEqual(ceg.data, "plaintext") - reset() - mock_decrypt.side_effect = EVPError - self.assertRaises(PluginExecutionError, - ceg.handle_event, event) - inner() + reset() + mock_decrypt.side_effect = EVPError + self.assertRaises(PluginExecutionError, + ceg.handle_event, event) + inner() - # to perform the tests from the parent test object, we - # make bruteforce_decrypt just return whatever data was - # given to it - mock_decrypt.side_effect = lambda d, **kw: d - TestCfgGenerator.test_handle_event(self) + # to perform the tests from the parent test object, we + # make bruteforce_decrypt just return whatever data was + # given to it + mock_decrypt.side_effect = lambda d, **kw: d + TestCfgGenerator.test_handle_event(self) - def test_get_data(self): - ceg = self.get_obj() - ceg.data = None - entry = lxml.etree.Element("Path", name="/test.txt") - metadata = Mock() + def test_get_data(self): + ceg = self.get_obj() + ceg.data = None + entry = lxml.etree.Element("Path", name="/test.txt") + metadata = Mock() - self.assertRaises(PluginExecutionError, - ceg.get_data, entry, metadata) + self.assertRaises(PluginExecutionError, + ceg.get_data, entry, metadata) - TestCfgGenerator.test_get_data(self) + TestCfgGenerator.test_get_data(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py index 25d2fb83b..0b74e4a60 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgEncryptedGenshiGenerator.py @@ -18,10 +18,9 @@ from TestServer.TestPlugins.TestCfg.TestCfgGenshiGenerator import \ TestCfgGenshiGenerator -if can_skip or HAS_CRYPTO: - class TestCfgEncryptedGenshiGenerator(TestCfgGenshiGenerator): - test_obj = CfgEncryptedGenshiGenerator +class TestCfgEncryptedGenshiGenerator(TestCfgGenshiGenerator): + test_obj = CfgEncryptedGenshiGenerator - @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") - def setUp(self): - TestCfgGenshiGenerator.setUp(self) + @skipUnless(HAS_CRYPTO, "Encryption libraries not found, skipping") + def setUp(self): + TestCfgGenshiGenerator.setUp(self) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py index c4961db1c..2967a23b6 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py @@ -22,25 +22,15 @@ while path != "/": break path = os.path.dirname(path) from common import * -from TestServer.TestPlugins.TestCfg.Test_init import TestCfgCreator -from TestServer.TestPlugin.Testhelpers import TestStructFile +from TestServer.TestPlugins.TestCfg.Test_init import TestXMLCfgCreator -class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): +class TestCfgPrivateKeyCreator(TestXMLCfgCreator): test_obj = CfgPrivateKeyCreator should_monitor = False def get_obj(self, name=None, fam=None): - return TestCfgCreator.get_obj(self, name=name) - - @patch("Bcfg2.Server.Plugins.Cfg.CfgCreator.handle_event") - @patch("Bcfg2.Server.Plugin.helpers.StructFile.HandleEvent") - def test_handle_event(self, mock_HandleEvent, mock_handle_event): - pkc = self.get_obj() - evt = Mock() - pkc.handle_event(evt) - mock_HandleEvent.assert_called_with(pkc, evt) - mock_handle_event.assert_called_with(pkc, evt) + return TestXMLCfgCreator.get_obj(self, name=name) @patch("shutil.rmtree") @patch("tempfile.mkdtemp") @@ -90,57 +80,6 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): self.assertRaises(CfgCreationError, pkc._gen_keypair, metadata) mock_rmtree.assert_called_with(datastore) - def test_get_specificity(self): - pkc = self.get_obj() - pkc.XMLMatch = Mock() - - metadata = Mock() - - def reset(): - pkc.XMLMatch.reset_mock() - metadata.group_in_category.reset_mock() - - Bcfg2.Options.setup.sshkeys_category = None - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(host=metadata.hostname)) - - Bcfg2.Options.setup.sshkeys_category = "foo" - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(group=metadata.group_in_category.return_value, - prio=50)) - metadata.group_in_category.assert_called_with("foo") - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey", - perhost="true") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(host=metadata.hostname)) - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey", - category="bar") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(group=metadata.group_in_category.return_value, - prio=50)) - metadata.group_in_category.assert_called_with("bar") - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey", - prio="10") - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(group=metadata.group_in_category.return_value, - prio=10)) - metadata.group_in_category.assert_called_with("foo") - - reset() - pkc.XMLMatch.return_value = lxml.etree.Element("PrivateKey") - metadata.group_in_category.return_value = '' - self.assertItemsEqual(pkc.get_specificity(metadata), - dict(host=metadata.hostname)) - metadata.group_in_category.assert_called_with("foo") - @patch("shutil.rmtree") @patch("%s.open" % builtins) def test_create_data(self, mock_open, mock_rmtree): @@ -153,7 +92,7 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): # the get_specificity() return value is being used # appropriately, we put some dummy data in it and test for # that data - pkc.get_specificity.side_effect = lambda m, s: dict(group="foo") + pkc.get_specificity.side_effect = lambda m: dict(group="foo") pkc._gen_keypair = Mock() privkey = os.path.join(datastore, "privkey") pkc._gen_keypair.return_value = privkey @@ -179,67 +118,32 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): mock_open.return_value.read.side_effect = open_read_rv reset() - passphrase = "Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.CfgPrivateKeyCreator.passphrase" - - @patch(passphrase, None) - def inner(): - self.assertEqual(pkc.create_data(entry, metadata), "privatekey") - pkc.XMLMatch.assert_called_with(metadata) - pkc.get_specificity.assert_called_with(metadata, - pkc.XMLMatch.return_value) - pkc._gen_keypair.assert_called_with(metadata, - pkc.XMLMatch.return_value) - self.assertItemsEqual(mock_open.call_args_list, - [call(privkey + ".pub"), call(privkey)]) - pkc.pubkey_creator.get_filename.assert_called_with(group="foo") - pkc.pubkey_creator.write_data.assert_called_with( - "ssh-rsa publickey pubkey.filename\n", group="foo") - pkc.write_data.assert_called_with("privatekey", group="foo") - mock_rmtree.assert_called_with(datastore) - - reset() - self.assertEqual(pkc.create_data(entry, metadata, return_pair=True), - ("ssh-rsa publickey pubkey.filename\n", - "privatekey")) - pkc.XMLMatch.assert_called_with(metadata) - pkc.get_specificity.assert_called_with(metadata, - pkc.XMLMatch.return_value) - pkc._gen_keypair.assert_called_with(metadata, - pkc.XMLMatch.return_value) - self.assertItemsEqual(mock_open.call_args_list, - [call(privkey + ".pub"), call(privkey)]) - pkc.pubkey_creator.get_filename.assert_called_with(group="foo") - pkc.pubkey_creator.write_data.assert_called_with( - "ssh-rsa publickey pubkey.filename\n", - group="foo") - pkc.write_data.assert_called_with("privatekey", group="foo") - mock_rmtree.assert_called_with(datastore) - - inner() - - if HAS_CRYPTO: - @patch(passphrase, "foo") - @patch("Bcfg2.Server.Encryption.ssl_encrypt") - def inner2(mock_ssl_encrypt): - reset() - mock_ssl_encrypt.return_value = "encryptedprivatekey" - Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.HAS_CRYPTO = True - self.assertEqual(pkc.create_data(entry, metadata), - "encryptedprivatekey") - pkc.XMLMatch.assert_called_with(metadata) - pkc.get_specificity.assert_called_with( - metadata, - pkc.XMLMatch.return_value) - pkc._gen_keypair.assert_called_with(metadata, - pkc.XMLMatch.return_value) - self.assertItemsEqual(mock_open.call_args_list, - [call(privkey + ".pub"), call(privkey)]) - pkc.pubkey_creator.get_filename.assert_called_with(group="foo") - pkc.pubkey_creator.write_data.assert_called_with( - "ssh-rsa publickey pubkey.filename\n", group="foo") - pkc.write_data.assert_called_with("encryptedprivatekey", - group="foo", ext=".crypt") - mock_ssl_encrypt.assert_called_with("privatekey", "foo") - mock_rmtree.assert_called_with(datastore) - - inner2() + self.assertEqual(pkc.create_data(entry, metadata), "privatekey") + pkc.XMLMatch.assert_called_with(metadata) + pkc.get_specificity.assert_called_with(metadata) + pkc._gen_keypair.assert_called_with(metadata, + pkc.XMLMatch.return_value) + self.assertItemsEqual(mock_open.call_args_list, + [call(privkey + ".pub"), call(privkey)]) + pkc.pubkey_creator.get_filename.assert_called_with(group="foo") + pkc.pubkey_creator.write_data.assert_called_with( + "ssh-rsa publickey pubkey.filename\n", group="foo") + pkc.write_data.assert_called_with("privatekey", group="foo") + mock_rmtree.assert_called_with(datastore) + + reset() + self.assertEqual(pkc.create_data(entry, metadata, return_pair=True), + ("ssh-rsa publickey pubkey.filename\n", + "privatekey")) + pkc.XMLMatch.assert_called_with(metadata) + pkc.get_specificity.assert_called_with(metadata) + pkc._gen_keypair.assert_called_with(metadata, + pkc.XMLMatch.return_value) + self.assertItemsEqual(mock_open.call_args_list, + [call(privkey + ".pub"), call(privkey)]) + pkc.pubkey_creator.get_filename.assert_called_with(group="foo") + pkc.pubkey_creator.write_data.assert_called_with( + "ssh-rsa publickey pubkey.filename\n", + group="foo") + pkc.write_data.assert_called_with("privatekey", group="foo") + mock_rmtree.assert_called_with(datastore) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py index 72be50299..307461918 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py @@ -3,7 +3,7 @@ import sys import errno import lxml.etree import Bcfg2.Options -from Bcfg2.Compat import walk_packages +from Bcfg2.Compat import walk_packages, ConfigParser from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Cfg import * from Bcfg2.Server.Plugin import PluginExecutionError, Specificity @@ -19,7 +19,7 @@ while path != "/": path = os.path.dirname(path) from common import * from TestPlugin import TestSpecificData, TestEntrySet, TestGroupSpool, \ - TestPullTarget + TestPullTarget, TestStructFile class TestCfgBaseFileMatcher(TestSpecificData): @@ -172,10 +172,12 @@ class TestCfgVerifier(TestCfgBaseFileMatcher): class TestCfgCreator(TestCfgBaseFileMatcher): test_obj = CfgCreator path = "/foo/bar/test.txt" + should_monitor = False def setUp(self): TestCfgBaseFileMatcher.setUp(self) set_setup_default("filemonitor", MagicMock()) + set_setup_default("cfg_passphrase", None) def get_obj(self, name=None): if name is None: @@ -245,6 +247,75 @@ class TestCfgCreator(TestCfgBaseFileMatcher): self.assertRaises(CfgCreationError, cc.write_data, data) +class TestXMLCfgCreator(TestCfgCreator, TestStructFile): + test_obj = XMLCfgCreator + + def setUp(self): + TestCfgCreator.setUp(self) + TestStructFile.setUp(self) + + @patch("Bcfg2.Server.Plugins.Cfg.CfgCreator.handle_event") + @patch("Bcfg2.Server.Plugin.helpers.StructFile.HandleEvent") + def test_handle_event(self, mock_HandleEvent, mock_handle_event): + cc = self.get_obj() + evt = Mock() + cc.handle_event(evt) + mock_HandleEvent.assert_called_with(cc, evt) + mock_handle_event.assert_called_with(cc, evt) + + def test_get_specificity(self): + cc = self.get_obj() + metadata = Mock() + + def reset(): + metadata.group_in_category.reset_mock() + + category = "%s.%s.category" % (self.test_obj.__module__, + self.test_obj.__name__) + @patch(category, None) + def inner(): + cc.xdata = lxml.etree.Element("PrivateKey") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(host=metadata.hostname)) + inner() + + @patch(category, "foo") + def inner2(): + cc.xdata = lxml.etree.Element("PrivateKey") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(group=metadata.group_in_category.return_value, + prio=50)) + metadata.group_in_category.assert_called_with("foo") + + reset() + cc.xdata = lxml.etree.Element("PrivateKey", perhost="true") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(host=metadata.hostname)) + + reset() + cc.xdata = lxml.etree.Element("PrivateKey", category="bar") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(group=metadata.group_in_category.return_value, + prio=50)) + metadata.group_in_category.assert_called_with("bar") + + reset() + cc.xdata = lxml.etree.Element("PrivateKey", prio="10") + self.assertItemsEqual(cc.get_specificity(metadata), + dict(group=metadata.group_in_category.return_value, + prio=10)) + metadata.group_in_category.assert_called_with("foo") + + reset() + cc.xdata = lxml.etree.Element("PrivateKey") + metadata.group_in_category.return_value = '' + self.assertItemsEqual(cc.get_specificity(metadata), + dict(host=metadata.hostname)) + metadata.group_in_category.assert_called_with("foo") + + inner2() + + class TestCfgDefaultInfo(TestCfgInfo): test_obj = CfgDefaultInfo @@ -276,6 +347,7 @@ class TestCfgEntrySet(TestEntrySet): def setUp(self): TestEntrySet.setUp(self) set_setup_default("cfg_validation", False) + set_setup_default("cfg_handlers", []) def test__init(self): pass @@ -283,14 +355,14 @@ class TestCfgEntrySet(TestEntrySet): def test_handle_event(self): eset = self.get_obj() eset.entry_init = Mock() - eset._handlers = [Mock(), Mock(), Mock()] - for hdlr in eset._handlers: + Bcfg2.Options.setup.cfg_handlers = [Mock(), Mock(), Mock()] + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.__name__ = "handler" eset.entries = dict() def reset(): eset.entry_init.reset_mock() - for hdlr in eset._handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.reset_mock() # test that a bogus deleted event is discarded @@ -300,18 +372,19 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in eset._handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) # test creation of a new file for action in ["exists", "created", "changed"]: + print("Testing handling of %s events" % action) evt = Mock() evt.code2str.return_value = action evt.filename = os.path.join(datastore, "test.txt") # test with no handler that handles - for hdlr in eset._handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.handles.return_value = False hdlr.ignore.return_value = False @@ -319,16 +392,16 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in eset._handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.handles.assert_called_with(evt, basename=eset.path) hdlr.ignore.assert_called_with(evt, basename=eset.path) # test with a handler that handles the entry reset() - eset._handlers[-1].handles.return_value = True + Bcfg2.Options.setup.cfg_handlers[-1].handles.return_value = True eset.handle_event(evt) - eset.entry_init.assert_called_with(evt, eset._handlers[-1]) - for hdlr in eset._handlers: + eset.entry_init.assert_called_with(evt, Bcfg2.Options.setup.cfg_handlers[-1]) + for hdlr in Bcfg2.Options.setup.cfg_handlers: hdlr.handles.assert_called_with(evt, basename=eset.path) if not hdlr.return_value: hdlr.ignore.assert_called_with(evt, basename=eset.path) @@ -336,14 +409,14 @@ class TestCfgEntrySet(TestEntrySet): # test with a handler that ignores the entry before one # that handles it reset() - eset._handlers[0].ignore.return_value = True + Bcfg2.Options.setup.cfg_handlers[0].ignore.return_value = True eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - eset._handlers[0].handles.assert_called_with(evt, - basename=eset.path) - eset._handlers[0].ignore.assert_called_with(evt, - basename=eset.path) - for hdlr in eset._handlers[1:]: + Bcfg2.Options.setup.cfg_handlers[0].handles.assert_called_with( + evt, basename=eset.path) + Bcfg2.Options.setup.cfg_handlers[0].ignore.assert_called_with( + evt, basename=eset.path) + for hdlr in Bcfg2.Options.setup.cfg_handlers[1:]: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) @@ -355,7 +428,7 @@ class TestCfgEntrySet(TestEntrySet): eset.entries[evt.filename] = Mock() eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in eset._handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) eset.entries[evt.filename].handle_event.assert_called_with(evt) @@ -365,7 +438,7 @@ class TestCfgEntrySet(TestEntrySet): evt.code2str.return_value = "deleted" eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in eset._handlers: + for hdlr in Bcfg2.Options.setup.cfg_handlers: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) self.assertItemsEqual(eset.entries, dict()) @@ -730,6 +803,11 @@ class TestCfgEntrySet(TestEntrySet): class TestCfg(TestGroupSpool, TestPullTarget): test_obj = Cfg + def setUp(self): + TestGroupSpool.setUp(self) + TestPullTarget.setUp(self) + set_setup_default("cfg_handlers", []) + def get_obj(self, core=None): if core is None: core = Mock() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py index 7be3d8e84..9b4a6af88 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestDefaults.py @@ -1,5 +1,6 @@ import os import sys +import copy import lxml.etree from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Defaults import * @@ -62,3 +63,31 @@ class TestDefaults(TestRules, TestGoalValidator): def test__regex_enabled(self): r = self.get_obj() self.assertTrue(r._regex_enabled) + + def _do_test(self, name, groups=None): + if groups is None: + groups = [] + d = self.get_obj() + metadata = Mock(groups=groups) + config = lxml.etree.Element("Configuration") + struct = lxml.etree.SubElement(config, "Bundle", name=name) + entry = copy.deepcopy(self.abstract[name]) + struct.append(entry) + d.validate_goals(metadata, config) + self.assertXMLEqual(entry, self.concrete[name]) + + def _do_test_failure(self, name, groups=None, handles=None): + if groups is None: + groups = [] + d = self.get_obj() + metadata = Mock(groups=groups) + config = lxml.etree.Element("Configuration") + struct = lxml.etree.SubElement(config, "Bundle", name=name) + orig = copy.deepcopy(self.abstract[name]) + entry = copy.deepcopy(self.abstract[name]) + struct.append(entry) + d.validate_goals(metadata, config) + self.assertXMLEqual(entry, orig) + + def test_regex(self): + self._do_test('regex') diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index 274b5e302..290edb83a 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -91,7 +91,7 @@ def get_groups_test_tree(): </Groups>''').getroottree() -def get_metadata_object(core=None, watch_clients=False): +def get_metadata_object(core=None): if core is None: core = Mock() core.metadata_cache = MagicMock() @@ -101,7 +101,7 @@ def get_metadata_object(core=None, watch_clients=False): @patchIf(not isinstance(lxml.etree.Element, Mock), "lxml.etree.Element", Mock()) def inner(): - return Metadata(core, datastore, watch_clients=watch_clients) + return Metadata(core, datastore) return inner() @@ -110,118 +110,117 @@ class TestMetadataDB(DBModelTestCase): models = [MetadataClientModel] -if HAS_DJANGO or can_skip: - class TestClientVersions(TestDatabaseBacked): - test_clients = dict(client1="1.2.0", - client2="1.2.2", - client3="1.3.0pre1", - client4="1.1.0", - client5=None, - client6=None) - - @skipUnless(HAS_DJANGO, "Django not found") - def setUp(self): - TestDatabaseBacked.setUp(self) - self.test_obj = ClientVersions - syncdb(TestMetadataDB) - for client, version in self.test_clients.items(): - MetadataClientModel(hostname=client, version=version).save() - - def test__contains(self): - v = self.get_obj() - self.assertIn("client1", v) - self.assertIn("client5", v) - self.assertNotIn("client__contains", v) - - def test_keys(self): - v = self.get_obj() - self.assertItemsEqual(self.test_clients.keys(), v.keys()) - - def test__setitem(self): - v = self.get_obj() - - # test setting version of existing client - v["client1"] = "1.2.3" - self.assertIn("client1", v) - self.assertEqual(v['client1'], "1.2.3") - client = MetadataClientModel.objects.get(hostname="client1") - self.assertEqual(client.version, "1.2.3") - - # test adding new client - new = "client__setitem" - v[new] = "1.3.0" - self.assertIn(new, v) - self.assertEqual(v[new], "1.3.0") - client = MetadataClientModel.objects.get(hostname=new) - self.assertEqual(client.version, "1.3.0") - - # test adding new client with no version - new2 = "client__setitem_2" - v[new2] = None - self.assertIn(new2, v) - self.assertEqual(v[new2], None) - client = MetadataClientModel.objects.get(hostname=new2) - self.assertEqual(client.version, None) - - def test__getitem(self): - v = self.get_obj() - - # test getting existing client - self.assertEqual(v['client2'], "1.2.2") - self.assertIsNone(v['client5']) - - # test exception on nonexistent client - expected = KeyError - try: - v['clients__getitem'] - except expected: - pass - except: - err = sys.exc_info()[1] - self.assertFalse(True, "%s raised instead of %s" % - (err.__class__.__name__, - expected.__class__.__name__)) - else: - self.assertFalse(True, - "%s not raised" % expected.__class__.__name__) +class TestClientVersions(TestDatabaseBacked): + test_clients = dict(client1="1.2.0", + client2="1.2.2", + client3="1.3.0pre1", + client4="1.1.0", + client5=None, + client6=None) - def test__len(self): - v = self.get_obj() - self.assertEqual(len(v), MetadataClientModel.objects.count()) + @skipUnless(HAS_DJANGO, "Django not found") + def setUp(self): + TestDatabaseBacked.setUp(self) + self.test_obj = ClientVersions + syncdb(TestMetadataDB) + for client, version in self.test_clients.items(): + MetadataClientModel(hostname=client, version=version).save() + + def test__contains(self): + v = self.get_obj() + self.assertIn("client1", v) + self.assertIn("client5", v) + self.assertNotIn("client__contains", v) + + def test_keys(self): + v = self.get_obj() + self.assertItemsEqual(self.test_clients.keys(), v.keys()) + + def test__setitem(self): + v = self.get_obj() + + # test setting version of existing client + v["client1"] = "1.2.3" + self.assertIn("client1", v) + self.assertEqual(v['client1'], "1.2.3") + client = MetadataClientModel.objects.get(hostname="client1") + self.assertEqual(client.version, "1.2.3") + + # test adding new client + new = "client__setitem" + v[new] = "1.3.0" + self.assertIn(new, v) + self.assertEqual(v[new], "1.3.0") + client = MetadataClientModel.objects.get(hostname=new) + self.assertEqual(client.version, "1.3.0") + + # test adding new client with no version + new2 = "client__setitem_2" + v[new2] = None + self.assertIn(new2, v) + self.assertEqual(v[new2], None) + client = MetadataClientModel.objects.get(hostname=new2) + self.assertEqual(client.version, None) + + def test__getitem(self): + v = self.get_obj() + + # test getting existing client + self.assertEqual(v['client2'], "1.2.2") + self.assertIsNone(v['client5']) + + # test exception on nonexistent client + expected = KeyError + try: + v['clients__getitem'] + except expected: + pass + except: + err = sys.exc_info()[1] + self.assertFalse(True, "%s raised instead of %s" % + (err.__class__.__name__, + expected.__class__.__name__)) + else: + self.assertFalse(True, + "%s not raised" % expected.__class__.__name__) + + def test__len(self): + v = self.get_obj() + self.assertEqual(len(v), MetadataClientModel.objects.count()) - def test__iter(self): - v = self.get_obj() - self.assertItemsEqual([h for h in iter(v)], v.keys()) + def test__iter(self): + v = self.get_obj() + self.assertItemsEqual([h for h in iter(v)], v.keys()) - def test__delitem(self): - v = self.get_obj() + def test__delitem(self): + v = self.get_obj() - # test adding new client - new = "client__delitem" - v[new] = "1.3.0" + # test adding new client + new = "client__delitem" + v[new] = "1.3.0" - del v[new] - self.assertIn(new, v) - self.assertIsNone(v[new]) + del v[new] + self.assertIn(new, v) + self.assertIsNone(v[new]) class TestXMLMetadataConfig(TestXMLFileBacked): test_obj = XMLMetadataConfig path = os.path.join(datastore, 'Metadata', 'clients.xml') - def get_obj(self, basefile="clients.xml", core=None, watch_clients=False): - self.metadata = get_metadata_object(core=core, - watch_clients=watch_clients) + def get_obj(self, basefile="clients.xml", core=None): + self.metadata = get_metadata_object(core=core) @patchIf(not isinstance(lxml.etree.Element, Mock), "lxml.etree.Element", Mock()) def inner(): - return XMLMetadataConfig(self.metadata, watch_clients, basefile) + return XMLMetadataConfig(self.metadata, basefile) return inner() @patch("Bcfg2.Server.FileMonitor.get_fam", Mock()) def test__init(self): xmc = self.get_obj() - self.assertFalse(xmc.fam.AddMonitor.called) + self.assertNotIn(call(xmc.basefile), + xmc.fam.AddMonitor.call_args_list) def test_xdata(self): config = self.get_obj() @@ -273,12 +272,6 @@ class TestXMLMetadataConfig(TestXMLFileBacked): config.extras = [] config.add_monitor(fpath) - self.assertFalse(config.fam.AddMonitor.called) - self.assertEqual(config.extras, [fpath]) - - config = self.get_obj(watch_clients=True) - config.fam = Mock() - config.add_monitor(fpath) config.fam.AddMonitor.assert_called_with(fpath, config.metadata) self.assertItemsEqual(config.extras, [fpath]) @@ -487,8 +480,8 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): Bcfg2.Options.setup.metadata_db = False Bcfg2.Options.setup.authentication = "cert+password" - def get_obj(self, core=None, watch_clients=False): - return get_metadata_object(core=core, watch_clients=watch_clients) + def get_obj(self, core=None): + return get_metadata_object(core=core) @skipUnless(HAS_DJANGO, "Django not found") def test__use_db(self): @@ -510,19 +503,8 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): @patch("Bcfg2.Server.FileMonitor.get_fam") def test__init(self, mock_get_fam): - # test with watch_clients=False core = MagicMock() metadata = self.get_obj(core=core) - self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Plugin) - self.assertIsInstance(metadata, Bcfg2.Server.Plugin.Metadata) - self.assertIsInstance(metadata, Bcfg2.Server.Plugin.ClientRunHooks) - self.assertIsInstance(metadata.clients_xml, XMLMetadataConfig) - self.assertIsInstance(metadata.groups_xml, XMLMetadataConfig) - self.assertIsInstance(metadata.query, MetadataQuery) - self.assertEqual(metadata.states, dict()) - - # test with watch_clients=True - metadata = self.get_obj(core=core, watch_clients=True) self.assertEqual(len(metadata.states), 2) mock_get_fam.return_value.AddMonitor.assert_any_call( os.path.join(metadata.data, "groups.xml"), @@ -536,7 +518,7 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): fam.AddMonitor = Mock(side_effect=IOError) mock_get_fam.return_value = fam self.assertRaises(Bcfg2.Server.Plugin.PluginInitError, - self.get_obj, core=core, watch_clients=True) + self.get_obj, core=core) @patch('os.makedirs', Mock()) @patch('%s.open' % builtins) @@ -848,21 +830,18 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): self.assertEqual(metadata.groups['group4'].category, 'category1') self.assertEqual(metadata.default, "group1") - all_groups = [] - negated_groups = [] + all_groups = set() + negated_groups = set() for group in get_groups_test_tree().xpath("//Groups/Client//*") + \ get_groups_test_tree().xpath("//Groups/Group//*"): if group.tag == 'Group' and not group.getchildren(): if group.get("negate", "false").lower() == 'true': - negated_groups.append(group.get("name")) + negated_groups.add(group.get("name")) else: - all_groups.append(group.get("name")) - self.assertItemsEqual([g.name - for g in metadata.group_membership.values()], - all_groups) - self.assertItemsEqual([g.name - for g in metadata.negated_groups.values()], - negated_groups) + all_groups.add(group.get("name")) + self.assertItemsEqual(metadata.ordered_groups, all_groups) + self.assertItemsEqual(metadata.group_membership.keys(), all_groups) + self.assertItemsEqual(metadata.negated_groups.keys(), negated_groups) @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) def test_set_profile(self): @@ -1296,7 +1275,7 @@ class TestMetadataBase(TestMetadata): @patch('Bcfg2.Server.FileMonitor.get_fam') def test__init(self, mock_get_fam, mock_exists): mock_exists.return_value = False - metadata = self.get_obj(watch_clients=True) + metadata = self.get_obj() self.assertIsInstance(metadata, Bcfg2.Server.Plugin.DatabaseBacked) mock_get_fam.return_value.AddMonitor.assert_called_with( os.path.join(metadata.data, "groups.xml"), @@ -1304,7 +1283,7 @@ class TestMetadataBase(TestMetadata): mock_exists.return_value = True mock_get_fam.reset_mock() - metadata = self.get_obj(watch_clients=True) + metadata = self.get_obj() mock_get_fam.return_value.AddMonitor.assert_any_call( os.path.join(metadata.data, "groups.xml"), metadata) @@ -1374,12 +1353,7 @@ class TestMetadataBase(TestMetadata): class TestMetadata_NoClientsXML(TestMetadataBase): """ test Metadata without a clients.xml. we have to disable or override tests that rely on client options """ - # only run these tests if it's possible to skip tests or if we - # have django. otherwise they'll all get run because our fake - # skipping decorators for python < 2.7 won't work when they - # decorate setUp() - if can_skip or HAS_DJANGO: - __test__ = True + __test__ = True def load_groups_data(self, metadata=None, xdata=None): if metadata is None: @@ -1543,12 +1517,7 @@ class TestMetadata_NoClientsXML(TestMetadataBase): class TestMetadata_ClientsXML(TestMetadataBase): """ test Metadata with a clients.xml. """ - # only run these tests if it's possible to skip tests or if we - # have django. otherwise they'll all get run because our fake - # skipping decorators for python < 2.7 won't work when they - # decorate setUp() - if can_skip or HAS_DJANGO: - __test__ = True + __test__ = True def load_clients_data(self, metadata=None, xdata=None): if metadata is None: diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index b0e6e9142..4830f9f2f 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -1,7 +1,7 @@ import os import sys -import copy -import time +import shutil +import tempfile import lxml.etree import Bcfg2.version import Bcfg2.Server @@ -36,47 +36,6 @@ test_data = dict(a=1, b=[1, 2, 3], c="test", d=dict(a=1, b=dict(a=1), c=(1, "2", 3))) -class FakeElement(lxml.etree._Element): - getroottree = Mock() - - def __init__(self, el): - self._element = el - - def __getattribute__(self, attr): - el = lxml.etree._Element.__getattribute__(self, - '__dict__')['_element'] - if attr == "getroottree": - return FakeElement.getroottree - elif attr == "_element": - return el - else: - return getattr(el, attr) - - -class StoringElement(object): - OriginalElement = copy.copy(lxml.etree.Element) - - def __init__(self): - self.element = None - self.return_value = None - - def __call__(self, *args, **kwargs): - self.element = self.OriginalElement(*args, **kwargs) - self.return_value = FakeElement(self.element) - return self.return_value - - -class StoringSubElement(object): - OriginalSubElement = copy.copy(lxml.etree.SubElement) - - def __call__(self, parent, tag, **kwargs): - try: - return self.OriginalSubElement(parent._element, tag, - **kwargs) - except AttributeError: - return self.OriginalSubElement(parent, tag, **kwargs) - - class FakeList(list): pass @@ -87,18 +46,6 @@ class TestProbesDB(DBModelTestCase): ProbesDataModel] -class TestClientProbeDataSet(Bcfg2TestCase): - def test__init(self): - ds = ClientProbeDataSet() - self.assertLessEqual(ds.timestamp, time.time()) - self.assertIsInstance(ds, dict) - self.assertNotIn("timestamp", ds) - - ds = ClientProbeDataSet(timestamp=123) - self.assertEqual(ds.timestamp, 123) - self.assertNotIn("timestamp", ds) - - class TestProbeData(Bcfg2TestCase): def test_str(self): # a value that is not valid XML, JSON, or YAML @@ -253,377 +200,162 @@ group-specific""" assert False, "Strange probe found in get_probe_data() return" -class TestProbes(TestProbing, TestConnector, TestDatabaseBacked): +class TestProbes(Bcfg2TestCase): test_obj = Probes - def get_obj(self, core=None, load_data=None): - core = MagicMock() - if load_data is None: - load_data = MagicMock() - - @patch("%s.%s.load_data" % (self.test_obj.__module__, - self.test_obj.__name__), new=load_data) - def inner(): - return TestDatabaseBacked.get_obj(self, core=core) - - return inner() - - def get_test_probedata(self): - test_xdata = lxml.etree.Element("test") - lxml.etree.SubElement(test_xdata, "test", foo="foo") - rv = dict() - rv["foo.example.com"] = ClientProbeDataSet( - timestamp=time.time()) - rv["foo.example.com"]["xml"] = \ - ProbeData(lxml.etree.tostring( - test_xdata, - xml_declaration=False).decode('UTF-8')) - rv["foo.example.com"]["text"] = \ - ProbeData("freeform text") - rv["foo.example.com"]["multiline"] = \ - ProbeData("""multiple + test_xdata = lxml.etree.Element("test") + lxml.etree.SubElement(test_xdata, "test", foo="foo") + test_xdoc = lxml.etree.tostring(test_xdata, + xml_declaration=False).decode('UTF-8') + + data = dict() + data['xml'] = "group:group\n" + test_xdoc + data['text'] = "freeform text" + data['multiline'] = """multiple lines of freeform text -""") - rv["bar.example.com"] = ClientProbeDataSet( - timestamp=time.time()) - rv["bar.example.com"]["empty"] = \ - ProbeData("") - if HAS_JSON: - rv["bar.example.com"]["json"] = \ - ProbeData(json.dumps(test_data)) - if HAS_YAML: - rv["bar.example.com"]["yaml"] = \ - ProbeData(yaml.dump(test_data)) - return rv - - def get_test_cgroups(self): - return {"foo.example.com": ["group", "group with spaces", - "group-with-dashes"], - "bar.example.com": []} +group:group-with-dashes +group: group:with:colons +""" + data['empty'] = '' + data['almost_empty'] = 'group: other_group' + if HAS_JSON: + data['json'] = json.dumps(test_data) + if HAS_YAML: + data['yaml'] = yaml.dump(test_data) + + def setUp(self): + Bcfg2TestCase.setUp(self) + set_setup_default("probes_db") + self.datastore = None + Bcfg2.Server.Cache.expire("Probes") + + def tearDown(self): + if self.datastore is not None: + shutil.rmtree(self.datastore) + self.datastore = None + + def get_obj(self, core=None): + if core is None: + core = Mock() + if Bcfg2.Options.setup.probes_db: + @patch("os.makedirs", Mock()) + def inner(): + return self.test_obj(core, datastore) + return inner() + else: + # actually use a real datastore so we can read and write + # probed.xml + if self.datastore is None: + self.datastore = tempfile.mkdtemp() + return self.test_obj(core, self.datastore) + + def test_GetProbes(self): + p = self.get_obj() + p.probes = Mock() + metadata = Mock() + p.GetProbes(metadata) + p.probes.get_probe_data.assert_called_with(metadata) - def test__init(self): - mock_load_data = Mock() - probes = self.get_obj(load_data=mock_load_data) - mock_load_data.assert_any_call() - self.assertEqual(probes.probedata, - ClientProbeDataSet()) - self.assertEqual(probes.cgroups, dict()) + def additionalDataEqual(self, actual, expected): + self.assertItemsEqual( + dict([(k, str(d)) for k, d in actual.items()]), + expected) - def test_write_data_xml(self): - Bcfg2.Options.setup.probes_db = False - probes = self.get_obj() - probes._write_data_db = Mock() - probes._write_data_xml = Mock() - probes.write_data("test") - probes._write_data_xml.assert_called_with("test") - self.assertFalse(probes._write_data_db.called) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - def test_write_data_db(self): - Bcfg2.Options.setup.probes_db = True - probes = self.get_obj() - probes._write_data_db = Mock() - probes._write_data_xml = Mock() - probes.write_data("test") - probes._write_data_db.assert_called_with("test") - self.assertFalse(probes._write_data_xml.called) - - def test__write_data_xml(self): - Bcfg2.Options.setup.probes_db = False - probes = self.get_obj() - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - - @patch("lxml.etree.Element") - @patch("lxml.etree.SubElement", StoringSubElement()) - def inner(mock_Element): - mock_Element.side_effect = StoringElement() - probes._write_data_xml(None) - - top = mock_Element.side_effect.return_value - write = top.getroottree.return_value.write - self.assertEqual(write.call_args[0][0], - os.path.join(datastore, probes.name, - "probed.xml")) - - data = top._element - foodata = data.find("Client[@name='foo.example.com']") - self.assertIsNotNone(foodata) - self.assertIsNotNone(foodata.get("timestamp")) - self.assertEqual(len(foodata.findall("Probe")), - len(probes.probedata['foo.example.com'])) - self.assertEqual(len(foodata.findall("Group")), - len(probes.cgroups['foo.example.com'])) - xml = foodata.find("Probe[@name='xml']") - self.assertIsNotNone(xml) - self.assertIsNotNone(xml.get("value")) - xdata = lxml.etree.XML(xml.get("value")) - self.assertIsNotNone(xdata) - self.assertIsNotNone(xdata.find("test")) - self.assertEqual(xdata.find("test").get("foo"), "foo") - text = foodata.find("Probe[@name='text']") - self.assertIsNotNone(text) - self.assertIsNotNone(text.get("value")) - multiline = foodata.find("Probe[@name='multiline']") - self.assertIsNotNone(multiline) - self.assertIsNotNone(multiline.get("value")) - self.assertGreater(len(multiline.get("value").splitlines()), 1) - - bardata = data.find("Client[@name='bar.example.com']") - self.assertIsNotNone(bardata) - self.assertIsNotNone(bardata.get("timestamp")) - self.assertEqual(len(bardata.findall("Probe")), - len(probes.probedata['bar.example.com'])) - self.assertEqual(len(bardata.findall("Group")), - len(probes.cgroups['bar.example.com'])) - empty = bardata.find("Probe[@name='empty']") - self.assertIsNotNone(empty) - self.assertIsNotNone(empty.get("value")) - self.assertEqual(empty.get("value"), "") - if HAS_JSON: - jdata = bardata.find("Probe[@name='json']") - self.assertIsNotNone(jdata) - self.assertIsNotNone(jdata.get("value")) - self.assertItemsEqual(test_data, - json.loads(jdata.get("value"))) - if HAS_YAML: - ydata = bardata.find("Probe[@name='yaml']") - self.assertIsNotNone(ydata) - self.assertIsNotNone(ydata.get("value")) - self.assertItemsEqual(test_data, - yaml.load(ydata.get("value"))) - - inner() - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - def test__write_data_db(self): - syncdb(TestProbesDB) - Bcfg2.Options.setup.probes_db = True - probes = self.get_obj() - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - - for cname in ["foo.example.com", "bar.example.com"]: - client = Mock() - client.hostname = cname - probes._write_data_db(client) - - pdata = ProbesDataModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pdata), len(probes.probedata[cname])) - - for probe in pdata: - self.assertEqual(probe.hostname, client.hostname) - self.assertIsNotNone(probe.data) - if probe.probe == "xml": - xdata = lxml.etree.XML(probe.data) - self.assertIsNotNone(xdata) - self.assertIsNotNone(xdata.find("test")) - self.assertEqual(xdata.find("test").get("foo"), "foo") - elif probe.probe == "text": - pass - elif probe.probe == "multiline": - self.assertGreater(len(probe.data.splitlines()), 1) - elif probe.probe == "empty": - self.assertEqual(probe.data, "") - elif probe.probe == "yaml": - self.assertItemsEqual(test_data, yaml.load(probe.data)) - elif probe.probe == "json": - self.assertItemsEqual(test_data, json.loads(probe.data)) - else: - assert False, "Strange probe found in _write_data_db data" - - pgroups = ProbesGroupsModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pgroups), len(probes.cgroups[cname])) - - # test that old probe data is removed properly - cname = 'foo.example.com' - del probes.probedata[cname]['text'] - probes.cgroups[cname].pop() - client = Mock() - client.hostname = cname - probes._write_data_db(client) - - pdata = ProbesDataModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pdata), len(probes.probedata[cname])) - pgroups = ProbesGroupsModel.objects.filter(hostname=cname).all() - self.assertEqual(len(pgroups), len(probes.cgroups[cname])) - - def test_load_data_xml(self): + def test_probes_xml(self): + """ Set and retrieve probe data with database disabled """ Bcfg2.Options.setup.probes_db = False - probes = self.get_obj() - probes._load_data_db = Mock() - probes._load_data_xml = Mock() - probes.load_data() - probes._load_data_xml.assert_any_call() - self.assertFalse(probes._load_data_db.called) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - def test_load_data_db(self): - Bcfg2.Options.setup.probes_db = True - probes = self.get_obj() - probes._load_data_db = Mock() - probes._load_data_xml = Mock() - probes.load_data() - probes._load_data_db.assert_any_call(client=None) - self.assertFalse(probes._load_data_xml.called) - - @patch("lxml.etree.parse") - def test__load_data_xml(self, mock_parse): - Bcfg2.Options.setup.probes_db = False - probes = self.get_obj() - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - - # to get the value for lxml.etree.parse to parse, we call - # _write_data_xml, mock the lxml.etree._ElementTree.write() - # call, and grab the data that gets "written" to probed.xml - @patch("lxml.etree.Element") - @patch("lxml.etree.SubElement", StoringSubElement()) - def inner(mock_Element): - mock_Element.side_effect = StoringElement() - probes._write_data_xml(None) - top = mock_Element.side_effect.return_value - return top._element - - xdata = inner() - mock_parse.return_value = xdata.getroottree() - probes.probedata = dict() - probes.cgroups = dict() - - probes._load_data_xml() - mock_parse.assert_called_with(os.path.join(datastore, probes.name, - 'probed.xml'), - parser=Bcfg2.Server.XMLParser) - self.assertItemsEqual(probes.probedata, self.get_test_probedata()) - self.assertItemsEqual(probes.cgroups, self.get_test_cgroups()) - - @skipUnless(HAS_DJANGO, "Django not found, skipping") - def test__load_data_db(self): - syncdb(TestProbesDB) - Bcfg2.Options.setup.probes_db = True - probes = self.get_obj() - probes.probedata = self.get_test_probedata() - probes.cgroups = self.get_test_cgroups() - for cname in probes.probedata.keys(): - client = Mock() - client.hostname = cname - probes._write_data_db(client) - - probes.probedata = dict() - probes.cgroups = dict() - probes._load_data_db() - self.assertItemsEqual(probes.probedata, self.get_test_probedata()) - # the db backend does not store groups at all if a client has - # no groups set, so we can't just use assertItemsEqual here, - # because loading saved data may _not_ result in the original - # data if some clients had no groups set. - test_cgroups = self.get_test_cgroups() - for cname, groups in test_cgroups.items(): - if cname in probes.cgroups: - self.assertEqual(groups, probes.cgroups[cname]) - else: - self.assertEqual(groups, []) + self._perform_tests() - @patch("Bcfg2.Server.Plugins.Probes.ProbeSet.get_probe_data") - def test_GetProbes(self, mock_get_probe_data): - probes = self.get_obj() - metadata = Mock() - probes.GetProbes(metadata) - mock_get_probe_data.assert_called_with(metadata) - - def test_ReceiveData(self): - # we use a simple (read: bogus) datalist here to make this - # easy to test - datalist = ["a", "b", "c"] - - probes = self.get_obj() - probes.write_data = Mock() - probes.ReceiveDataItem = Mock() - probes.core.metadata_cache_mode = 'off' - client = Mock() - client.hostname = "foo.example.com" - probes.ReceiveData(client, datalist) - - cgroups = [] - cprobedata = ClientProbeDataSet() - self.assertItemsEqual(probes.ReceiveDataItem.call_args_list, - [call(client, "a", cgroups, cprobedata), - call(client, "b", cgroups, cprobedata), - call(client, "c", cgroups, cprobedata)]) - probes.write_data.assert_called_with(client) - self.assertFalse(probes.core.metadata_cache.expire.called) - - # change the datalist, ensure that the cache is cleared - probes.cgroups[client.hostname] = datalist - probes.core.metadata_cache_mode = 'aggressive' - probes.ReceiveData(client, ['a', 'b', 'd']) - - probes.write_data.assert_called_with(client) - probes.core.metadata_cache.expire.assert_called_with(client.hostname) - - def test_ReceiveDataItem(self): - probes = self.get_obj() - for cname, cdata in self.get_test_probedata().items(): - client = Mock() - client.hostname = cname - cgroups = [] - cprobedata = ClientProbeDataSet() - for pname, pdata in cdata.items(): - dataitem = lxml.etree.Element("Probe", name=pname) - if pname == "text": - # add some groups to the plaintext test to test - # group parsing - data = [pdata] - for group in self.get_test_cgroups()[cname]: - data.append("group:%s" % group) - dataitem.text = "\n".join(data) - else: - dataitem.text = str(pdata) - - probes.ReceiveDataItem(client, dataitem, cgroups, cprobedata) - - probes.cgroups[client.hostname] = cgroups - probes.probedata[client.hostname] = cprobedata - self.assertIn(client.hostname, probes.probedata) - self.assertIn(pname, probes.probedata[cname]) - self.assertEqual(pdata, probes.probedata[cname][pname]) - self.assertIn(client.hostname, probes.cgroups) - self.assertEqual(probes.cgroups[cname], - self.get_test_cgroups()[cname]) - - def test_get_additional_groups(self): - TestConnector.test_get_additional_groups(self) - - probes = self.get_obj() - test_cgroups = self.get_test_cgroups() - probes.cgroups = self.get_test_cgroups() - for cname in test_cgroups.keys(): - metadata = Mock() - metadata.hostname = cname - self.assertEqual(test_cgroups[cname], - probes.get_additional_groups(metadata)) - # test a non-existent client - metadata = Mock() - metadata.hostname = "nonexistent" - self.assertEqual(probes.get_additional_groups(metadata), - list()) - - def test_get_additional_data(self): - TestConnector.test_get_additional_data(self) - - probes = self.get_obj() - test_probedata = self.get_test_probedata() - probes.probedata = self.get_test_probedata() - for cname in test_probedata.keys(): - metadata = Mock() - metadata.hostname = cname - self.assertEqual(test_probedata[cname], - probes.get_additional_data(metadata)) - # test a non-existent client - metadata = Mock() - metadata.hostname = "nonexistent" - self.assertEqual(probes.get_additional_data(metadata), - ClientProbeDataSet()) + @skipUnless(HAS_DJANGO, "Django not found") + def test_probes_db(self): + """ Set and retrieve probe data with database enabled """ + Bcfg2.Options.setup.probes_db = True + self._perform_tests() + + def _perform_tests(self): + p = self.get_obj() + + # first, sanity checks + foo_md = Mock(hostname="foo.example.com") + bar_md = Mock(hostname="bar.example.com") + self.assertItemsEqual(p.get_additional_groups(foo_md), []) + self.assertItemsEqual(p.get_additional_data(foo_md), dict()) + self.assertItemsEqual(p.get_additional_groups(bar_md), []) + self.assertItemsEqual(p.get_additional_data(bar_md), dict()) + + # next, set some initial probe data + foo_datalist = [] + for key in ['xml', 'text', 'multiline']: + pdata = lxml.etree.Element("Probe", name=key) + pdata.text = self.data[key] + foo_datalist.append(pdata) + foo_addl_data = dict(xml=self.test_xdoc, + text="freeform text", + multiline="""multiple +lines +of +freeform +text""") + bar_datalist = [] + for key in ['empty', 'almost_empty', 'json', 'yaml']: + if key in self.data: + pdata = lxml.etree.Element("Probe", name=key) + pdata.text = self.data[key] + bar_datalist.append(pdata) + bar_addl_data = dict(empty="", almost_empty="") + if HAS_JSON: + bar_addl_data['json'] = self.data['json'] + if HAS_YAML: + bar_addl_data['yaml'] = self.data['yaml'] + + p.ReceiveData(foo_md, foo_datalist) + self.assertItemsEqual(p.get_additional_groups(foo_md), + ["group", "group-with-dashes", + "group:with:colons"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + + p.ReceiveData(bar_md, bar_datalist) + self.assertItemsEqual(p.get_additional_groups(foo_md), + ["group", "group-with-dashes", + "group:with:colons"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) + + # instantiate a new Probes object and clear Probes caches to + # imitate a server restart + p = self.get_obj() + Bcfg2.Server.Cache.expire("Probes") + + self.assertItemsEqual(p.get_additional_groups(foo_md), + ["group", "group-with-dashes", + "group:with:colons"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) + + # set new data (and groups) for foo + foo_datalist = [] + pdata = lxml.etree.Element("Probe", name='xml') + pdata.text = self.data['xml'] + foo_datalist.append(pdata) + foo_addl_data = dict(xml=self.test_xdoc) + + p.ReceiveData(foo_md, foo_datalist) + self.assertItemsEqual(p.get_additional_groups(foo_md), ["group"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) + + # instantiate a new Probes object and clear Probes caches to + # imitate a server restart + p = self.get_obj() + Bcfg2.Server.Cache.expire("Probes") + + self.assertItemsEqual(p.get_additional_groups(foo_md), ["group"]) + self.additionalDataEqual(p.get_additional_data(foo_md), foo_addl_data) + self.assertItemsEqual(p.get_additional_groups(bar_md), ['other_group']) + self.additionalDataEqual(p.get_additional_data(bar_md), bar_addl_data) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py index 9fa2cc4db..159dc6e66 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py @@ -89,86 +89,84 @@ class TestPropertyFile(Bcfg2TestCase): mock_copy.assert_called_with(pf) -if can_skip or HAS_JSON: - class TestJSONPropertyFile(TestFileBacked, TestPropertyFile): - test_obj = JSONPropertyFile - - @skipUnless(HAS_JSON, "JSON libraries not found, skipping") - def setUp(self): - TestFileBacked.setUp(self) - TestPropertyFile.setUp(self) - - @patch("%s.loads" % JSON) - def test_Index(self, mock_loads): - pf = self.get_obj() - pf.Index() - mock_loads.assert_called_with(pf.data) - self.assertEqual(pf.json, mock_loads.return_value) - - mock_loads.reset_mock() - mock_loads.side_effect = ValueError - self.assertRaises(PluginExecutionError, pf.Index) - mock_loads.assert_called_with(pf.data) - - @patch("%s.dump" % JSON) - @patch("%s.open" % builtins) - def test__write(self, mock_open, mock_dump): - pf = self.get_obj() - self.assertTrue(pf._write()) - mock_open.assert_called_with(pf.name, 'wb') - mock_dump.assert_called_with(pf.json, mock_open.return_value) - - @patch("%s.dumps" % JSON) - def test_validate_data(self, mock_dumps): - pf = self.get_obj() - pf.validate_data() - mock_dumps.assert_called_with(pf.json) - - mock_dumps.reset_mock() - mock_dumps.side_effect = ValueError - self.assertRaises(PluginExecutionError, pf.validate_data) - mock_dumps.assert_called_with(pf.json) - - -if can_skip or HAS_YAML: - class TestYAMLPropertyFile(TestFileBacked, TestPropertyFile): - test_obj = YAMLPropertyFile - - @skipUnless(HAS_YAML, "YAML libraries not found, skipping") - def setUp(self): - TestFileBacked.setUp(self) - TestPropertyFile.setUp(self) - - @patch("yaml.load") - def test_Index(self, mock_load): - pf = self.get_obj() - pf.Index() - mock_load.assert_called_with(pf.data) - self.assertEqual(pf.yaml, mock_load.return_value) - - mock_load.reset_mock() - mock_load.side_effect = yaml.YAMLError - self.assertRaises(PluginExecutionError, pf.Index) - mock_load.assert_called_with(pf.data) - - @patch("yaml.dump") - @patch("%s.open" % builtins) - def test__write(self, mock_open, mock_dump): - pf = self.get_obj() - self.assertTrue(pf._write()) - mock_open.assert_called_with(pf.name, 'wb') - mock_dump.assert_called_with(pf.yaml, mock_open.return_value) - - @patch("yaml.dump") - def test_validate_data(self, mock_dump): - pf = self.get_obj() - pf.validate_data() - mock_dump.assert_called_with(pf.yaml) - - mock_dump.reset_mock() - mock_dump.side_effect = yaml.YAMLError - self.assertRaises(PluginExecutionError, pf.validate_data) - mock_dump.assert_called_with(pf.yaml) +class TestJSONPropertyFile(TestFileBacked, TestPropertyFile): + test_obj = JSONPropertyFile + + @skipUnless(HAS_JSON, "JSON libraries not found, skipping") + def setUp(self): + TestFileBacked.setUp(self) + TestPropertyFile.setUp(self) + + @patch("%s.loads" % JSON) + def test_Index(self, mock_loads): + pf = self.get_obj() + pf.Index() + mock_loads.assert_called_with(pf.data) + self.assertEqual(pf.json, mock_loads.return_value) + + mock_loads.reset_mock() + mock_loads.side_effect = ValueError + self.assertRaises(PluginExecutionError, pf.Index) + mock_loads.assert_called_with(pf.data) + + @patch("%s.dump" % JSON) + @patch("%s.open" % builtins) + def test__write(self, mock_open, mock_dump): + pf = self.get_obj() + self.assertTrue(pf._write()) + mock_open.assert_called_with(pf.name, 'wb') + mock_dump.assert_called_with(pf.json, mock_open.return_value) + + @patch("%s.dumps" % JSON) + def test_validate_data(self, mock_dumps): + pf = self.get_obj() + pf.validate_data() + mock_dumps.assert_called_with(pf.json) + + mock_dumps.reset_mock() + mock_dumps.side_effect = ValueError + self.assertRaises(PluginExecutionError, pf.validate_data) + mock_dumps.assert_called_with(pf.json) + + +class TestYAMLPropertyFile(TestFileBacked, TestPropertyFile): + test_obj = YAMLPropertyFile + + @skipUnless(HAS_YAML, "YAML libraries not found, skipping") + def setUp(self): + TestFileBacked.setUp(self) + TestPropertyFile.setUp(self) + + @patch("yaml.load") + def test_Index(self, mock_load): + pf = self.get_obj() + pf.Index() + mock_load.assert_called_with(pf.data) + self.assertEqual(pf.yaml, mock_load.return_value) + + mock_load.reset_mock() + mock_load.side_effect = yaml.YAMLError + self.assertRaises(PluginExecutionError, pf.Index) + mock_load.assert_called_with(pf.data) + + @patch("yaml.dump") + @patch("%s.open" % builtins) + def test__write(self, mock_open, mock_dump): + pf = self.get_obj() + self.assertTrue(pf._write()) + mock_open.assert_called_with(pf.name, 'wb') + mock_dump.assert_called_with(pf.yaml, mock_open.return_value) + + @patch("yaml.dump") + def test_validate_data(self, mock_dump): + pf = self.get_obj() + pf.validate_data() + mock_dump.assert_called_with(pf.yaml) + + mock_dump.reset_mock() + mock_dump.side_effect = yaml.YAMLError + self.assertRaises(PluginExecutionError, pf.validate_data) + mock_dump.assert_called_with(pf.yaml) class TestXMLPropertyFile(TestPropertyFile, TestStructFile): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py index 0bd69b371..45f3671e8 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestRules.py @@ -1,9 +1,11 @@ import os import sys +import copy import lxml.etree -import Bcfg2.Server.Plugin +import Bcfg2.Options from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Rules import * +from Bcfg2.Server.Plugin import PluginExecutionError # add all parent testsuite directories to sys.path to allow (most) # relative imports in python 2.4 @@ -15,68 +17,159 @@ while path != "/": break path = os.path.dirname(path) from common import * -from TestPlugin import TestPrioDir +from TestPlugin.Testhelpers import TestPrioDir class TestRules(TestPrioDir): test_obj = Rules + abstract = dict( + basic=lxml.etree.Element("Path", name="/etc/basic"), + unhandled=lxml.etree.Element("Path", name="/etc/unhandled"), + priority=lxml.etree.Element("Path", name="/etc/priority"), + content=lxml.etree.Element("Path", name="/etc/text-content"), + duplicate=lxml.etree.Element("SEBoolean", name="duplicate"), + group=lxml.etree.Element("SEPort", name="6789/tcp"), + children=lxml.etree.Element("Path", name="/etc/child-entries"), + regex=lxml.etree.Element("Package", name="regex"), + slash=lxml.etree.Element("Path", name="/etc/trailing/slash"), + no_slash=lxml.etree.Element("Path", name="/etc/no/trailing/slash/")) + + concrete = dict( + basic=lxml.etree.Element("Path", name="/etc/basic", type="directory", + owner="root", group="root", mode="0600"), + priority=lxml.etree.Element("Path", name="/etc/priority", + type="directory", owner="root", + group="root", mode="0600"), + content=lxml.etree.Element("Path", name="/etc/text-content", + type="file", owner="bar", group="bar", + mode="0644"), + duplicate=lxml.etree.Element("SEBoolean", name="duplicate", + value="on"), + group=lxml.etree.Element("SEPort", name="6789/tcp", + selinuxtype="bcfg2_server_t"), + children=lxml.etree.Element("Path", name="/etc/child-entries", + type="directory", owner="root", + group="root", mode="0775"), + regex=lxml.etree.Element("Package", name="regex", type="yum", + version="any"), + slash=lxml.etree.Element("Path", name="/etc/trailing/slash", + type="directory", owner="root", group="root", + mode="0600"), + no_slash=lxml.etree.Element("Path", name="/etc/no/trailing/slash/", + type="directory", owner="root", + group="root", mode="0600")) + + concrete['content'].text = "Text content" + lxml.etree.SubElement(concrete['children'], + "ACL", type="default", scope="user", user="foouser", + perms="rw") + lxml.etree.SubElement(concrete['children'], + "ACL", type="default", scope="group", group="users", + perms="rx") + + in_file = copy.deepcopy(concrete) + in_file['regex'].set("name", ".*") + in_file['slash'].set("name", "/etc/trailing/slash/") + in_file['no_slash'].set("name", "/etc/no/trailing/slash") + + rules1 = lxml.etree.Element("Rules", priority="10") + rules1.append(in_file['basic']) + lxml.etree.SubElement(rules1, "Path", name="/etc/priority", + type="directory", owner="foo", group="foo", + mode="0644") + foogroup = lxml.etree.SubElement(rules1, "Group", name="foogroup") + foogroup.append(in_file['group']) + rules1.append(in_file['content']) + rules1.append(copy.copy(in_file['duplicate'])) + + rules2 = lxml.etree.Element("Rules", priority="20") + rules2.append(in_file['priority']) + rules2.append(in_file['children']) + rules2.append(in_file['no_slash']) + + rules3 = lxml.etree.Element("Rules", priority="10") + rules3.append(in_file['duplicate']) + rules3.append(in_file['regex']) + rules3.append(in_file['slash']) + + rules = {"rules1.xml": rules1, "rules2.xml": rules2, "rules3.xml": rules3} + def setUp(self): TestPrioDir.setUp(self) + set_setup_default("lax_decryption", True) set_setup_default("rules_regex", False) - def test_HandlesEntry(self): + def get_child(self, name): + """ Turn one of the XML documents in `rules` into a child + object """ + filename = os.path.join(datastore, self.test_obj.name, name) + rv = self.test_obj.__child__(filename) + rv.data = lxml.etree.tostring(self.rules[name]) + rv.Index() + return rv + + def get_obj(self, core=None): + r = TestPrioDir.get_obj(self, core=core) + r.entries = dict([(n, self.get_child(n)) for n in self.rules.keys()]) + return r + + def _do_test(self, name, groups=None): + if groups is None: + groups = [] r = self.get_obj() - r.Entries = dict(Path={"/etc/foo.conf": Mock(), - "/etc/bar.conf": Mock()}) - r._matches = Mock() - metadata = Mock() - - entry = lxml.etree.Element("Path", name="/etc/foo.conf") - self.assertEqual(r.HandlesEntry(entry, metadata), - r._matches.return_value) - r._matches.assert_called_with(entry, metadata, - r.Entries['Path'].keys()) - - r._matches.reset_mock() - entry = lxml.etree.Element("Path", name="/etc/baz.conf") - self.assertEqual(r.HandlesEntry(entry, metadata), - r._matches.return_value) - r._matches.assert_called_with(entry, metadata, - r.Entries['Path'].keys()) - - r._matches.reset_mock() - entry = lxml.etree.Element("Package", name="foo") - self.assertFalse(r.HandlesEntry(entry, metadata)) - - @patch("Bcfg2.Server.Plugin.PrioDir._matches") - def test__matches(self, mock_matches): + metadata = Mock(groups=groups) + entry = copy.deepcopy(self.abstract[name]) + self.assertTrue(r.HandlesEntry(entry, metadata)) + r.HandleEntry(entry, metadata) + self.assertXMLEqual(entry, self.concrete[name]) + + def _do_test_failure(self, name, groups=None, handles=None): + if groups is None: + groups = [] r = self.get_obj() - metadata = Mock() - - # test parent _matches() returning True - entry = lxml.etree.Element("Path", name="/etc/foo.conf") - candidate = lxml.etree.Element("Path", name="/etc/bar.conf") - mock_matches.return_value = True - self.assertTrue(r._matches(entry, metadata, candidate)) - mock_matches.assert_called_with(r, entry, metadata, candidate) - - # test all conditions returning False - mock_matches.reset_mock() - mock_matches.return_value = False - self.assertFalse(r._matches(entry, metadata, candidate)) - mock_matches.assert_called_with(r, entry, metadata, candidate) - - # test special Path cases -- adding and removing trailing slash - mock_matches.reset_mock() - withslash = lxml.etree.Element("Path", name="/etc/foo") - withoutslash = lxml.etree.Element("Path", name="/etc/foo/") - self.assertTrue(r._matches(withslash, metadata, withoutslash)) - self.assertTrue(r._matches(withoutslash, metadata, withslash)) - - if r._regex_enabled: - mock_matches.reset_mock() - candidate = lxml.etree.Element("Path", name="/etc/.*\.conf") - self.assertTrue(r._matches(entry, metadata, candidate)) - mock_matches.assert_called_with(r, entry, metadata, candidate) - self.assertIn("/etc/.*\.conf", r._regex_cache.keys()) + metadata = Mock(groups=groups) + entry = self.abstract[name] + if handles is not None: + self.assertEqual(handles, r.HandlesEntry(entry, metadata)) + self.assertRaises(PluginExecutionError, + r.HandleEntry, entry, metadata) + + def test_basic(self): + """ Test basic Rules usage """ + self._do_test('basic') + self._do_test_failure('unhandled', handles=False) + + def test_priority(self): + """ Test that Rules respects priority """ + self._do_test('priority') + + def test_duplicate(self): + """ Test that Rules raises exceptions for duplicate entries """ + self._do_test_failure('duplicate') + + def test_content(self): + """ Test that Rules copies text content from concrete entries """ + self._do_test('content') + + def test_group(self): + """ Test that Rules respects <Group/> tags """ + self._do_test('group', groups=['foogroup']) + self._do_test_failure('group', groups=['bargroup'], handles=False) + + def test_children(self): + """ Test that Rules copies child elements from concrete entries """ + self._do_test('children') + + def test_regex(self): + """ Test that Rules handles regular expressions properly """ + Bcfg2.Options.setup.rules_regex = False + self._do_test_failure('regex', handles=False) + Bcfg2.Options.setup.rules_regex = True + self._do_test('regex') + Bcfg2.Options.setup.rules_regex = False + + def test_slash(self): + """ Test that Rules handles trailing slashes on Path entries """ + self._do_test('slash') + self._do_test('no_slash') diff --git a/testsuite/common.py b/testsuite/common.py index b375d3703..04c446f67 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -12,11 +12,13 @@ import os import re import sys import codecs -import unittest import lxml.etree import Bcfg2.Options from mock import patch, MagicMock, _patch, DEFAULT -from Bcfg2.Compat import wraps +try: + from unittest import skip, skipIf, skipUnless, TestCase +except ImportError: + from unittest2 import skip, skipIf, skipUnless, TestCase #: The path to the Bcfg2 specification root for the tests. Using the #: root directory exposes a lot of potential problems with building @@ -110,203 +112,46 @@ else: return codecs.unicode_escape_decode(s)[0] -#: Whether or not skipping tests is natively supported by -#: :mod:`unittest`. If it isn't, then we have to make tests that -#: would be skipped succeed instead. -can_skip = False - -if hasattr(unittest, "skip"): - can_skip = True - - #: skip decorator from :func:`unittest.skip` - skip = unittest.skip - - #: skipIf decorator from :func:`unittest.skipIf` - skipIf = unittest.skipIf - - #: skipUnless decorator from :func:`unittest.skipUnless` - skipUnless = unittest.skipUnless -else: - # we can't actually skip tests, we just make them pass - can_skip = False - - def skip(msg): - """ skip decorator used when :mod:`unittest` doesn't support - skipping tests. Replaces the decorated function with a - no-op. """ - def decorator(func): - return lambda *args, **kwargs: None - return decorator - - def skipIf(condition, msg): - """ skipIf decorator used when :mod:`unittest` doesn't support - skipping tests """ - if not condition: - return lambda f: f - else: - return skip(msg) - - def skipUnless(condition, msg): - """ skipUnless decorator used when :mod:`unittest` doesn't - support skipping tests """ - if condition: - return lambda f: f - else: - return skip(msg) - - -def _count_diff_all_purpose(actual, expected): - '''Returns list of (cnt_act, cnt_exp, elem) triples where the - counts differ''' - # elements need not be hashable - s, t = list(actual), list(expected) - m, n = len(s), len(t) - NULL = object() - result = [] - for i, elem in enumerate(s): - if elem is NULL: - continue - cnt_s = cnt_t = 0 - for j in range(i, m): - if s[j] == elem: - cnt_s += 1 - s[j] = NULL - for j, other_elem in enumerate(t): - if other_elem == elem: - cnt_t += 1 - t[j] = NULL - if cnt_s != cnt_t: - diff = (cnt_s, cnt_t, elem) - result.append(diff) - - for i, elem in enumerate(t): - if elem is NULL: - continue - cnt_t = 0 - for j in range(i, n): - if t[j] == elem: - cnt_t += 1 - t[j] = NULL - diff = (0, cnt_t, elem) - result.append(diff) - return result - - -def _assertion(predicate, default_msg=None): - @wraps(predicate) - def inner(self, *args, **kwargs): - if 'msg' in kwargs: - msg = kwargs['msg'] - del kwargs['msg'] - else: - try: - msg = default_msg % args - except TypeError: - # message passed as final (non-keyword) argument? - msg = args[-1] - args = args[:-1] - assert predicate(*args, **kwargs), msg - return inner - - -def _regex_matches(val, regex): - if hasattr(regex, 'search'): - return regex.search(val) - else: - return re.search(regex, val) - - -class Bcfg2TestCase(unittest.TestCase): +class Bcfg2TestCase(TestCase): """ Base TestCase class that inherits from - :class:`unittest.TestCase`. This class does a few things: - - * Adds :func:`assertXMLEqual`, a useful assertion method given all - the XML used by Bcfg2; - - * Defines convenience methods that were (mostly) added in Python - 2.7. + :class:`unittest.TestCase`. This class adds + :func:`assertXMLEqual`, a useful assertion method given all the + XML used by Bcfg2. """ - if not hasattr(unittest.TestCase, "assertItemsEqual"): - # TestCase in Py3k lacks assertItemsEqual, but has the other - # convenience methods. this code is (mostly) cribbed from the - # py2.7 unittest library - def assertItemsEqual(self, expected_seq, actual_seq, msg=None): - """ Implementation of - :func:`unittest.TestCase.assertItemsEqual` for python - versions that lack it """ - first_seq, second_seq = list(actual_seq), list(expected_seq) - differences = _count_diff_all_purpose(first_seq, second_seq) - - if differences: - standardMsg = 'Element counts were not equal:\n' - lines = ['First has %d, Second has %d: %r' % diff - for diff in differences] - diffMsg = '\n'.join(lines) - standardMsg = self._truncateMessage(standardMsg, diffMsg) - msg = self._formatMessage(msg, standardMsg) - self.fail(msg) - - if not hasattr(unittest.TestCase, "assertRegexpMatches"): - # Some versions of TestCase in Py3k seem to lack - # assertRegexpMatches, but have the other convenience methods. - assertRegexpMatches = _assertion(lambda s, r: _regex_matches(s, r), - "%s does not contain /%s/") - - if not hasattr(unittest.TestCase, "assertNotRegexpMatches"): - # Some versions of TestCase in Py3k seem to lack - # assertNotRegexpMatches even though they have - # assertRegexpMatches - assertNotRegexpMatches = \ - _assertion(lambda s, r: not _regex_matches(s, r), - "%s contains /%s/") - - if not hasattr(unittest.TestCase, "assertIn"): - # versions of TestCase before python 2.7 and python 3.1 lacked - # a lot of the really handy convenience methods, so we provide - # them -- at least the easy ones and the ones we use. - assertIs = _assertion(lambda a, b: a is b, "%s is not %s") - assertIsNot = _assertion(lambda a, b: a is not b, "%s is %s") - assertIsNone = _assertion(lambda x: x is None, "%s is not None") - assertIsNotNone = _assertion(lambda x: x is not None, "%s is None") - assertIn = _assertion(lambda a, b: a in b, "%s is not in %s") - assertNotIn = _assertion(lambda a, b: a not in b, "%s is in %s") - assertIsInstance = _assertion(isinstance, "%s is not instance of %s") - assertNotIsInstance = _assertion(lambda a, b: not isinstance(a, b), - "%s is instance of %s") - assertGreater = _assertion(lambda a, b: a > b, - "%s is not greater than %s") - assertGreaterEqual = _assertion(lambda a, b: a >= b, - "%s is not greater than or equal to %s") - assertLess = _assertion(lambda a, b: a < b, "%s is not less than %s") - assertLessEqual = _assertion(lambda a, b: a <= b, - "%s is not less than or equal to %s") - def assertXMLEqual(self, el1, el2, msg=None): - """ Test that the two XML trees given are equal. Both - elements and all children are expected to have ``name`` - attributes. """ + """ Test that the two XML trees given are equal. """ if msg is None: - msg = "XML trees were not equal" + msg = "XML trees are not equal: %s" + else: + msg += ": %s" fullmsg = msg + "\nFirst: %s" % lxml.etree.tostring(el1) + \ "\nSecond: %s" % lxml.etree.tostring(el2) - self.assertEqual(el1.tag, el2.tag, msg=fullmsg) - self.assertEqual(el1.text, el2.text, msg=fullmsg) + self.assertEqual(el1.tag, el2.tag, msg=fullmsg % "Tags differ") + self.assertEqual(el1.text, el2.text, + msg=fullmsg % "Text content differs") self.assertItemsEqual(el1.attrib.items(), el2.attrib.items(), - msg=fullmsg) + msg=fullmsg % "Attributes differ") self.assertEqual(len(el1.getchildren()), len(el2.getchildren()), - msg=fullmsg) + msg=fullmsg % "Different numbers of children") + matched = [] for child1 in el1.getchildren(): - cname = child1.get("name") - self.assertIsNotNone(cname, - msg="Element %s has no 'name' attribute" % - child1.tag) - children2 = el2.xpath("%s[@name='%s']" % (child1.tag, cname)) - self.assertEqual(len(children2), 1, - msg="More than one %s element named %s" % \ - (child1.tag, cname)) - self.assertXMLEqual(child1, children2[0], msg=msg) + for child2 in el2.xpath(child1.tag): + if child2 in matched: + continue + try: + self.assertXMLEqual(child1, child2) + matched.append(child2) + break + except AssertionError: + continue + else: + assert False, \ + fullmsg % ("Element %s is missing from second" % + lxml.etree.tostring(child1)) + self.assertItemsEqual(el2.getchildren(), matched, + msg=fullmsg % "Second has extra element(s)") class DBModelTestCase(Bcfg2TestCase): diff --git a/testsuite/install.sh b/testsuite/install.sh index 43a6057c0..6895034d5 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -6,6 +6,10 @@ pip install -r testsuite/requirements.txt --use-mirrors PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') +if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" ]]; then + pip install --use-mirrors unittest2 +fi + if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then pip install --use-mirrors PyYAML pyinotify if [[ $PYVER == "2.5" ]]; then diff --git a/tools/README b/tools/README index 9aef03104..6fc99bc6d 100644 --- a/tools/README +++ b/tools/README @@ -55,6 +55,9 @@ export.sh generate-manpages.bash - Generate man pages from the Sphinx source +git_commit.py + - Trigger script to commit local changes back to a git repository + pkgmgr_gen.py - Generate Pkgmgr XML files from a list of directories that contain RPMS diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 21b5ad8d4..5e5bca777 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -47,7 +47,10 @@ class LocalProxy(object): func = getattr(self.core, attr) if func.exposed: def inner(*args, **kwargs): - args = ((self.ipaddr, self.hostname), ) + args + # the port portion of the addresspair tuple isn't + # actually used, so it's safe to hardcode 6789 + # here. + args = ((self.ipaddr, 6789), ) + args return func(*args, **kwargs) return inner raise AttributeError(attr) diff --git a/tools/git_commit.py b/tools/git_commit.py new file mode 100755 index 000000000..cc4061f25 --- /dev/null +++ b/tools/git_commit.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +""" Trigger script to commit selected changes to a local repository +back to git. To use this script, enable the Trigger plugin, put this +script in /var/lib/bcfg2/Trigger/, and create /etc/bcfg2-commit.conf. + +The config file, /etc/bcfg2-commit.conf, may contain four options in +the [global] section: + +* "config" is the path to the Bcfg2 server config file. (Default: + /etc/bcfg2.conf) +* "commit" is a comma-separated list of globs giving the paths that + should be committed back to the repository. Default is 'SSLCA/*, + SSHbase/*, Cfg/*', which will commit data back for SSLCA, SSHbase, + Cfg, FileProbes, etc., but not, for instance, Probes/probed.xml. + You may wish to add Metadata/clients.xml to the commit list. +* "debug" and "verbose" let you set the log level for git_commit.py + itself. +""" + + +import os +import sys +import git +import logging +import Bcfg2.Logger +import Bcfg2.Options +from Bcfg2.Compat import ConfigParser +from fnmatch import fnmatch + +# config file path +CONFIG = "/etc/bcfg2-commit.conf" + +# config defaults. all config options are in the [global] section +DEFAULTS = dict(config='/etc/bcfg2.conf', + commit="SSLCA/*, SSHbase/*, Cfg/*") + + +def list_changed_files(repo): + return [d for d in repo.index.diff(None) + if (d.a_blob is not None and not d.deleted_file and + not d.renamed and not d.new_file)] + + +def add_to_commit(patterns, path, repo, relpath): + progname = os.path.basename(sys.argv[0]) + logger = logging.getLogger(progname) + for pattern in patterns: + if fnmatch(path, os.path.join(relpath, pattern)): + logger.debug("%s: Adding %s to commit" % (progname, path)) + repo.index.add([path]) + return True + return False + + +def parse_options(): + config = ConfigParser.SafeConfigParser(DEFAULTS) + config.read(CONFIG) + + optinfo = dict( + profile=Bcfg2.Options.CLIENT_PROFILE, + dryrun=Bcfg2.Options.CLIENT_DRYRUN, + groups=Bcfg2.Options.Option("Groups", + default=[], + cmd="-g", + odesc='<group>:<group>', + cook=Bcfg2.Options.colon_split)) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + argv = [Bcfg2.Options.CFILE.cmd, config.get("global", "config")] + argv.extend(sys.argv[1:]) + setup = Bcfg2.Options.OptionParser(optinfo, argv=argv) + setup.parse(argv) + + setup['commit'] = Bcfg2.Options.list_split(config.get("global", + "commit")) + for opt in ['debug', 'verbose']: + try: + setup[opt] = config.getboolean("global", opt) + except ConfigParser.NoOptionError: + pass + + try: + hostname = setup['args'][0] + except IndexError: + print(setup.hm) + raise SystemExit(1) + return (setup, hostname) + + +def setup_logging(setup): + progname = os.path.basename(sys.argv[0]) + log_args = dict(to_syslog=setup['syslog'], to_console=sys.stdout.isatty(), + to_file=setup['logging'], level=logging.WARNING) + if setup['debug']: + log_args['level'] = logging.DEBUG + elif setup['verbose']: + log_args['level'] = logging.INFO + Bcfg2.Logger.setup_logging(progname, **log_args) + return logging.getLogger(progname) + + +def main(): + progname = os.path.basename(sys.argv[0]) + setup, hostname = parse_options() + logger = setup_logging(setup) + if setup['dryrun']: + logger.info("%s: In dry-run mode, changes will not be committed" % + progname) + + if setup['vcs_root']: + gitroot = os.path.realpath(setup['vcs_root']) + else: + gitroot = os.path.realpath(setup['repo']) + logger.info("%s: Using Git repo at %s" % (progname, gitroot)) + try: + repo = git.Repo(gitroot) + except: # pylint: disable=W0702 + logger.error("%s: Error setting up Git repo at %s: %s" % + (progname, gitroot, sys.exc_info()[1])) + return 1 + + # canonicalize the repo path so that git will recognize it as + # being inside the git repo + bcfg2root = os.path.realpath(setup['repo']) + + if not bcfg2root.startswith(gitroot): + logger.error("%s: Bcfg2 repo %s is not inside Git repo %s" % + (progname, bcfg2root, gitroot)) + return 1 + + # relative path to Bcfg2 root from VCS root + if gitroot == bcfg2root: + relpath = '' + else: + relpath = bcfg2root[len(gitroot) + 1:] + + new = 0 + changed = 0 + logger.debug("%s: Untracked files: %s" % (progname, repo.untracked_files)) + for path in repo.untracked_files: + if add_to_commit(setup['commit'], path, repo, relpath): + new += 1 + else: + logger.debug("%s: Not adding %s to commit" % (progname, path)) + logger.debug("%s: Untracked files after building commit: %s" % + (progname, repo.untracked_files)) + + changes = list_changed_files(repo) + logger.info("%s: Changed files: %s" % (progname, + [d.a_blob.path for d in changes])) + for diff in changes: + if add_to_commit(setup['commit'], diff.a_blob.path, repo, relpath): + changed += 1 + else: + logger.debug("%s: Not adding %s to commit" % (progname, + diff.a_blob.path)) + logger.info("%s: Changed files after building commit: %s" % + (progname, [d.a_blob.path for d in list_changed_files(repo)])) + + if new + changed > 0: + logger.debug("%s: Committing %s new files and %s changed files" % + (progname, new, changed)) + if setup['dryrun']: + logger.warning("%s: In dry-run mode, skipping commit and push" % + progname) + else: + output = repo.index.commit("Auto-commit with %s from %s run" % + (progname, hostname)) + if output: + logger.debug("%s: %s" % (progname, output)) + remote = repo.remote() + logger.debug("%s: Pushing to remote %s at %s" % (progname, remote, + remote.url)) + output = remote.push() + if output: + logger.debug("%s: %s" % (progname, output)) + else: + logger.info("%s: No changes to commit" % progname) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/upgrade/1.4/README b/tools/upgrade/1.4/README index 8dde8b8b5..b03cb9b74 100644 --- a/tools/upgrade/1.4/README +++ b/tools/upgrade/1.4/README @@ -6,5 +6,9 @@ migrate_decisions.py files into structured XML convert_bundles.py - - Remove deprecated explicit bundle names, renames .genshi bundles + - Remove deprecated explicit bundle names, rename .genshi bundles to .xml + +migrate_sslca.py + - Migrate from the standalone SSLCA plugin to the built-in SSL + certificate generation abilities of the Cfg plugin
\ No newline at end of file diff --git a/tools/upgrade/1.4/migrate_sslca.py b/tools/upgrade/1.4/migrate_sslca.py new file mode 100755 index 000000000..958228c86 --- /dev/null +++ b/tools/upgrade/1.4/migrate_sslca.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +import os +import sys +import shutil +import Bcfg2.Options + + +def main(): + parser = Bcfg2.Options.get_parser( + description="Migrate from the SSLCA plugin to built-in Cfg SSL cert " + "generation") + parser.add_options([Bcfg2.Options.Common.repository]) + parser.parse() + + sslcadir = os.path.join(Bcfg2.Options.setup.repository, 'SSLCA') + cfgdir = os.path.join(Bcfg2.Options.setup.repository, 'Cfg') + for root, _, files in os.walk(sslcadir): + if not files: + continue + newpath = cfgdir + root[len(sslcadir):] + if not os.path.exists(newpath): + print("Creating %s and copying contents from %s" % (newpath, root)) + shutil.copytree(root, newpath) + else: + print("Copying contents from %s to %s" % (root, newpath)) + for fname in files: + newfpath = os.path.exists(os.path.join(newpath, fname)) + if newfpath: + print("%s already exists, skipping" % newfpath) + else: + shutil.copy(os.path.join(root, fname), newpath) + cert = os.path.join(newpath, "cert.xml") + newcert = os.path.join(newpath, "sslcert.xml") + key = os.path.join(newpath, "key.xml") + newkey = os.path.join(newpath, "sslkey.xml") + if os.path.exists(cert): + os.rename(cert, newcert) + if os.path.exists(key): + os.rename(key, newkey) + + +if __name__ == '__main__': + sys.exit(main()) |